Merge branch 'main' into bmz-UID2-4588-remote-config-endpoint

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.uid2</groupId>
     <artifactId>uid2-core</artifactId>
-    <version>2.22.0</version>
+    <version>2.23.17</version>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -24,7 +24,7 @@
         <vertx.verticle>com.uid2.core.vertx.CoreVerticle</vertx.verticle>
         <launcher.class>io.vertx.core.Launcher</launcher.class>
 
-        <uid2-shared.version>8.0.0</uid2-shared.version>
+        <uid2-shared.version>8.0.32</uid2-shared.version>
         <image.version>${project.version}</image.version>
     </properties>
 

src/main/java/com/uid2/core/Main.java

@@ -25,13 +25,15 @@
 import com.uid2.shared.secure.nitro.InMemoryAWSCertificateStore;
 import com.uid2.shared.store.CloudPath;
 import com.uid2.shared.store.scope.GlobalScope;
+import com.uid2.shared.util.HTTPPathMetricFilter;
 import com.uid2.shared.vertx.RotatingStoreVerticle;
 import com.uid2.shared.vertx.VertxUtils;
 import io.micrometer.core.instrument.Gauge;
 import io.micrometer.core.instrument.Metrics;
 import io.micrometer.core.instrument.config.MeterFilter;
 import io.micrometer.prometheus.PrometheusMeterRegistry;
 import io.micrometer.prometheus.PrometheusRenameFilter;
+import io.vertx.core.DeploymentOptions;
 import io.vertx.core.Vertx;
 import io.vertx.core.VertxOptions;
 import io.vertx.core.http.HttpServerOptions;
@@ -49,6 +51,8 @@
 
 public class Main {
 
+    private static final int vertxServiceInstances = 1;
+
     public static void main(String[] args) {
         final String vertxConfigPath = System.getProperty(Const.Config.VERTX_CONFIG_PATH_PROP);
         if (vertxConfigPath != null) {
@@ -157,17 +161,19 @@ public static void main(String[] args) {
                 );
 
                 JwtService jwtService = new JwtService(config);
-
                 coreVerticle = new CoreVerticle(cloudStorage, operatorKeyProvider, attestationService, attestationTokenService, enclaveIdProvider, operatorJWTTokenProvider, jwtService, cloudEncryptionKeyProvider);
             } catch (Exception e) {
                 System.out.println("failed to initialize core verticle: " + e.getMessage());
                 System.exit(-1);
             }
 
+            createVertxInstancesMetric();
+            createVertxEventLoopsMetric();
+
             vertx.deployVerticle(enclaveRotatingVerticle);
             vertx.deployVerticle(operatorRotatingVerticle);
             vertx.deployVerticle(cloudEncryptionKeyRotatingVerticle);
-            vertx.deployVerticle(coreVerticle);
+            vertx.deployVerticle(coreVerticle, new DeploymentOptions().setInstances(vertxServiceInstances));
         });
     }
 
@@ -182,14 +188,8 @@ private static void setupMetrics(MicrometerMetricsOptions metricOptions) {
             prometheusRegistry.config()
                     // providing common renaming for prometheus metric, e.g. "hello.world" to "hello_world"
                     .meterFilter(new PrometheusRenameFilter())
-                    .meterFilter(MeterFilter.replaceTagValues(Label.HTTP_PATH.toString(), actualPath -> {
-                        try {
-                            String normalized = HttpUtils.normalizePath(actualPath).split("\\?")[0];
-                            return Endpoints.pathSet().contains(normalized) ? normalized : "/unknown";
-                        } catch (IllegalArgumentException e) {
-                            return actualPath;
-                        }
-                    }))
+                    .meterFilter(MeterFilter.replaceTagValues(Label.HTTP_PATH.toString(),
+                            actualPath -> HTTPPathMetricFilter.filterPath(actualPath, Endpoints.pathSet())))
                     // Don't record metrics for 404s.
                     .meterFilter(MeterFilter.deny(id ->
                         id.getName().startsWith(MetricsDomain.HTTP_SERVER.getPrefix()) &&
@@ -210,6 +210,19 @@ private static void setupMetrics(MicrometerMetricsOptions metricOptions) {
                 .register(Metrics.globalRegistry);
     }
 
+    private static void createVertxInstancesMetric() {
+        Gauge.builder("uid2.vertx_service_instances", () -> vertxServiceInstances)
+                .description("gauge for number of vertx service instances requested")
+                .register(Metrics.globalRegistry);
+    }
+
+    private static void createVertxEventLoopsMetric() {
+        Gauge.builder("uid2.vertx_event_loop_threads", () -> VertxOptions.DEFAULT_EVENT_LOOP_POOL_SIZE)
+                .description("gauge for number of vertx event loop threads")
+                .register(Metrics.globalRegistry);
+    }
+
+
     /*
     private static CommandLine parseArgs(String[] args) {
         final CLI cli = CLI.create("uid2-core")

src/main/java/com/uid2/core/service/ClientSideKeypairMetadataProvider.java

@@ -1,6 +1,7 @@
 package com.uid2.core.service;
 
 import com.uid2.core.model.SecretStore;
+import com.uid2.core.util.OperatorInfo;
 import com.uid2.shared.cloud.ICloudStorage;
 import com.uid2.shared.store.CloudPath;
 import com.uid2.shared.store.scope.GlobalScope;
@@ -11,6 +12,7 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 
+import static com.uid2.core.util.MetadataHelper.getMetadataPathName;
 import static com.uid2.core.util.MetadataHelper.readToEndAsString;
 
 public class ClientSideKeypairMetadataProvider implements IClientSideKeypairMetadataProvider {
@@ -24,8 +26,8 @@ public ClientSideKeypairMetadataProvider(ICloudStorage cloudStorage) {
     }
 
     @Override
-    public String getMetadata() throws Exception {
-        String pathname = new GlobalScope(new CloudPath(SecretStore.Global.get(ClientSideKeypairMetadataPathName))).getMetadataPath().toString();
+    public String getMetadata(OperatorInfo info) throws Exception {
+        String pathname = getMetadataPathName(info, SecretStore.Global.get(ClientSideKeypairMetadataPathName));
         String original = readToEndAsString(metadataStreamProvider.download(pathname));
         JsonObject main = (JsonObject) Json.decodeValue(original);
         JsonObject obj = main.getJsonObject("client_side_keypairs");

src/main/java/com/uid2/core/service/IClientSideKeypairMetadataProvider.java

@@ -1,6 +1,8 @@
 package com.uid2.core.service;
 
 
+import com.uid2.core.util.OperatorInfo;
+
 public interface IClientSideKeypairMetadataProvider {
-    String getMetadata() throws Exception;
+    String getMetadata(OperatorInfo info) throws Exception;
 }

src/main/java/com/uid2/core/service/ISaltMetadataProvider.java

@@ -1,5 +1,7 @@
 package com.uid2.core.service;
 
+import com.uid2.core.util.OperatorInfo;
+
 public interface ISaltMetadataProvider {
-    String getMetadata() throws Exception;
+    String getMetadata(OperatorInfo info) throws Exception;
 }

src/main/java/com/uid2/core/service/SaltMetadataProvider.java

@@ -1,17 +1,22 @@
 package com.uid2.core.service;
 
 import com.uid2.core.model.SecretStore;
+import com.uid2.core.util.OperatorInfo;
 import com.uid2.shared.cloud.ICloudStorage;
 import io.vertx.core.json.Json;
 import io.vertx.core.json.JsonArray;
 import io.vertx.core.json.JsonObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import static com.uid2.core.util.MetadataHelper.readToEndAsString;
+
+import static com.uid2.core.util.MetadataHelper.*;
 
 public class SaltMetadataProvider implements ISaltMetadataProvider {
+    private static final Logger LOGGER = LoggerFactory.getLogger(SaltMetadataProvider.class);
 
     public static final String SaltsMetadataPathName = "salts_metadata_path";
 
@@ -28,8 +33,9 @@ public SaltMetadataProvider(ICloudStorage fileStreamProvider, ICloudStorage down
     }
 
     @Override
-    public String getMetadata() throws Exception {
-        String original = readToEndAsString(metadataStreamProvider.download(SecretStore.Global.get(SaltsMetadataPathName)));
+    public String getMetadata(OperatorInfo info) throws Exception {
+        String pathname = getMetadataPathNameOldPrivateNoSite(info, SecretStore.Global.get(SaltsMetadataPathName));
+        String original = readToEndAsString(metadataStreamProvider.download(pathname));
         JsonObject main = (JsonObject) Json.decodeValue(original);
         JsonArray salts = main.getJsonArray("salts");
         for(int i=0;i<salts.size();++i) {

src/main/java/com/uid2/core/util/MetadataHelper.java

@@ -55,6 +55,25 @@ public static String getMetadataPathName(OperatorType operatorType, int siteId,
         return store.getMetadataPath().toString();
     }
 
+    // This exists because salts were never split into site folders for private operators.
+    public static String getMetadataPathNameOldPrivateNoSite(OperatorInfo info, String metadataPathName) {
+        StoreScope store;
+        Boolean providePrivateSiteData = ConfigStore.Global.getBoolean("provide_private_site_data");
+        if (info.getSupportsEncryption()) { // Check if decryption is possible
+            if (info.getOperatorType() == OperatorType.PUBLIC ) //siteId_public folder
+            {
+                store = new EncryptedScope(new CloudPath(metadataPathName), info.getSiteId(), true);
+            } else //siteId_private folder
+            {
+                store = new EncryptedScope(new CloudPath(metadataPathName), info.getSiteId(), false);
+            }
+        } else {
+            store = new GlobalScope(new CloudPath(metadataPathName));
+        }
+
+        return store.getMetadataPath().toString();
+    }
+
     public static String readToEndAsString(InputStream stream) throws IOException {
         final InputStreamReader reader = new InputStreamReader(stream);
         final char[] buff = new char[1024];

src/main/java/com/uid2/core/vertx/CoreVerticle.java

@@ -415,8 +415,9 @@ private void handleSiteRefresh(RoutingContext rc) {
 
     private void handleSaltRefresh(RoutingContext rc) {
         try {
+            OperatorInfo info = OperatorInfo.getOperatorInfo(rc);
             rc.response().putHeader(HttpHeaders.CONTENT_TYPE, "application/json")
-                    .end(saltMetadataProvider.getMetadata());
+                    .end(saltMetadataProvider.getMetadata(info));
         } catch (Exception e) {
             logger.warn("exception in handleSaltRefresh: " + e.getMessage(), e);
             Error("error", 500, rc, "error processing salt refresh");
@@ -486,7 +487,7 @@ private void handleClientSideKeypairRefresh(RoutingContext rc) {
                 return;
             }
             rc.response().putHeader(HttpHeaders.CONTENT_TYPE, "application/json")
-                    .end(clientSideKeypairMetadataProvider.getMetadata());
+                    .end(clientSideKeypairMetadataProvider.getMetadata(info));
         } catch (Exception e) {
             logger.warn("exception in handleClientSideKeypairRefresh: " + e.getMessage(), e);
             Error("error", 500, rc, "error processing client_side_keypairs refresh");
@@ -644,7 +645,7 @@ void handleCloudEncryptionKeysRetrieval(RoutingContext rc) {
             }
 
             JsonObject response = new JsonObject()
-                    .put("cloudEncryptionKeys", new JsonArray(cloudEncryptionKeys));
+                    .put("cloud_encryption_keys", new JsonArray(cloudEncryptionKeys));
 
             rc.response().putHeader(HttpHeaders.CONTENT_TYPE, "application/json")
                     .end(response.encode());

src/test/java/com/uid2/core/vertx/TestCoreVerticle.java

@@ -674,7 +674,7 @@ void cloudEncryptionKeyRetrieveSuccess(Vertx vertx, VertxTestContext testContext
                     assertEquals(200, response.statusCode());
 
                     JsonObject json = response.bodyAsJsonObject();
-                    JsonArray cloudEncryptionKeysArray = json.getJsonArray("cloudEncryptionKeys");
+                    JsonArray cloudEncryptionKeysArray = json.getJsonArray("cloud_encryption_keys");
 
                     assertNotNull(cloudEncryptionKeysArray);
                     assertEquals(1, cloudEncryptionKeysArray.size());
@@ -722,7 +722,7 @@ void cloudEncryptionKeyRetrieveSuccessWithThreeKeys(Vertx vertx, VertxTestContex
                     assertEquals(200, response.statusCode());
 
                     JsonObject json = response.bodyAsJsonObject();
-                    JsonArray cloudEncryptionKeysArray = json.getJsonArray("cloudEncryptionKeys");
+                    JsonArray cloudEncryptionKeysArray = json.getJsonArray("cloud_encryption_keys");
 
                     assertNotNull(cloudEncryptionKeysArray);
                     assertEquals(3, cloudEncryptionKeysArray.size());

version.json

@@ -1 +1 @@
-{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "2.22", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } }
+{ "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", "version": "2.23", "publicReleaseRefSpec": [ "^refs/heads/master$", "^refs/heads/v\\d+(?:\\.\\d+)?$" ], "cloudBuild": { "setVersionVariables": true, "buildNumber": { "enabled": true, "includeCommitId": { "when": "always" } } } }