fix: Semantic of onboarded Gateways in the multitenancy deployment (#…

…3884) * draft Signed-off-by: Pavel Jareš <[email protected]> * InstanceRetrievalServiceTest Signed-off-by: Pavel Jareš <[email protected]> * CachedProductFamilyServiceTest Signed-off-by: Pavel Jareš <[email protected]> * verify routes update Signed-off-by: ac892247 <[email protected]> * update routes Signed-off-by: ac892247 <[email protected]> * fix proper instance of Gateway in the services - ie. to fix login to central domain (InstanceLookupExecutor + InstanceLookupExecutorTest) Signed-off-by: Pavel Jareš <[email protected]> * verify that catalog can work with multiple gateways registered Signed-off-by: ac892247 <[email protected]> * new amount of gateways Signed-off-by: ac892247 <[email protected]> * wait for all services to start Signed-off-by: ac892247 <[email protected]> * specify zaas 2 hostname Signed-off-by: ac892247 <[email protected]> --------- Signed-off-by: Pavel Jareš <[email protected]> Signed-off-by: ac892247 <[email protected]> Co-authored-by: ac892247 <[email protected]>
zowe · Nov 12, 2024 · a94029b · a94029b
1 parent 5912607
commit a94029b
Show file tree

Hide file tree

Showing 17 changed files with 826 additions and 374 deletions.
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -307,8 +307,8 @@ jobs:
                     APIML_SERVICE_PORT: 10037
                     APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10031/eureka/
                     ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_DISCOVERYSERVICEURLS: https://discovery-service:10011/eureka
-                    ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_ROUTES_GATEWAYURL: /
-                    ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_ROUTES_SERVICEURL: /
+                    ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_ROUTES_0_GATEWAYURL: /
+                    ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_ROUTES_0_SERVICEURL: /
             zaas-service-2:
                 image: ghcr.io/balhar-jakub/zaas-service:${{ github.run_id }}-${{ github.run_number }}
                 env:
@@ -325,8 +325,8 @@ jobs:
                     APIML_SECURITY_X509_REGISTRY_ALLOWEDUSERS: USER,UNKNOWNUSER
                     APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10031/eureka/
                     ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_DISCOVERYSERVICEURLS: https://discovery-service:10011/eureka
-                    ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_ROUTES_GATEWAYURL: /
-                    ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_ROUTES_SERVICEURL: /
+                    ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_ROUTES_0_GATEWAYURL: /
+                    ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_ROUTES_0_SERVICEURL: /
 
         steps:
             -   uses: actions/checkout@v4
@@ -1098,7 +1098,7 @@ jobs:
 
             -   uses: ./.github/actions/teardown
 
-    CITestsDicoverableClientChaoticHA:
+    CITestsDiscoverableClientChaoticHA:
         needs: PublishJibContainers
         container: ubuntu:latest
         runs-on: ubuntu-latest
@@ -1169,7 +1169,7 @@ jobs:
                 uses: actions/upload-artifact@v4
                 if: always()
                 with:
-                    name: CITestsDicoverableClientChaoticHA-${{ env.JOB_ID }}
+                    name: CITestsDiscoverableClientChaoticHA-${{ env.JOB_ID }}
                     path: |
                         integration-tests/build/reports/**
 
@@ -1366,11 +1366,40 @@ jobs:
                     SPRING_SECURITY_OAUTH2_CLIENT_PROVIDER_OKTA_USERNAMEATTRIBUTE: sub
                     SPRING_SECURITY_OAUTH2_CLIENT_PROVIDER_OKTA_JWKSETURI: ${{ secrets.OKTA_JWKSET_URI }}
                     APIML_SECURITY_OIDC_COOKIE_SAMESITE: None
+                    APIML_HEALTH_PROTECTED: false
             zaas-service:
                 image: ghcr.io/balhar-jakub/zaas-service:${{ github.run_id }}-${{ github.run_number }}
+                env:
+                    APIML_HEALTH_PROTECTED: false
             mock-services:
                 image: ghcr.io/balhar-jakub/mock-services:${{ github.run_id }}-${{ github.run_number }}
 
+            discovery-service-2:
+                image: ghcr.io/balhar-jakub/discovery-service:${{ github.run_id }}-${{ github.run_number }}
+                volumes:
+                    - /api-defs:/api-defs
+                env:
+                    APIML_SERVICE_HOSTNAME: discovery-service-2
+            gateway-service-2:
+                image: ghcr.io/balhar-jakub/gateway-service:${{ github.run_id }}-${{ github.run_number }}
+                env:
+                    APIML_SERVICE_APIMLID: apiml2
+                    APIML_SERVICE_HOSTNAME: gateway-service-2
+                    APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka/
+                    APIML_HEALTH_PROTECTED: false
+                    ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_DISCOVERYSERVICEURLS: https://discovery-service:10011/eureka
+                    ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_ROUTES_0_GATEWAYURL: /
+                    ZWE_CONFIGS_APIML_SERVICE_ADDITIONALREGISTRATION_0_ROUTES_0_SERVICEURL: /
+            zaas-service-2:
+                image: ghcr.io/balhar-jakub/zaas-service:${{ github.run_id }}-${{ github.run_number }}
+                env:
+                    APIML_SECURITY_X509_ENABLED: true
+                    APIML_SECURITY_X509_ACCEPTFORWARDEDCERT: true
+                    APIML_SECURITY_X509_CERTIFICATESURL: https://gateway-service-2:10010/gateway/certificates
+                    APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka/
+                    APIML_HEALTH_PROTECTED: false
+                    APIML_SERVICE_HOSTNAME: zaas-service-2
+
         steps:
             -   uses: actions/checkout@v4
                 with:
@@ -1388,6 +1417,9 @@ jobs:
                         ~/.cache/Cypress
                         api-catalog-ui/frontend/node_modules
                     key: my-cache-${{ runner.os }}-${{ hashFiles('api-catalog-ui/frontend/*.json') }}
+            -   name: Run startup check
+                run: >
+                    ./gradlew runStartUpCheck --info --scan -Denvironment.config=-ha -Ddiscoverableclient.instances=1
             -   name: Cypress run API Catalog
                 run: |
                     cd api-catalog-ui/frontend

diff --git a/...g-services/src/main/java/org/zowe/apiml/apicatalog/instance/InstanceRetrievalService.java b/...g-services/src/main/java/org/zowe/apiml/apicatalog/instance/InstanceRetrievalService.java
@@ -15,9 +15,11 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.netflix.appinfo.InstanceInfo;
 import com.netflix.discovery.converters.jackson.EurekaJsonJacksonCodec;
+import com.netflix.discovery.shared.Application;
 import com.netflix.discovery.shared.Applications;
 import jakarta.validation.constraints.NotBlank;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hc.client5.http.classic.methods.HttpGet;
 import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
 import org.apache.hc.core5.http.Header;
@@ -30,16 +32,19 @@
 import org.springframework.http.MediaType;
 import org.springframework.stereotype.Service;
 import org.zowe.apiml.apicatalog.discovery.DiscoveryConfigProperties;
+import org.zowe.apiml.constants.EurekaMetadataDefinition;
 import org.zowe.apiml.message.log.ApimlLogger;
 import org.zowe.apiml.product.instance.InstanceInitializationException;
 import org.zowe.apiml.product.logging.annotations.InjectApimlLogger;
 import org.zowe.apiml.product.registry.ApplicationWrapper;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Base64;
-import java.util.List;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Predicate;
+
+import static org.zowe.apiml.product.constants.CoreService.GATEWAY;
 
 /**
  * Service for instance retrieval from Eureka
@@ -58,40 +63,63 @@ public class InstanceRetrievalService {
     @InjectApimlLogger
     private final ApimlLogger apimlLog = ApimlLogger.empty();
 
+    private ObjectMapper mapper = new ObjectMapper();
+
     @Autowired
     public InstanceRetrievalService(DiscoveryConfigProperties discoveryConfigProperties,
                                     CloseableHttpClient httpClient) {
         this.discoveryConfigProperties = discoveryConfigProperties;
         this.httpClient = httpClient;
     }
 
-    /**
-     * Retrieves {@link InstanceInfo} of particular service
-     *
-     * @param serviceId the service to search for
-     * @return service instance
-     */
-    public InstanceInfo getInstanceInfo(@NotBlank(message = "Service Id must be supplied") String serviceId) {
-        if (serviceId.equalsIgnoreCase(UNKNOWN)) {
-            return null;
-        }
-
+    private InstanceInfo getInstanceInfo(String serviceId, AtomicBoolean instanceFound, Predicate<InstanceInfo> selector) {
         List<EurekaServiceInstanceRequest> eurekaServiceInstanceRequests = constructServiceInfoQueryRequest(serviceId, false);
         // iterate over list of discovery services, return at first success
         for (EurekaServiceInstanceRequest eurekaServiceInstanceRequest : eurekaServiceInstanceRequests) {
             // call Eureka REST endpoint to fetch single or all Instances
             try {
                 String responseBody = queryDiscoveryForInstances(eurekaServiceInstanceRequest);
                 if (responseBody != null) {
-                    return extractSingleInstanceFromApplication(serviceId, responseBody);
+                    instanceFound.set(true);
+                    return extractSingleInstanceFromApplication(serviceId, responseBody, selector);
                 }
             } catch (Exception e) {
                 log.debug("Error obtaining instance information from {}, error message: {}",
                     eurekaServiceInstanceRequest.getEurekaRequestUrl(), e.getMessage());
             }
         }
-        String msg = "An error occurred when trying to get instance info for:  " + serviceId;
-        throw new InstanceInitializationException(msg);
+        return null;
+    }
+
+    /**
+     * Retrieves {@link InstanceInfo} of particular service
+     *
+     * @param serviceId the service to search for
+     * @return service instance
+     */
+    public InstanceInfo getInstanceInfo(@NotBlank(message = "Service Id must be supplied") String serviceId) {
+        if (serviceId.equalsIgnoreCase(UNKNOWN)) {
+            return null;
+        }
+
+        // identification if there was no instance or any error happened during fetching
+        AtomicBoolean instanceFound = new AtomicBoolean(false);
+        InstanceInfo instanceInfo = getInstanceInfo(serviceId, instanceFound,
+            ii -> EurekaMetadataDefinition.RegistrationType.of(ii.getMetadata()).isPrimary()
+        );
+        if (instanceInfo == null) {
+            // maybe the input is apimlId, try to find the matching Gateway (multi-tenancy use case)
+            instanceInfo = getInstanceInfo(GATEWAY.getServiceId(), instanceFound,
+                ii -> EurekaMetadataDefinition.RegistrationType.of(ii.getMetadata()).isAdditional()
+            );
+        }
+
+        if (!instanceFound.get()) {
+            String msg = "An error occurred when trying to get instance info for:  " + serviceId;
+            throw new InstanceInitializationException(msg);
+        }
+
+        return instanceInfo;
     }
 
     /**
@@ -141,7 +169,7 @@ private Applications extractApplications(String responseBody) {
      * @param eurekaServiceInstanceRequest information used to query the discovery service
      * @return ResponseEntity<String> query response
      */
-    private String queryDiscoveryForInstances(EurekaServiceInstanceRequest eurekaServiceInstanceRequest) throws IOException {
+    String queryDiscoveryForInstances(EurekaServiceInstanceRequest eurekaServiceInstanceRequest) throws IOException {
         HttpGet httpGet = new HttpGet(eurekaServiceInstanceRequest.getEurekaRequestUrl());
         for (Header header : createRequestHeader(eurekaServiceInstanceRequest)) {
             httpGet.setHeader(header);
@@ -176,25 +204,22 @@ private String queryDiscoveryForInstances(EurekaServiceInstanceRequest eurekaSer
      * @param responseBody the fetch attempt response body
      * @return service instance
      */
-    private InstanceInfo extractSingleInstanceFromApplication(String serviceId, String responseBody) {
+    InstanceInfo extractSingleInstanceFromApplication(String serviceId, String responseBody, Predicate<InstanceInfo> selector) {
         ApplicationWrapper application = null;
-        ObjectMapper mapper = new ObjectMapper();
         mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
         try {
             application = mapper.readValue(responseBody, ApplicationWrapper.class);
         } catch (IOException e) {
             log.debug("Could not extract service: {} info from discovery --{}", serviceId, e.getMessage(), e);
         }
 
-
-        if (application != null
-            && application.getApplication() != null
-            && application.getApplication().getInstances() != null
-            && !application.getApplication().getInstances().isEmpty()) {
-            return application.getApplication().getInstances().get(0);
-        } else {
-            return null;
-        }
+        return Optional.ofNullable(application)
+            .map(ApplicationWrapper::getApplication)
+            .map(Application::getInstances)
+            .orElse(Collections.emptyList())
+            .stream().filter(selector)
+            .findFirst()
+            .orElse(null);
     }
 
     /**
@@ -221,7 +246,7 @@ private List<EurekaServiceInstanceRequest> constructServiceInfoQueryRequest(Stri
 
             log.debug("Querying instance information of the service {} from the URL {} with the user {} and password {}",
                 serviceId, discoveryServiceLocatorUrl, eurekaUsername,
-                eurekaUserPassword.isEmpty() ? "NO PASSWORD" : "*******");
+                StringUtils.isEmpty(eurekaUserPassword) ? "NO PASSWORD" : "*******");
 
             EurekaServiceInstanceRequest eurekaServiceInstanceRequest = EurekaServiceInstanceRequest.builder()
                 .serviceId(serviceId)

diff --git a/...s/src/main/java/org/zowe/apiml/apicatalog/services/cached/CachedProductFamilyService.java b/...s/src/main/java/org/zowe/apiml/apicatalog/services/cached/CachedProductFamilyService.java
@@ -14,6 +14,7 @@
 import com.netflix.discovery.shared.Application;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.BooleanUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.zowe.apiml.apicatalog.model.APIContainer;
@@ -24,7 +25,6 @@
 import org.zowe.apiml.config.ApiInfo;
 import org.zowe.apiml.eurekaservice.client.util.EurekaMetadataParser;
 import org.zowe.apiml.message.log.ApimlLogger;
-import org.zowe.apiml.product.constants.CoreService;
 import org.zowe.apiml.product.logging.annotations.InjectApimlLogger;
 import org.zowe.apiml.product.routing.RoutedServices;
 import org.zowe.apiml.product.routing.ServiceType;
@@ -35,6 +35,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.zowe.apiml.constants.EurekaMetadataDefinition.*;
+import static org.zowe.apiml.product.constants.CoreService.GATEWAY;
 
 /**
  * Caching service for eureka services
@@ -67,8 +68,7 @@ public class CachedProductFamilyService {
 
     public CachedProductFamilyService(CachedServicesService cachedServicesService,
                                       TransformService transformService,
-                                      @Value("${apiml.service-registry.cacheRefreshUpdateThresholdInMillis}")
-                                          Integer cacheRefreshUpdateThresholdInMillis,
+                                      @Value("${apiml.service-registry.cacheRefreshUpdateThresholdInMillis}") Integer cacheRefreshUpdateThresholdInMillis,
                                       CustomStyleConfig customStyleConfig) {
         this.cachedServicesService = cachedServicesService;
         this.transformService = transformService;
@@ -275,7 +275,7 @@ private String getInstanceHomePageUrl(InstanceInfo instanceInfo) {
         String instanceHomePage = instanceInfo.getHomePageUrl();
 
         //Gateway homePage is used to hold DVIPA address and must not be modified
-        if (hasHomePage(instanceInfo) && !isGateway(instanceInfo)) {
+        if (hasHomePage(instanceInfo) && !StringUtils.equalsIgnoreCase(GATEWAY.getServiceId(), instanceInfo.getAppName())) {
             instanceHomePage = instanceHomePage.trim();
             RoutedServices routes = metadataParser.parseRoutes(instanceInfo.getMetadata());
             try {
@@ -326,10 +326,6 @@ private boolean hasHomePage(InstanceInfo instanceInfo) {
             && !instanceHomePage.isEmpty();
     }
 
-    private boolean isGateway(InstanceInfo instanceInfo) {
-        return instanceInfo.getAppName().equalsIgnoreCase(CoreService.GATEWAY.getServiceId());
-    }
-
     /**
      * Create a new container based on information in a new instance
      *
@@ -362,7 +358,7 @@ private APIContainer createNewContainerFromService(String productFamilyId, Insta
      * @param instanceInfo the service instance
      * @return a APIService object
      */
-    private APIService createAPIServiceFromInstance(InstanceInfo instanceInfo) {
+    APIService createAPIServiceFromInstance(InstanceInfo instanceInfo) {
         boolean secureEnabled = instanceInfo.isPortEnabled(InstanceInfo.PortType.SECURE);
 
         String instanceHomePage = getInstanceHomePageUrl(instanceInfo);
@@ -384,8 +380,21 @@ private APIService createAPIServiceFromInstance(InstanceInfo instanceInfo) {
             log.info("createApiServiceFromInstance#incorrectVersions {}", ex.getMessage());
         }
 
-        return new APIService.Builder(instanceInfo.getAppName().toLowerCase())
-            .title(instanceInfo.getMetadata().get(SERVICE_TITLE))
+        String serviceId = instanceInfo.getAppName();
+        String title = instanceInfo.getMetadata().get(SERVICE_TITLE);
+        if (StringUtils.equalsIgnoreCase(GATEWAY.getServiceId(), serviceId)) {
+            if (RegistrationType.of(instanceInfo.getMetadata()).isAdditional()) {
+                // additional registration for GW means domain one, update serviceId with the ApimlId
+                String apimlId = instanceInfo.getMetadata().get(APIML_ID);
+                if (apimlId != null) {
+                    serviceId = apimlId;
+                    title += " (" + apimlId + ")";
+                }
+            }
+        }
+
+        return new APIService.Builder(StringUtils.lowerCase(serviceId))
+            .title(title)
             .description(instanceInfo.getMetadata().get(SERVICE_DESCRIPTION))
             .secured(secureEnabled)
             .baseUrl(instanceInfo.getHomePageUrl())