updated the cross site health check script to suit the new deployments

Signed-off-by: Kamesh Akella <[email protected]>

.github/actions/get-keycloak-url/action.yml

@@ -14,3 +14,13 @@ runs:
       run: |
         KEYCLOAK_URL=https://$(kubectl get routes -n "${{ inputs.project }}" -l app=keycloak -o jsonpath='{.items[*].spec.host}')
         echo "KEYCLOAK_URL=$KEYCLOAK_URL" >> "$GITHUB_ENV"
+    - id: get-keycloak-site-url
+      shell: bash
+      run: |
+        KEYCLOAK_SITE_URL=https://$(kubectl -n "${{ inputs.project }}" get svc accelerator-loadbalancer --template="{{range .status.loadBalancer.ingress}}{{.hostname}}{{end}}")
+        echo "KEYCLOAK_SITE_URL=$KEYCLOAK_SITE_URL" >> "$GITHUB_ENV"
+    - id: get-ispn-rest-url
+      shell: bash
+      run: |
+        KEYCLOAK_ISPN_REST_URL=https://$(kubectl get routes -n "${{ inputs.project }}"  -l app=infinispan-service-external -o jsonpath='{.items[*].spec.host}')
+        echo "KEYCLOAK_ISPN_REST_URL=$KEYCLOAK_ISPN_REST_URL" >> "$GITHUB_ENV"

doc/kubernetes/modules/ROOT/pages/util/healthchecks-keycloak.adoc

@@ -52,7 +52,9 @@ curl -s https://keycloak_site_b_url/lb-check
 ----
 
 === Infinispan Cache Health
-Checks the health of the default cache manager and individual caches in an external Infinispan cluster. This is vital for Keycloak performance and reliability, as Infinispan is often used for distributed caching and session clustering in Keycloak deployments.
+Check the health of the default cache manager and individual caches in an external Infinispan cluster.
+This is vital for Keycloak performance and reliability,
+as Infinispan is often used for distributed caching and session clustering in Keycloak deployments.
 
 This command returns the overall health of the Infinispan cache manager, this is useful as the Admin user doesn't need to provide user credentials to get the health status.
 [source,bash]
@@ -91,7 +93,7 @@ oc get infinispan -n <NAMESPACE> -o json  \
 ----
 
 === Keycloak Readiness in Openshift
-Specifically checks for the readiness and rolling update conditions of Keycloak deployments in Red Hat OpenShift, ensuring that the Keycloak instances are fully operational and not undergoing updates that could impact availability.
+Specifically, checks for the readiness and rolling update conditions of Keycloak deployments in Red Hat OpenShift, ensuring that the Keycloak instances are fully operational and not undergoing updates that could impact availability.
 
 [source,bash]
 ----
@@ -101,31 +103,69 @@ oc wait --for=condition=RollingUpdate=False --timeout=10s keycloaks.k8s.keycloak
 ==== Optional Bash script
 You can use the link:{github-files}/provision/rosa-cross-dc/cross-site-health-checks.sh[cross-site-health-checks.sh] script and extend it to perform the necessary checks and integrate this into your monitoring architecture.
 
-To run the script, as a pre-requisite, you need to establish a session from your terminal to the target OCP cluster with a command such as,
+To run the script, as a pre-requisite,
+you need to establish a session from your terminal to the target OCP cluster with a command.
 
 [source,bash]
 ----
 oc login --token=sha256~masked-key --server=https://api.gh-keycloak-a.masked.openshiftapps.com:6443
 ----
 
+Also note, it would be necessary to run this script against all the clusters in the cluster group,
+so the Administrator would have to repeat the above `oc` login command for all the clusters,
+which could be automated.
+
 To run the script itself once you have an active `oc` session below is an example usage.
 
 [source,bash]
 ----
-./cross-site-health-checks.sh -d gh-keycloak-a-gh-keycloak-b-masked.keycloak-benchmark.com \
--u developer -p masked-password \
--s gh-keycloak-a.masked.openshiftapps.com \
--c 3 -n runner-keycloak
+./cross-site-health-checks.sh \
+-n runner-keycloak \
+-l <KEYCLOAK_LB_URL> \
+-k <KEYCLOAK_SITE_URL> \
+-i <KEYCLOAK_ISPN_REST_URL> \
+-u developer \
+-p <ISPN_REST_URL_PWD> \
+-c 3
+
+Verify the Keycloak Load Balancer health check
+Checking health for: KEYCLOAK_LB_URL/lb-check
+"HEALTHY"
+
+Verify the Load Balancer health check on the Site
+Checking health for: KEYCLOAK_SITE_URL/lb-check
+"HEALTHY"
+
+Verify the default cache manager health in external ISPN
+Checking health for: KEYCLOAK_ISPN_REST_URL/rest/v2/cache-managers/default/health/status
+"HEALTHY"
+
+Verify individual cache health
+"HEALTHY"
+
+ISPN Cluster Distribution
+"HEALTHY"
+
+ISPN Overall Status
+"HEALTHY"
+
+Verify for Keycloak condition in ROSA cluster
+keycloak.k8s.keycloak.org/keycloak condition met
+keycloak.k8s.keycloak.org/keycloak condition met
 ----
 
-Usage of the script with details around the different options
+==== Usage of the script with details around the different options
 [source, bash]
 ----
-Usage: ./cross-site-health-checks.sh [-d domain] [-u infinispan_user] [-p infinispan_pwd] [-s infinispan_url_suffix] [-c expected_count] [-n namespace]
-  -d domain: Keycloak domain
+Usage: [-n namespace] [-l keycloak_lb_url] [-k keycloak_site_url]
+[-i infinispan_rest_url] [-u infinispan_user] [-p infinispan_pwd]
+[-c expected_ispn_count]
+
+  -n namespace: Kubernetes namespace
+  -l keycloak_lb_url: Keycloak Load Balancer URL
+  -k keycloak_site_url: Keycloak Site URL
+  -i infinispan_rest_url: Infinispan REST URL
   -u infinispan_user: Infinispan user
   -p infinispan_pwd: Infinispan password
-  -s infinispan_url_suffix: Infinispan URL suffix
-  -c expected_count: Expected Node Count in the Infinispan cluster
-  -n namespace: Kubernetes namespace
+  -c expected_ispn_count: Expected Node Count in the Infinispan cluster
 ----

provision/rosa-cross-dc/cross-site-health-checks.sh

@@ -1,21 +1,23 @@
 #!/usr/bin/env bash
 
 # Default values
-domain="temp-domain"
+namespace="runner-keycloak"
+keycloak_lb_url="temp-lb-url"
+keycloak_site_url="temp-site-url"
+infinispan_rest_url="temp-rest-url"
 infinispan_user="developer"
 infinispan_pwd="password-is-not-set"
-infinispan_url_suffix="temp-suffix"
-namespace="runner-keycloak"
 
 # Usage function to display help for the script
 usage() {
-    echo "Usage: $0 [-d domain] [-u infinispan_user] [-p infinispan_pwd] [-s infinispan_url_suffix] [-c expected_count] [-n namespace]"
-    echo "  -d domain: Keycloak domain"
+    echo "Usage: $0 [-n namespace] [-l keycloak_lb_url] [-k keycloak_site_url] [-i infinispan_rest_url] [-u infinispan_user] [-p infinispan_pwd] [-c expected_ispn_count]"
+    echo "  -n namespace: Kubernetes namespace"
+    echo "  -l keycloak_lb_url: Keycloak Load Balancer URL"
+    echo "  -k keycloak_site_url: Keycloak Site URL"
+    echo "  -i infinispan_rest_url: Infinispan REST URL"
     echo "  -u infinispan_user: Infinispan user"
     echo "  -p infinispan_pwd: Infinispan password"
-    echo "  -s infinispan_url_suffix: Infinispan URL suffix"
-    echo "  -c expected_count: Expected Node Count in the Infinispan cluster"
-    echo "  -n namespace: Kubernetes namespace"
+    echo "  -c expected_ispn_count: Expected Node Count in the Infinispan cluster"
     exit 1
 }
 
@@ -26,19 +28,21 @@ if [ $# -eq 0 ]; then
 fi
 
 # Parse input arguments
-while getopts ":d:u:p:s:n:c:h" opt; do
+while getopts ":n:l:k:i:u:p:c:h" opt; do
   case ${opt} in
-    d ) domain=$OPTARG
+    n ) namespace=$OPTARG
+          ;;
+    l ) keycloak_lb_url=$OPTARG
+      ;;
+    k ) keycloak_site_url=$OPTARG
       ;;
+    i ) infinispan_rest_url=$OPTARG
+          ;;
     u ) infinispan_user=$OPTARG
       ;;
     p ) infinispan_pwd=$OPTARG
       ;;
-    s ) infinispan_url_suffix=$OPTARG
-      ;;
-    n ) namespace=$OPTARG
-      ;;
-    c ) expected_count=$OPTARG
+    c ) expected_ispn_count=$OPTARG
       ;;
     h ) usage
       ;;
@@ -62,13 +66,6 @@ for cmd in curl jq oc; do
     fi
 done
 
-
-# Base URLs
-keycloak_lb_url="https://client.$domain"
-keycloak_site_a_url="https://primary.$domain"
-keycloak_site_b_url="https://backup.$domain"
-infinispan_rest_url="https://infinispan-external-runner-keycloak.apps.$infinispan_url_suffix"
-
 health_check() {
     local url=$1
     local parse_json=${2:-false}  # Optional second argument; default is 'false'
@@ -95,23 +92,23 @@ health_check() {
 }
 
 echo "Verify the Keycloak Load Balancer health check"
-health_check $keycloak_lb_url/lb-check
+health_check "$keycloak_lb_url"/lb-check
 
-echo "Verify the Load Balancer health check on Site A and Site B"
-health_check $keycloak_site_a_url/lb-check
-health_check $keycloak_site_b_url/lb-check
+echo "Verify the Load Balancer health check on the Site"
+health_check "$keycloak_site_url"/lb-check
 
 echo "Verify the default cache manager health in external ISPN"
-health_check $infinispan_rest_url/rest/v2/cache-managers/default/health/status
+health_check "$infinispan_rest_url"/rest/v2/cache-managers/default/health/status
 
 echo "Verify individual cache health"
-curl -u $infinispan_user:$infinispan_pwd -sk $infinispan_rest_url/rest/v2/cache-managers/default/health \
+curl -u "$infinispan_user":"$infinispan_pwd" -sk "$infinispan_rest_url"/rest/v2/cache-managers/default/health \
  | jq 'if .cluster_health.health_status == "HEALTHY" and (all(.cache_health[].status; . == "HEALTHY")) then "HEALTHY" else "UNHEALTHY" end'
 echo
 
 echo "ISPN Cluster Distribution"
-curl -u $infinispan_user:$infinispan_pwd -sk $infinispan_rest_url/rest/v2/cluster\?action\=distribution \
- | jq --argjson expectedCount $expected_count 'if map(select(.node_addresses | length > 0)) | length == $expectedCount then "HEALTHY" else "UNHEALTHY" end'
+# shellcheck disable=SC2086
+curl -u "$infinispan_user":"$infinispan_pwd" -sk $infinispan_rest_url/rest/v2/cluster\?action\=distribution \
+ | jq --argjson expectedCount "$expected_ispn_count" 'if map(select(.node_addresses | length > 0)) | length == $expectedCount then "HEALTHY" else "UNHEALTHY" end'
 echo
 
 echo "ISPN Overall Status"
@@ -122,5 +119,5 @@ oc get infinispan -n runner-keycloak -o json  \
 echo
 
 echo "Verify for Keycloak condition in ROSA cluster"
-oc wait --for=condition=Ready --timeout=10s keycloaks.k8s.keycloak.org/keycloak -n runner-keycloak
-oc wait --for=condition=RollingUpdate=False --timeout=10s keycloaks.k8s.keycloak.org/keycloak -n runner-keycloak
+oc wait --for=condition=Ready --timeout=10s keycloaks.k8s.keycloak.org/keycloak -n "$namespace"
+oc wait --for=condition=RollingUpdate=False --timeout=10s keycloaks.k8s.keycloak.org/keycloak -n "$namespace"