feat: extend functionality of load tests to run from pod

k6/README.md

@@ -26,20 +26,57 @@ from the [podman-grapher](./local_setup/podman-grapher/) directory. This will la
 This test requires a client with a service account to run. E.g if using the default `admin-cli` client of the master realm locally, make sure the following are configured for it:
 
 - In the client settings, set the **Access Type** to confidential, and then toggle on **Service accounts enabled**.
-- Make sure that the clientID and clientSecret in [env.js](./env.js) match that client's credentials.
+- Make sure that the clientID and clientSecret in [config.json](./k6-runner/src/config/config.json) match that client's credentials.
 
 ### Testing the Quarkus release:
 
 Make certain that 'Client authentication' is toggled on and select 'Direct access grants' and 'Service accounts roles' from Authentication Flow.
 
 **Do not do this in a production environment**. In the master realm, go to `Authentication->Direct grant` and disable "Condition- user configured" and "OTP".
 
+If testing a live application, pick an appropriate client to use with a confidential service account.
 
+- Copy `k6-runner/src/config/config.example.json` to `k6-runner/src/config/config.json`. Provide credentials for an account with permissions to create realms and users. If you are setting up locally, use the baseURL `http://localhost:8080/auth`, and you can use the admin-cli client ID with the admin admin credentials for username and password.
 
-If testing a live application, pick an appropriate client to use with a confidential service account.
+#### Running the test locally
+
+The tests can be un locally by using the `docker-compose.yml` file in the `k6-runner` directory. From the `k6-runner` directory, run:
+
+```
+docker-compose run k6 run -e CONFIG=/config/config.json /scripts/constantRateAllFlows.js
+```
+
+#### Running the test remotely
+
+The remote test runs on the script `/k6/k6-runner/openshift/k6/start.sh`. To change the test, point the run command at a different file.
+
+To run the test from a kubernetes pod you will need to do the following:
+
+Create a docker image a push it to be hosted in the bcgov ghcr repos.  **Note: do not build any secrets into the image, it is a public repo**. From the `ku-runner` directiory run:
+```
+docker build . -t ghcr.io/bcgov/sso-k6:latest
+docker push ghcr.io/bcgov/sso-k6:latest
+```
+This will create the image and host it. Each time you change the test code of config, this image will need to be rebuilt and pushed.
+
+Next create the k6-config file in the namespace from which you want to run the tests:
+```
+oc create -n <NAMESPACE> configmap k6-config --from-file=./src/config/config.json
+```
+
+Lastly deploy the config from the `sso-keycloak/k6/k6-runner/openshift/k6` directory.
+```
+oc -n <NAMESPACE> process -f dc.yaml | oc -n <NAMESPACE> apply -f -
+```
+
+Be sure to delete the job when done to prevent the load test from re-running in the cluster.
+
+#### Running the test locally without docker.
+
+The tests can also be run without docker, by running:
+`k6 run -e CONFIG=../config/config.json ./tests/constantRateAllFlows.js`
+from the `sso-keycloak/k6/k6-runner/src` directory.
 
-- Copy `env.example.js` to `env.js`. Provide credentials for an account with permissions to create realms and users. If you are setting up locally, use the baseURL `http://localhost:8080/auth`, and you can use the admin-cli client ID with the admin admin credentials for username and password.
-- Run tests with `k6 run <js file>`
 
 ## Tests
 

k6/k6-runner/Dockerfile

@@ -0,0 +1,8 @@
+FROM loadimpact/k6:latest
+
+WORKDIR /var/opt
+
+COPY /src/tests /var/opt/scripts
+COPY /openshift/k6/start.sh /var/opt
+
+ENTRYPOINT [ "sh", "/var/opt/start.sh" ]

k6/k6-runner/README.md

@@ -0,0 +1,37 @@
+# K6 Load Testing
+These tests are adapted from the Ministry of Education's Student Online Access Module (SOAM), [load testing framework](https://github.com/bcgov/EDUC-KEYCLOAK-SOAM/blob/refs%2Fheads%2Fmaster/testing%2Fk6%2FREADME.md).
+
+## Running Locally
+* Docker is installed
+* Create a config.json folder from the example file. Enter correct values into `src/config/config.json`
+
+* Run the following command to run tests
+```
+docker-compose run k6 run -e CONFIG=/config/config.json /scripts/soamLoadTest.ts
+```
+* You can view the test metrics on the [Grafana dashboard](http://localhost:3000/d/XJhgbUpil/soam-load-testing-dashboard)
+
+## Deploy to OpenShift
+### Influxdb
+Coming soon
+### Grafana
+Coming soon
+### K6
+* Ensure `config.json` is filled out with correct values, navigate to the openshift/k6 folder, and run the log into OpenShift command
+* Create config map
+```
+oc create -n <NAMESPACE> configmap k6-config --from-file=../../src/config/config.json
+```
+* Build K6 job
+```
+oc -n <NAMESPACE> process -f bc.yaml | oc -n <NAMESPACE> apply -f -
+```
+* Creating the K6 job will automatically run the load test on creation, and will terminate the container upon completion
+```
+oc -n <NAMESPACE> process -f dc.yaml -p IMAGE_NAMESPACE=<NAMESPACE> | oc -n <NAMESPACE> apply -f -
+```
+* Once tests are complete, clean up K6 job artifacts
+```
+oc -n <NAMESPACE> get all,configmap,secret,pvc -l group=educ-k6
+oc -n <NAMESPACE> delete all,configmap,secret,pvc -l group=educ-k6
+```

k6/k6-runner/docker-compose.yml

@@ -0,0 +1,15 @@
+version: '3.0'
+
+networks:
+  k6:
+
+services:
+  k6:
+    image: loadimpact/k6:latest
+    networks:
+      - k6
+    ports:
+      - "6565:6565"
+    volumes:
+      - ./src/tests:/scripts
+      - ./src/config:/config

k6/k6-runner/openshift/k6/dc.yaml

@@ -0,0 +1,45 @@
+kind: Template
+apiVersion: v1
+objects:
+  - apiVersion: batch/v1
+    kind: Job
+    spec:
+      backoffLimit: 0
+      template:
+        metadata:
+          creationTimestamp: null
+        spec:
+          volumes:
+            - name: k6-config
+              configMap:
+                name: k6-config
+          containers:
+            - image: ${IMAGE_REPOSITORY}:${IMAGE_TAG}
+              name: ${NAME}
+              resources:
+                limits:
+                  cpu: 2
+                  memory: 2Gi
+                requests:
+                  cpu: 500m
+                  memory: 500Mi
+              volumeMounts:
+                - name: k6-config
+                  mountPath: /var/opt/config
+          restartPolicy: Never
+    metadata:
+      name: ${NAME}
+      labels:
+        app: ${NAME}
+        group: educ-k6
+        component: ${NAME}-job
+parameters:
+  - name: NAME
+    value: sso-k6
+  - name: IMAGE_TAG
+    value: latest
+  - name: IMAGE_REPOSITORY
+    value: ghcr.io/bcgov/sso-k6
+  - name: HTTP_DEBUG
+    description: enable http debug logging in the k6 pod
+    value: "false"

k6/k6-runner/openshift/k6/start.sh

@@ -0,0 +1 @@
+k6 run -e CONFIG=/var/opt/config/config.json /var/opt/scripts/constantRateAllFlows.js

k6/k6-runner/src/config/.gitignore

@@ -0,0 +1 @@
+config.json

k6/k6-runner/src/config/config.example.json

@@ -0,0 +1,10 @@
+
+{
+  "kcLoadTest": {
+    "password": "",
+    "clientId": "",
+    "baseUrl": "",
+    "clientSecret": "",
+    "username": ""
+  }
+}

k6/k6-runner/src/tests/README.md

@@ -0,0 +1,88 @@
+# Overview
+
+This folder contains load tests for our sso application
+
+## Setting up
+
+**Developing and Running tests in a local environment**: If you would like to use a local environment for developing and running tests, there is a [podman-compose.yaml](./local_setup/podman-compose.yaml) file in this repository you can use to run our custom redhat image with a postgres database. To use it, from the [loca setup folder](./local_setup/), run:
+
+- `podman-compose up`
+
+**Note**: _You will need to have installed podman and podman-compose. Alternatively, you can use the same commands with docker compose, just specify the file with the -f flag._
+
+This will start our custom keycloak image on localhost:8080, you can login with credentials username=admin, password=admin. To stop the image, you can ctrl+c out (alternatively, add the -d flag to run detached), and run `podman-compose down`. To clear out the volumes, with the image stopped, run `podman volume prune` (or specify the volumes if you have additional ones to keep). The image is currently set to use `ghcr.io/bcgov/sso:7.6.25-build.1`, this can be updated as later builds come up.
+
+**Tracking stats locally**: If you would like to compare resource usage of the different tests on your local machine, there is a small electron app to graph the output of podman stats. With the podman-compose running, you can run:
+
+- `npm i`
+- `npm start`
+
+from the [podman-grapher](./local_setup/podman-grapher/) directory. This will launch a browser window graphing the CPU and Memory usage of the local keycloak docker container over time. You can run tests with it open, and save the png's if you want to check the relative differences.
+
+**Note**:  _In `podman stats`, the CPU usage is per core. So the percent used can go up to 100 * (number of machine cores)_.
+
+## Using
+
+This test requires a client with a service account to run. E.g if using the default `admin-cli` client of the master realm locally, make sure the following are configured for it:
+
+- In the client settings, set the **Access Type** to confidential, and then toggle on **Service accounts enabled**.
+- Make sure that the clientID and clientSecret in [env.js](./env.js) match that client's credentials.
+
+### Testing the Quarkus release:
+
+Make certain that 'Client authentication' is toggled on and select 'Direct access grants' and 'Service accounts roles' from Authentication Flow.
+
+**Do not do this in a production environment**. In the master realm, go to `Authentication->Direct grant` and disable "Condition- user configured" and "OTP".
+
+
+
+If testing a live application, pick an appropriate client to use with a confidential service account.
+
+- Copy `env.example.js` to `env.js`. Provide credentials for an account with permissions to create realms and users. If you are setting up locally, use the baseURL `http://localhost:8080/auth`, and you can use the admin-cli client ID with the admin admin credentials for username and password.
+- Run tests with `k6 run <js file>`
+
+## Tests
+
+### [Active Sessions](./activeSessions.js)
+
+This test is setup to see how requesting access tokens affects the system. It can be configured with the following variables at the top of the file:
+
+**CONCURRENT_LOOPS**: The number of loops to run concurrently. Increasing this number will allow the test to fire more requests at the same time. E.g running 3 concurrent loops would send 3 requests for an access token at once, and then wait the **LOOP_DELAY**, then fire all three again in the next realm.
+**ITERATIONS_PER_LOOP**: The number of times each loop will run. Each loop requests an access token from every realm, waiting a small delay between access token requests set by the **LOOP_DELAY** variable.
+**TOTAL_REALMS** = The number of realms to create. Each loop will request an access token from all realms on an iteration. So the total number of requested access tokens by a test will be `TOTAL_REALMS * ITERATIONS_PER_LOOP * CONCURRENT_LOOPS`. Increase this number to test if requesting access tokens from different realms with different users affects performance.
+**MAX_ALLOWED_FAILURE_RATE**: The percentage of requests to allow to fail before counting the test as failed. Enter as a string of a decimal number, e.g `'0.01'` is 1%.
+**OFFLINE** Set true to request offline_access tokens.
+**LOOP_DELAY**: The amount of time to wait between token requests in each loop, in seconds. e.g 0.1 is 100ms. Set to 0 to fire as soon as possible.
+
+### [Token Introspection](./tokenIntrospection.js)
+
+Run this test to see how hitting the token introspection endpoint affects the system.
+
+The test run can be configured with the following variables at the top of the file:
+
+**CONCURRENT_LOOPS**: The number of loops to run concurrently. Increasing this number will allow the test to fire more requests at the same time. E.g running 3 concurrent loops would send 3 requests to the introspection endpoint at once, and then wait the **LOOP_DELAY**, then fire all three again in the next realm.
+**ITERATIONS_PER_LOOP**: The number of times each loop will run. Each loop will hit the introspection endpoint this number of times, waiting a small delay between requests set by the **LOOP_DELAY** variable.
+**LOOP_DELAY**: The amount of time to wait between requests in each loop, in seconds. e.g 0.1 is 100ms. Set to 0 to fire as soon as possible.
+
+### [User Info](./userInfo.js)
+
+Run this test to see how hitting the user info endpoint affects the system.
+
+The test run can be configured with the following variables at the top of the file:
+
+**CONCURRENT_LOOPS**: The number of loops to run concurrently. Increasing this number will allow the test to fire more requests at the same time. E.g running 3 concurrent loops would send 3 requests to the user info endpoint at once, and then wait the **LOOP_DELAY**, then fire all three again in the next realm.
+**ITERATIONS_PER_LOOP**: The number of times each loop will run. Each loop will hit the user info endpoint this number of times, waiting a small delay between requests set by the **LOOP_DELAY** variable.
+**LOOP_DELAY**: The amount of time to wait between requests in each loop, in seconds. e.g 0.1 is 100ms. Set to 0 to fire as soon as possible.
+
+### [Constant Rate all Flows](./constantRateAllFlows.js)
+
+Run this test to simulate fetching an access token, grabbing user info, and introspecting the token all together. This test has two scenarios, `peakProfile` and `stress`. The peak profile test is used to imitate our peak traffic running against the application for a two hour period. The stress test will ramp up traffic linearly over a 1 hour period until API requests start to fail, and then abort.
+
+When stress testing, the application may get saturated with requests which prevents the teardown logic from succeeding, since it depends on the keycloak API being able to receive and act on requests. In this case, the test realms will not delete properly. These realms are all prefixed with "newrealm" and will need to be deleted manually.
+
+The test run can be configured with the following variables at the top of the file:
+
+**TOTAL_REALMS** = The number of realms to create.
+**MAX_ALLOWED_FAILURE_RATE**: The percentage of requests to allow to fail before counting the test as failed. Enter as a string of a decimal number, e.g `'0.01'` is 1%.
+**OFFLINE** Set true to request offline_access tokens.
+**BASELINE_RATE**: If running the peakProfile scenario, this is the peak rate per minute of requests to use. It will also determine the start rate of the stress test.

k6/activeSessions.js → k6/k6-runner/src/tests/activeSessions.js

@@ -1,8 +1,12 @@
 import { sleep } from 'k6';
 import { createRealm, deleteRealm, createUser, generateRealms, getAccessToken } from './helpers.js';
 import { user } from './constants.js';
-import { username, password, clientId } from './env.js';
 
+let config = JSON.parse(open(__ENV.CONFIG));
+
+const username = config.kcLoadTest.username;
+const password = config.kcLoadTest.password;
+const clientId = config.kcLoadTest.clientId;
 // Alter configuration to run separate tests. See this test in the readme for configuration details.
 const CONCURRENT_LOOPS = 5;
 const ITERATIONS_PER_LOOP = 50;

k6/constantRateAllFlows.js → k6/k6-runner/src/tests/constantRateAllFlows.js

@@ -1,8 +1,11 @@
 import { sleep } from 'k6';
 import { createRealm, deleteRealm, createUser, generateRealms, getAccessToken, hitIntrospectionRoute, hitUserInfoRoute, createClient } from './helpers.js';
 import { user, client } from './constants.js';
-import { username, password, clientId } from './env.js';
 
+let config = JSON.parse(open(__ENV.CONFIG));
+const username = config.kcLoadTest.username;
+const password = config.kcLoadTest.password;
+const clientId = config.kcLoadTest.clientId;
 // Alter configuration to run separate tests. See this test in the readme for configuration details.
 const TOTAL_REALMS = 1;
 // This essentially just means no dropped requests allowed since we dont get to 10000 on the peak profile.

k6/helpers.js → k6/k6-runner/src/tests/helpers.js

@@ -1,8 +1,12 @@
 import http, { head } from 'k6/http';
-import { baseUrl, clientId, clientSecret } from './env.js';
 import { realm, client } from './constants.js';
 import encoding from 'k6/encoding';
 
+let config = JSON.parse(open(__ENV.CONFIG));
+
+const baseUrl = config.kcLoadTest.baseUrl;
+const clientSecret = config.kcLoadTest.clientSecret;
+
 const getHeaders = (accessToken) => ({
   Authorization: `Bearer ${accessToken}`,
   'Content-Type': 'application/json',

k6/tokenIntrospection.js → k6/k6-runner/src/tests/tokenIntrospection.js

@@ -1,8 +1,12 @@
 import { sleep } from 'k6';
 import { hitIntrospectionRoute, getAccessToken, createClient, deleteClient } from './helpers.js';
-import { username, password, clientId } from './env.js';
 import { client } from './constants.js';
 
+let config = JSON.parse(open(__ENV.CONFIG));
+const username = config.kcLoadTest.username;
+const password = config.kcLoadTest.password;
+const clientId = config.kcLoadTest.clientId;
+
 const CONCURRENT_LOOPS = 1;
 const ITERATIONS_PER_LOOP = 10;
 const LOOP_DELAY = 0.01;

k6/userInfo.js → k6/k6-runner/src/tests/userInfo.js

@@ -1,6 +1,10 @@
 import { sleep } from 'k6';
 import { getAccessToken, hitUserInfoRoute } from './helpers.js';
-import { username, password, clientId } from './env.js';
+
+let config = JSON.parse(open(__ENV.CONFIG));
+const username = config.kcLoadTest.username;
+const password = config.kcLoadTest.password;
+const clientId = config.kcLoadTest.clientId;
 
 const CONCURRENT_LOOPS = 1;
 const ITERATIONS_PER_LOOP = 100;