From b470b521b6a9cc74ca36fd86170ce280c8879ffc Mon Sep 17 00:00:00 2001 From: thegentlemanphysicist Date: Thu, 4 Jul 2024 10:52:08 -0700 Subject: [PATCH] feat: extend load tests to run from a pod (#361) * chore: reduce dev and test backup frequency to save space * chore: update load tests, add checks * feat: extend functionality of load tests to run from pod * chore: remove redundant readmes * chore: make make comands --- k6/README.md | 45 +++++++++++++++++-- k6/env.example.js | 6 --- k6/k6-runner/Dockerfile | 8 ++++ k6/k6-runner/Makefile | 16 +++++++ k6/k6-runner/docker-compose.yml | 15 +++++++ k6/k6-runner/openshift/k6/dc.yaml | 44 ++++++++++++++++++ k6/k6-runner/openshift/k6/start.sh | 1 + k6/k6-runner/src/config/.gitignore | 1 + k6/k6-runner/src/config/config.example.json | 10 +++++ .../src/tests}/activeSessions.js | 6 ++- .../src/tests}/constantRateAllFlows.js | 5 ++- k6/{ => k6-runner/src/tests}/constants.js | 0 k6/{ => k6-runner/src/tests}/helpers.js | 6 ++- .../src/tests}/tokenIntrospection.js | 6 ++- k6/{ => k6-runner/src/tests}/userInfo.js | 6 ++- 15 files changed, 160 insertions(+), 15 deletions(-) delete mode 100644 k6/env.example.js create mode 100644 k6/k6-runner/Dockerfile create mode 100644 k6/k6-runner/Makefile create mode 100644 k6/k6-runner/docker-compose.yml create mode 100644 k6/k6-runner/openshift/k6/dc.yaml create mode 100644 k6/k6-runner/openshift/k6/start.sh create mode 100644 k6/k6-runner/src/config/.gitignore create mode 100644 k6/k6-runner/src/config/config.example.json rename k6/{ => k6-runner/src/tests}/activeSessions.js (91%) rename k6/{ => k6-runner/src/tests}/constantRateAllFlows.js (96%) rename k6/{ => k6-runner/src/tests}/constants.js (100%) rename k6/{ => k6-runner/src/tests}/helpers.js (96%) rename k6/{ => k6-runner/src/tests}/tokenIntrospection.js (90%) rename k6/{ => k6-runner/src/tests}/userInfo.js (85%) diff --git a/k6/README.md b/k6/README.md index 6ffb691d..3b6827cd 100644 --- a/k6/README.md +++ b/k6/README.md @@ -26,7 +26,7 @@ from the [podman-grapher](./local_setup/podman-grapher/) directory. This will la This test requires a client with a service account to run. E.g if using the default `admin-cli` client of the master realm locally, make sure the following are configured for it: - In the client settings, set the **Access Type** to confidential, and then toggle on **Service accounts enabled**. -- Make sure that the clientID and clientSecret in [env.js](./env.js) match that client's credentials. +- Make sure that the clientID and clientSecret in [config.json](./k6-runner/src/config/config.json) match that client's credentials. ### Testing the Quarkus release: @@ -34,12 +34,49 @@ Make certain that 'Client authentication' is toggled on and select 'Direct acces **Do not do this in a production environment**. In the master realm, go to `Authentication->Direct grant` and disable "Condition- user configured" and "OTP". +If testing a live application, pick an appropriate client to use with a confidential service account. +- Copy `k6-runner/src/config/config.example.json` to `k6-runner/src/config/config.json`. Provide credentials for an account with permissions to create realms and users. If you are setting up locally, use the baseURL `http://localhost:8080/auth`, and you can use the admin-cli client ID with the admin admin credentials for username and password. -If testing a live application, pick an appropriate client to use with a confidential service account. +#### Running the test locally + +These tests are adapted from the Ministry of Education's Student Online Access Module (SOAM), [load testing framework](https://github.com/bcgov/EDUC-KEYCLOAK-SOAM/blob/refs%2Fheads%2Fmaster/testing%2Fk6%2FREADME.md). The tests can be un locally by using the `docker-compose.yml` file in the `k6-runner` directory. From the `k6-runner` directory, run: + +``` +docker-compose run k6 run -e CONFIG=/config/config.json /scripts/constantRateAllFlows.js +``` + +#### Running the test remotely + +The remote test runs on the script `/k6/k6-runner/openshift/k6/start.sh`. To change the test, point the run command at a different file. + +To run the test from a kubernetes pod you will need to do the following: + +Create a docker image a push it to be hosted in the bcgov ghcr repos. **Note: do not build any secrets into the image, it is a public repo**. From the `ku-runner` directiory run: +``` +docker build . -t ghcr.io/bcgov/sso-k6:latest +docker push ghcr.io/bcgov/sso-k6:latest +``` +This will create the image and host it. Each time you change the test code of config, this image will need to be rebuilt and pushed. + +Next create the k6-config file in the namespace from which you want to run the tests: +``` +oc create -n configmap k6-config --from-file=./src/config/config.json +``` + +Lastly deploy the config from the `sso-keycloak/k6/k6-runner/openshift/k6` directory. +``` +oc -n process -f dc.yaml | oc -n apply -f - +``` + +Be sure to delete the job when done to prevent the load test from re-running in the cluster. + +#### Running the test locally without docker. + +The tests can also be run without docker, by running: +`k6 run -e CONFIG=../config/config.json ./tests/constantRateAllFlows.js` +from the `sso-keycloak/k6/k6-runner/src` directory. -- Copy `env.example.js` to `env.js`. Provide credentials for an account with permissions to create realms and users. If you are setting up locally, use the baseURL `http://localhost:8080/auth`, and you can use the admin-cli client ID with the admin admin credentials for username and password. -- Run tests with `k6 run ` ## Tests diff --git a/k6/env.example.js b/k6/env.example.js deleted file mode 100644 index a59cdc4d..00000000 --- a/k6/env.example.js +++ /dev/null @@ -1,6 +0,0 @@ -export const username = ''; -export const password = ''; -export const clientId = 'admin-cli'; -export const baseUrl = '/auth'; -// get the client secret from the Credentials tab of client `clientId` -export const clientSecret = '' diff --git a/k6/k6-runner/Dockerfile b/k6/k6-runner/Dockerfile new file mode 100644 index 00000000..13fb02e3 --- /dev/null +++ b/k6/k6-runner/Dockerfile @@ -0,0 +1,8 @@ +FROM loadimpact/k6:latest + +WORKDIR /var/opt + +COPY /src/tests /var/opt/scripts +COPY /openshift/k6/start.sh /var/opt + +ENTRYPOINT [ "sh", "/var/opt/start.sh" ] diff --git a/k6/k6-runner/Makefile b/k6/k6-runner/Makefile new file mode 100644 index 00000000..a0bdc2cc --- /dev/null +++ b/k6/k6-runner/Makefile @@ -0,0 +1,16 @@ +SHELL := /usr/bin/env bash + +# make NAMESPACE="" + +.PHONY: config +config: + oc create -n $(NAMESPACE) configmap k6-config --from-file=./src/config/config.json + +.PHONY: run_job +run_job: + oc -n $(NAMESPACE) process -f ./openshift/k6/dc.yaml | oc -n $(NAMESPACE) apply -f - + +.PHONY: delete +delete: + oc -n $(NAMESPACE) delete job sso-k6 + oc -n $(NAMESPACE) delete configmap k6-config diff --git a/k6/k6-runner/docker-compose.yml b/k6/k6-runner/docker-compose.yml new file mode 100644 index 00000000..27949ebf --- /dev/null +++ b/k6/k6-runner/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3.0' + +networks: + k6: + +services: + k6: + image: loadimpact/k6:latest + networks: + - k6 + ports: + - "6565:6565" + volumes: + - ./src/tests:/scripts + - ./src/config:/config diff --git a/k6/k6-runner/openshift/k6/dc.yaml b/k6/k6-runner/openshift/k6/dc.yaml new file mode 100644 index 00000000..a7b09976 --- /dev/null +++ b/k6/k6-runner/openshift/k6/dc.yaml @@ -0,0 +1,44 @@ +kind: Template +apiVersion: v1 +objects: + - apiVersion: batch/v1 + kind: Job + spec: + backoffLimit: 0 + template: + metadata: + creationTimestamp: null + spec: + volumes: + - name: k6-config + configMap: + name: k6-config + containers: + - image: ${IMAGE_REPOSITORY}:${IMAGE_TAG} + name: ${NAME} + resources: + limits: + cpu: 2 + memory: 2Gi + requests: + cpu: 500m + memory: 500Mi + volumeMounts: + - name: k6-config + mountPath: /var/opt/config + restartPolicy: Never + metadata: + name: ${NAME} + labels: + app: ${NAME} + component: ${NAME}-job +parameters: + - name: NAME + value: sso-k6 + - name: IMAGE_TAG + value: latest + - name: IMAGE_REPOSITORY + value: ghcr.io/bcgov/sso-k6 + - name: HTTP_DEBUG + description: enable http debug logging in the k6 pod + value: "false" diff --git a/k6/k6-runner/openshift/k6/start.sh b/k6/k6-runner/openshift/k6/start.sh new file mode 100644 index 00000000..73ffd23e --- /dev/null +++ b/k6/k6-runner/openshift/k6/start.sh @@ -0,0 +1 @@ +k6 run -e CONFIG=/var/opt/config/config.json /var/opt/scripts/constantRateAllFlows.js diff --git a/k6/k6-runner/src/config/.gitignore b/k6/k6-runner/src/config/.gitignore new file mode 100644 index 00000000..d344ba6b --- /dev/null +++ b/k6/k6-runner/src/config/.gitignore @@ -0,0 +1 @@ +config.json diff --git a/k6/k6-runner/src/config/config.example.json b/k6/k6-runner/src/config/config.example.json new file mode 100644 index 00000000..21211cba --- /dev/null +++ b/k6/k6-runner/src/config/config.example.json @@ -0,0 +1,10 @@ + +{ + "kcLoadTest": { + "password": "", + "clientId": "", + "baseUrl": "", + "clientSecret": "", + "username": "" + } +} diff --git a/k6/activeSessions.js b/k6/k6-runner/src/tests/activeSessions.js similarity index 91% rename from k6/activeSessions.js rename to k6/k6-runner/src/tests/activeSessions.js index 7063cc1d..3dafd415 100644 --- a/k6/activeSessions.js +++ b/k6/k6-runner/src/tests/activeSessions.js @@ -1,8 +1,12 @@ import { sleep } from 'k6'; import { createRealm, deleteRealm, createUser, generateRealms, getAccessToken } from './helpers.js'; import { user } from './constants.js'; -import { username, password, clientId } from './env.js'; +let config = JSON.parse(open(__ENV.CONFIG)); + +const username = config.kcLoadTest.username; +const password = config.kcLoadTest.password; +const clientId = config.kcLoadTest.clientId; // Alter configuration to run separate tests. See this test in the readme for configuration details. const CONCURRENT_LOOPS = 5; const ITERATIONS_PER_LOOP = 50; diff --git a/k6/constantRateAllFlows.js b/k6/k6-runner/src/tests/constantRateAllFlows.js similarity index 96% rename from k6/constantRateAllFlows.js rename to k6/k6-runner/src/tests/constantRateAllFlows.js index 251b77d0..3258d7a3 100644 --- a/k6/constantRateAllFlows.js +++ b/k6/k6-runner/src/tests/constantRateAllFlows.js @@ -1,8 +1,11 @@ import { sleep } from 'k6'; import { createRealm, deleteRealm, createUser, generateRealms, getAccessToken, hitIntrospectionRoute, hitUserInfoRoute, createClient } from './helpers.js'; import { user, client } from './constants.js'; -import { username, password, clientId } from './env.js'; +let config = JSON.parse(open(__ENV.CONFIG)); +const username = config.kcLoadTest.username; +const password = config.kcLoadTest.password; +const clientId = config.kcLoadTest.clientId; // Alter configuration to run separate tests. See this test in the readme for configuration details. const TOTAL_REALMS = 1; // This essentially just means no dropped requests allowed since we dont get to 10000 on the peak profile. diff --git a/k6/constants.js b/k6/k6-runner/src/tests/constants.js similarity index 100% rename from k6/constants.js rename to k6/k6-runner/src/tests/constants.js diff --git a/k6/helpers.js b/k6/k6-runner/src/tests/helpers.js similarity index 96% rename from k6/helpers.js rename to k6/k6-runner/src/tests/helpers.js index 103076aa..377235b9 100644 --- a/k6/helpers.js +++ b/k6/k6-runner/src/tests/helpers.js @@ -1,8 +1,12 @@ import http, { head } from 'k6/http'; -import { baseUrl, clientId, clientSecret } from './env.js'; import { realm, client } from './constants.js'; import encoding from 'k6/encoding'; +let config = JSON.parse(open(__ENV.CONFIG)); + +const baseUrl = config.kcLoadTest.baseUrl; +const clientSecret = config.kcLoadTest.clientSecret; + const getHeaders = (accessToken) => ({ Authorization: `Bearer ${accessToken}`, 'Content-Type': 'application/json', diff --git a/k6/tokenIntrospection.js b/k6/k6-runner/src/tests/tokenIntrospection.js similarity index 90% rename from k6/tokenIntrospection.js rename to k6/k6-runner/src/tests/tokenIntrospection.js index c6880510..a8e1ac2e 100644 --- a/k6/tokenIntrospection.js +++ b/k6/k6-runner/src/tests/tokenIntrospection.js @@ -1,8 +1,12 @@ import { sleep } from 'k6'; import { hitIntrospectionRoute, getAccessToken, createClient, deleteClient } from './helpers.js'; -import { username, password, clientId } from './env.js'; import { client } from './constants.js'; +let config = JSON.parse(open(__ENV.CONFIG)); +const username = config.kcLoadTest.username; +const password = config.kcLoadTest.password; +const clientId = config.kcLoadTest.clientId; + const CONCURRENT_LOOPS = 1; const ITERATIONS_PER_LOOP = 10; const LOOP_DELAY = 0.01; diff --git a/k6/userInfo.js b/k6/k6-runner/src/tests/userInfo.js similarity index 85% rename from k6/userInfo.js rename to k6/k6-runner/src/tests/userInfo.js index 2eea642f..766b51b9 100644 --- a/k6/userInfo.js +++ b/k6/k6-runner/src/tests/userInfo.js @@ -1,6 +1,10 @@ import { sleep } from 'k6'; import { getAccessToken, hitUserInfoRoute } from './helpers.js'; -import { username, password, clientId } from './env.js'; + +let config = JSON.parse(open(__ENV.CONFIG)); +const username = config.kcLoadTest.username; +const password = config.kcLoadTest.password; +const clientId = config.kcLoadTest.clientId; const CONCURRENT_LOOPS = 1; const ITERATIONS_PER_LOOP = 100;