From 944790b7aead9325b64d42d1a3e911383f90564b Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Mon, 13 Mar 2023 17:02:32 +0100 Subject: [PATCH 1/6] Test: added performance test script for irma server --- README.md | 19 +++++--- testdata/performance/irma-server.js | 69 +++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 6 deletions(-) create mode 100644 testdata/performance/irma-server.js diff --git a/README.md b/README.md index 8d7d3eda1..c326f27e9 100644 --- a/README.md +++ b/README.md @@ -122,14 +122,21 @@ irma server -vv --store-type redis --redis-addr "localhost:6379" --redis-allow-e ``` ## Performance tests -This project only includes performance tests for the `irma keyshare server`. These tests can be run using the [k6 load testing tool](https://k6.io/docs/) and need a running keyshare server instance to test against. Instructions on how to run a keyshare server locally can be found [above](#running). +This project only includes performance tests for the `irma server` and the `irma keyshare server`. These tests can be run using the [k6 load testing tool](https://k6.io/docs/) and need a running server instance to test against. -The performance tests can be started in the following way: +Instructions on how to run `irma server` locally with a Redis datastore can be found [here](#using-a-local-redis-datastore). Instructions on how to run a keyshare server locally can be found [here](#running). -``` -go install go.k6.io/k6@latest -k6 run ./testdata/performance/keyshare-server.js --env URL=http://localhost:8080 --env ISSUER_ID=test.test -``` +First, you need to install `k6`: + + go install go.k6.io/k6@latest + +The performance tests of the `irma server` can be started in the following way: + + k6 run ./testdata/performance/irma-server.js --env URL=http://localhost:8088 + +The performance tests of the keyshare server can be started in the following way: + + k6 run ./testdata/performance/keyshare-server.js --env URL=http://localhost:8080 --env ISSUER_ID=test.test By default, k6 runs a single test iteration using 1 virtual user. These defaults can be adjusted by specifying test stages using the [`-s` CLI parameter](https://k6.io/docs/using-k6/options/#stages). diff --git a/testdata/performance/irma-server.js b/testdata/performance/irma-server.js new file mode 100644 index 000000000..1df4c6859 --- /dev/null +++ b/testdata/performance/irma-server.js @@ -0,0 +1,69 @@ +import { check, fail, sleep } from 'k6'; +import http from 'k6/http'; + +const url = __ENV.URL; + +export const options = { + minIterationDuration: '30s', + thresholds: { + http_req_failed: ['rate<0.01'], // http errors should be less than 1% + http_req_duration: ['p(95)<200'], // 95% of requests should be below 200ms + }, +}; + +function checkResponse(response, expectedOutput = '') { + const checkOutput = check(response, { + 'verify status code': (r) => r.status === 200, + 'verify body': (r) => r.body.includes(expectedOutput), + }); + if (!checkOutput) fail('unexpected response'); +} + +export default function () { + const newSessionResp = http.post(`${url}/session`, JSON.stringify({ + "@context": "https://irma.app/ld/request/disclosure/v2", + "disclose": [ + [ + [ "irma-demo.sidn-pbdf.email.email" ] + ] + ] + }), { + headers: { + 'Content-Type': 'application/json', + }, + }); + checkResponse(newSessionResp); + + const sessionPackage = newSessionResp.json(); + const sessionPtrUrl = sessionPackage.sessionPtr.u; + + for (let i = 0; i < 10; i++) { + let statusResp = http.get(`${sessionPtrUrl}/status`); + checkResponse(statusResp, 'INITIALIZED'); + sleep(1); + } + + const sessionResp = http.get(sessionPtrUrl, { + headers: { + 'Authorization': '12345', + 'X-IRMA-MinProtocolVersion': '2.8', + 'X-IRMA-MaxProtocolVersion': '2.8', + }, + }); + checkResponse(sessionResp, '"protocolVersion":"2.8"'); + + for (let i = 0; i < 20; i++) { + let statusResp = http.get(`${sessionPtrUrl}/status`); + checkResponse(statusResp, 'CONNECTED'); + sleep(1); + } + + const sessionDeletedResp = http.del(sessionPtrUrl); + checkResponse(sessionDeletedResp); + + let statusResp = http.get(`${sessionPtrUrl}/status`); + checkResponse(statusResp, 'CANCELLED'); + + let sessionResultResp = http.get(`${url}/session/${sessionPackage.token}/result`); + checkResponse(sessionResultResp, 'CANCELLED'); +} From 7bd85bfbe5ed0ee3927b2583540734a536d3445d Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Tue, 24 Oct 2023 10:56:20 +0200 Subject: [PATCH 2/6] Improvement: let performance test check status codes --- testdata/performance/irma-server.js | 4 ++-- testdata/performance/keyshare-server.js | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/testdata/performance/irma-server.js b/testdata/performance/irma-server.js index 1df4c6859..10e8e4548 100644 --- a/testdata/performance/irma-server.js +++ b/testdata/performance/irma-server.js @@ -16,7 +16,7 @@ function checkResponse(response, expectedOutput = '') { 'verify status code': (r) => r.status === 200, 'verify body': (r) => r.body.includes(expectedOutput), }); - if (!checkOutput) fail('unexpected response'); + if (!checkOutput) fail('unexpected response: status ${response.status}'); } export default function () { @@ -24,7 +24,7 @@ export default function () { "@context": "https://irma.app/ld/request/disclosure/v2", "disclose": [ [ - [ "irma-demo.sidn-pbdf.email.email" ] + ["irma-demo.sidn-pbdf.email.email"] ] ] }), { diff --git a/testdata/performance/keyshare-server.js b/testdata/performance/keyshare-server.js index 0599adcda..2718be8e2 100644 --- a/testdata/performance/keyshare-server.js +++ b/testdata/performance/keyshare-server.js @@ -1,4 +1,4 @@ -import { fail } from 'k6'; +import { check, fail } from 'k6'; import { instance, vu } from 'k6/execution'; import http from 'k6/http'; @@ -14,6 +14,14 @@ export const options = { }, }; +function checkResponse(response, expectedOutput = '') { + const checkOutput = check(response, { + 'verify status code': (r) => r.status === 200, + 'verify body': (r) => r.body.includes(expectedOutput), + }); + if (!checkOutput) fail('unexpected response: status ${response.status}'); +} + export function setup() { if (!url || !issuerID) { fail('Must specify URL and ISSUER_ID options via environment variables'); @@ -27,7 +35,7 @@ export function setup() { const registerPayloadStr = JSON.stringify(registerPayload); // An IRMA account cannot be used in parallel, so every VU needs its own account. - const testAccounts = Array.from({length: instance.vusInitialized}, () => { + const testAccounts = Array.from({ length: instance.vusInitialized }, () => { const registerResp = http.post(`${url}/client/register`, registerPayloadStr, { headers: { 'Content-Type': 'application/json', @@ -41,8 +49,9 @@ export function setup() { 'X-IRMA-MaxProtocolVersion': '2.8', }, }); + checkResponse(sessionResp); - http.del(registerResp.json().u); + checkResponse(http.del(registerResp.json().u)); return { id: Object.values(sessionResp.json().request.credentials[0].attributes)[0], @@ -67,7 +76,7 @@ export default function ({ testAccounts }) { }, }; - http.post(`${url}/prove/getCommitments`, `["${issuerID}-0"]`, proveParams); + checkResponse(http.post(`${url}/prove/getCommitments`, `["${issuerID}-0"]`, proveParams)); - http.post(`${url}/prove/getResponse`, '"5adEmlEg9U2zjNlPxyPvRym2AzWkBo4kIZJ7ytNg0q0="', proveParams); + checkResponse(http.post(`${url}/prove/getResponse`, '"5adEmlEg9U2zjNlPxyPvRym2AzWkBo4kIZJ7ytNg0q0="', proveParams)); } From 6a759bc781544d8d885d3702bd7ae40ef33e479c Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Tue, 24 Oct 2023 12:09:46 +0200 Subject: [PATCH 3/6] Test: fix template string in performance test --- testdata/performance/irma-server.js | 2 +- testdata/performance/keyshare-server.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testdata/performance/irma-server.js b/testdata/performance/irma-server.js index 10e8e4548..742374ed6 100644 --- a/testdata/performance/irma-server.js +++ b/testdata/performance/irma-server.js @@ -16,7 +16,7 @@ function checkResponse(response, expectedOutput = '') { 'verify status code': (r) => r.status === 200, 'verify body': (r) => r.body.includes(expectedOutput), }); - if (!checkOutput) fail('unexpected response: status ${response.status}'); + if (!checkOutput) fail(`unexpected response: status ${response.status}`); } export default function () { diff --git a/testdata/performance/keyshare-server.js b/testdata/performance/keyshare-server.js index 2718be8e2..e349ea6ef 100644 --- a/testdata/performance/keyshare-server.js +++ b/testdata/performance/keyshare-server.js @@ -19,7 +19,7 @@ function checkResponse(response, expectedOutput = '') { 'verify status code': (r) => r.status === 200, 'verify body': (r) => r.body.includes(expectedOutput), }); - if (!checkOutput) fail('unexpected response: status ${response.status}'); + if (!checkOutput) fail(`unexpected response: status ${response.status}`); } export function setup() { From be5f6004d6000ec2ee1c0db1d18eb782e45737e6 Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Wed, 25 Oct 2023 11:05:05 +0200 Subject: [PATCH 4/6] Test: fix keyshare and myirmaserver test configuration for Docker setup --- testdata/configurations/keyshareserver.yml | 1 + testdata/configurations/myirmaserver.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/testdata/configurations/keyshareserver.yml b/testdata/configurations/keyshareserver.yml index 1d234de2c..ae4f9f669 100644 --- a/testdata/configurations/keyshareserver.yml +++ b/testdata/configurations/keyshareserver.yml @@ -1,6 +1,7 @@ verbose: 2 schemes_path: testdata/irma_configuration +schemes_assets_path: testdata/irma_configuration # To prevent assets from Docker image to be used schemes_update: 0 privkeys: testdata/privatekeys url: http://localhost:8080/ diff --git a/testdata/configurations/myirmaserver.yml b/testdata/configurations/myirmaserver.yml index 90ca1fade..f6d841494 100644 --- a/testdata/configurations/myirmaserver.yml +++ b/testdata/configurations/myirmaserver.yml @@ -1,6 +1,7 @@ verbose: 2 schemes_path: testdata/irma_configuration +schemes_assets_path: testdata/irma_configuration # To prevent assets from Docker image to be used schemes_update: 0 url: http://localhost:port/ port: 8081 From 5484d91cbb063ab6a6cdfc31f23968facbbdb905 Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Wed, 25 Oct 2023 11:05:34 +0200 Subject: [PATCH 5/6] Test: improve error logging in performance tests --- testdata/performance/irma-server.js | 5 +++-- testdata/performance/keyshare-server.js | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/testdata/performance/irma-server.js b/testdata/performance/irma-server.js index 742374ed6..3c4b997a2 100644 --- a/testdata/performance/irma-server.js +++ b/testdata/performance/irma-server.js @@ -13,10 +13,11 @@ export const options = { function checkResponse(response, expectedOutput = '') { const checkOutput = check(response, { + 'verify response': (r) => r.error === '', 'verify status code': (r) => r.status === 200, - 'verify body': (r) => r.body.includes(expectedOutput), + 'verify body': (r) => r.body != null && r.body.includes(expectedOutput), }); - if (!checkOutput) fail(`unexpected response: status ${response.status}`); + if (!checkOutput) fail(`unexpected response: status ${response.status}, error "${response.error}", body "${response.body}"`); } export default function () { diff --git a/testdata/performance/keyshare-server.js b/testdata/performance/keyshare-server.js index e349ea6ef..5c84e3a3e 100644 --- a/testdata/performance/keyshare-server.js +++ b/testdata/performance/keyshare-server.js @@ -16,10 +16,11 @@ export const options = { function checkResponse(response, expectedOutput = '') { const checkOutput = check(response, { + 'verify response': (r) => r.error === '', 'verify status code': (r) => r.status === 200, - 'verify body': (r) => r.body.includes(expectedOutput), + 'verify body': (r) => r.body != null && r.body.includes(expectedOutput), }); - if (!checkOutput) fail(`unexpected response: status ${response.status}`); + if (!checkOutput) fail(`unexpected response: status ${response.status}, error "${response.error}", body "${response.body}"`); } export function setup() { @@ -41,6 +42,7 @@ export function setup() { 'Content-Type': 'application/json', }, }); + checkResponse(registerResp); const sessionResp = http.get(registerResp.json().u, { headers: { From 09c4d8adf6f6e82d157f03d019185f2330df177f Mon Sep 17 00:00:00 2001 From: Ivar Derksen Date: Wed, 25 Oct 2023 11:21:57 +0200 Subject: [PATCH 6/6] Test: log request url on request failure in performance test --- testdata/performance/irma-server.js | 2 +- testdata/performance/keyshare-server.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/testdata/performance/irma-server.js b/testdata/performance/irma-server.js index 3c4b997a2..8aa5f208f 100644 --- a/testdata/performance/irma-server.js +++ b/testdata/performance/irma-server.js @@ -17,7 +17,7 @@ function checkResponse(response, expectedOutput = '') { 'verify status code': (r) => r.status === 200, 'verify body': (r) => r.body != null && r.body.includes(expectedOutput), }); - if (!checkOutput) fail(`unexpected response: status ${response.status}, error "${response.error}", body "${response.body}"`); + if (!checkOutput) fail(`unexpected response: url ${response.request.url}, status ${response.status}, error "${response.error}", body "${response.body}"`); } export default function () { diff --git a/testdata/performance/keyshare-server.js b/testdata/performance/keyshare-server.js index 5c84e3a3e..4bba16adf 100644 --- a/testdata/performance/keyshare-server.js +++ b/testdata/performance/keyshare-server.js @@ -20,7 +20,7 @@ function checkResponse(response, expectedOutput = '') { 'verify status code': (r) => r.status === 200, 'verify body': (r) => r.body != null && r.body.includes(expectedOutput), }); - if (!checkOutput) fail(`unexpected response: status ${response.status}, error "${response.error}", body "${response.body}"`); + if (!checkOutput) fail(`unexpected response: url ${response.request.url}, status ${response.status}, error "${response.error}", body "${response.body}"`); } export function setup() { @@ -70,6 +70,7 @@ export default function ({ testAccounts }) { const testAccount = testAccounts[vu.idInTest - 1]; const pinResp = http.post(`${url}/users/verify/pin`, JSON.stringify(testAccount)); + checkResponse(pinResp); const proveParams = { headers: {