feat: executor tests - container executor for jmeter and soapui, play…

…wright artifacts fixed, run script updated (#5045) * executor tests - container executor - soapui * run script extended - container-soapui-smoke * executor tests - container executor - soapui - artifacts * executor tests - container executor - jmeter * executor tests - container executor - jmeter - command fixed, soapui artifacts commented out * executor tests - cypress container - video artifacts only, playwright container - playwright-report artifacts fixed * empty lines added * empty lines added
kubeshop · Feb 22, 2024 · b9233fc · b9233fc
1 parent 24cc463
commit b9233fc
Show file tree

Hide file tree

Showing 11 changed files with 178 additions and 2 deletions.
diff --git a/test/container-executor/executor-smoke/crd/cypress.yaml b/test/container-executor/executor-smoke/crd/cypress.yaml
@@ -67,3 +67,38 @@ spec:
         - ./
     jobTemplate: "apiVersion: batch/v1\nkind: Job\nspec:\n  template:\n    spec:\n      containers:\n        - name: \"{{ .Name }}\"\n          image: {{ .Image }}\n          resources:\n            requests:\n              memory: 2Gi\n              cpu: 2\n"
     activeDeadlineSeconds: 600
+---
+apiVersion: tests.testkube.io/v3
+kind: Test
+metadata:
+  name: container-executor-cypress-v12.7.0-video-artifacts-only
+  labels:
+    core-tests: executors
+spec:
+  type: container-executor-cypress-v12.7.0/test
+  content:
+    type: git-dir
+    repository:
+      type: git-dir
+      uri: https://github.com/kubeshop/testkube
+      branch: main
+      path: test/cypress/executor-tests/cypress-12
+      workingDir: test/cypress/executor-tests/cypress-12
+  executionRequest:
+    variables:
+      CYPRESS_CUSTOM_ENV:
+        name: CYPRESS_CUSTOM_ENV
+        value: CYPRESS_CUSTOM_ENV_value
+        type: basic
+    args:
+      - --env
+      - NON_CYPRESS_ENV=NON_CYPRESS_ENV_value
+      - --config
+      - '{"screenshotsFolder":"/data/artifacts/screenshots","videosFolder":"/data/artifacts/videos"}'
+    artifactRequest:
+      storageClassName: standard
+      volumeMountPath: /data/artifacts/videos
+      dirs:
+        - ./
+    jobTemplate: "apiVersion: batch/v1\nkind: Job\nspec:\n  template:\n    spec:\n      containers:\n        - name: \"{{ .Name }}\"\n          image: {{ .Image }}\n          resources:\n            requests:\n              memory: 2Gi\n              cpu: 2\n"
+    activeDeadlineSeconds: 600
diff --git a/test/container-executor/executor-smoke/crd/jmeter.yaml b/test/container-executor/executor-smoke/crd/jmeter.yaml
@@ -0,0 +1,26 @@
+apiVersion: tests.testkube.io/v3
+kind: Test
+metadata:
+  name: container-executor-jmeter-smoke
+  labels:
+    core-tests: executors
+spec:
+  type: container-executor-jmeter-5.5/test
+  content:
+    type: git
+    repository:
+      type: git
+      uri: https://github.com/kubeshop/testkube.git
+      branch: main
+      path: test/jmeter/executor-tests/jmeter-executor-smoke.jmx
+      workingDir: test/jmeter/executor-tests
+  executionRequest:
+    executePostRunScriptBeforeScraping: true
+    postRunScript: "echo 'post-run script' && cd /data/artifacts && ls -lah"
+    args:
+      - "-n -t jmeter-executor-smoke.jmx -j /data/artifacts/jmeter.log -o /data/artifacts/report -l /data/artifacts/jtl-report.jtl -e"
+    artifactRequest:
+      storageClassName: standard
+      volumeMountPath: /data/artifacts
+      dirs:
+        - ./
diff --git a/test/container-executor/executor-smoke/crd/playwright.yaml b/test/container-executor/executor-smoke/crd/playwright.yaml
@@ -25,6 +25,11 @@ spec:
     preRunScript: "npm ci"
     args:
       - "tests/smoke2.spec.js"
+    variables:
+      PLAYWRIGHT_HTML_REPORT:
+        name: PLAYWRIGHT_HTML_REPORT
+        value: "/data/artifacts/playwright-report"
+        type: basic
 ---
 apiVersion: tests.testkube.io/v3
 kind: Test
@@ -50,3 +55,8 @@ spec:
         - ./
     jobTemplate: "apiVersion: batch/v1\nkind: Job\nspec:\n  template:\n    spec:\n      containers:\n        - name: \"{{ .Name }}\"\n          image: {{ .Image }}\n          resources:\n            requests:\n              memory: 2Gi\n              cpu: 2\n"
     activeDeadlineSeconds: 600
+    variables:
+      PLAYWRIGHT_HTML_REPORT:
+        name: PLAYWRIGHT_HTML_REPORT
+        value: "/data/artifacts/playwright-report"
+        type: basic
diff --git a/test/container-executor/executor-smoke/crd/soapui.yaml b/test/container-executor/executor-smoke/crd/soapui.yaml
@@ -0,0 +1,28 @@
+apiVersion: tests.testkube.io/v3
+kind: Test
+metadata:
+  name: container-executor-soapui-smoke
+  labels:
+    core-tests: executors
+spec:
+  type: container-executor-soapui-5.7/test
+  content:
+    type: git
+    repository:
+      type: git
+      uri: https://github.com/kubeshop/testkube
+      branch: main
+      path: test/soapui/executor-smoke/soapui-smoke-test.xml
+  executionRequest:
+    jobTemplate: "apiVersion: batch/v1\nkind: Job\nspec:\n  template:\n    spec:\n      containers:\n        - name: \"{{ .Name }}\"\n          image: {{ .Image }}\n          resources:\n            requests:\n              memory: 256Mi\n              cpu: 512m\n"
+    activeDeadlineSeconds: 180
+    variables:
+      COMMAND_LINE:
+        name: COMMAND_LINE
+        value: "-r -f /reports -a -j /data/repo/test/soapui/executor-smoke/soapui-smoke-incorrect-name.xml"
+        type: basic
+    # artifactRequest: # TODO: temporary disabled - not working for some reason
+    #   storageClassName: standard
+    #   volumeMountPath: /artifacts
+    #   dirs:
+    #     - ./
diff --git a/test/executors/container-executor-jmeter.yaml b/test/executors/container-executor-jmeter.yaml
@@ -0,0 +1,13 @@
+apiVersion: executor.testkube.io/v1
+kind: Executor
+metadata:
+  name: container-executor-jmeter-5.5
+spec:
+  image: justb4/jmeter:5.5
+  command:
+  - "jmeter"
+  executor_type: container
+  types:
+  - container-executor-jmeter-5.5/test
+  features:
+  - artifacts
diff --git a/test/executors/container-executor-playwright.yaml b/test/executors/container-executor-playwright.yaml
@@ -4,7 +4,7 @@ metadata:
   name: container-executor-playwright-v1.32.3-args
 spec:
   image: mcr.microsoft.com/playwright:v1.32.3-focal
-  command: ["npx", "--yes", "[email protected]", "test", "--output", "/data/artifacts"]
+  command: ["npx", "--yes", "[email protected]", "test", "--output", "/data/artifacts/playwright-results"]
   executor_type: container
   types:
   - container-executor-playwright-v1.32.3-args/test
@@ -19,7 +19,7 @@ spec:
   image: mcr.microsoft.com/playwright:v1.32.3-focal
   command: ["/bin/sh", "-c"]
   args:
-  - "npm ci && CI=1 npx --yes [email protected] test --output /data/artifacts"
+  - "npm ci && CI=1 npx --yes [email protected] test --output /data/artifacts/playwright-results"
   executor_type: container
   types:
   - container-executor-playwright-v1.32.3/test

diff --git a/test/executors/container-executor-soapui.yaml b/test/executors/container-executor-soapui.yaml
@@ -0,0 +1,11 @@
+apiVersion: executor.testkube.io/v1
+kind: Executor
+metadata:
+  name: container-executor-soapui-5.7
+spec:
+  image: smartbear/soapuios-testrunner:5.7.2
+  executor_type: container
+  types:
+  - container-executor-soapui-5.7/test
+  features:
+  - artifacts
diff --git a/test/scripts/executor-tests/run.sh b/test/scripts/executor-tests/run.sh
@@ -153,6 +153,17 @@ container-gradle-smoke() {
   common_run "$name" "$test_crd_file" "$testsuite_name" "$testsuite_file" "$custom_executor_crd_file"
 }
 
+container-jmeter-smoke() {
+  name="Container executor - JMeter"
+  test_crd_file="test/container-executor/executor-smoke/crd/jmeter.yaml"
+  testsuite_name="executor-container-jmeter-smoke-tests"
+  testsuite_file="test/suites/executor-container-jmeter-smoke-tests.yaml"
+
+  custom_executor_crd_file="test/executors/container-executor-jmeter.yaml"
+
+  common_run "$name" "$test_crd_file" "$testsuite_name" "$testsuite_file" "$custom_executor_crd_file"
+}
+
 container-k6-smoke() {
   name="Container executor - K6"
   test_crd_file="test/container-executor/executor-smoke/crd/k6.yaml"
@@ -197,6 +208,17 @@ container-postman-smoke() {
   common_run "$name" "$test_crd_file" "$testsuite_name" "$testsuite_file" "$custom_executor_crd_file"
 }
 
+container-soapui-smoke() {
+  name="Container executor - SoapUI"
+  test_crd_file="test/container-executor/executor-smoke/crd/soapui.yaml"
+  testsuite_name="executor-container-soapui-smoke-tests"
+  testsuite_file="test/suites/executor-container-soapui-smoke-tests.yaml"
+
+  custom_executor_crd_file="test/executors/container-executor-soapui.yaml"
+
+  common_run "$name" "$test_crd_file" "$testsuite_name" "$testsuite_file" "$custom_executor_crd_file"
+}
+
 curl-smoke() {
   name="curl"
   test_crd_file="test/curl/executor-tests/crd/smoke.yaml"
@@ -367,10 +389,12 @@ main() {
       container-curl-smoke
       container-cypress-smoke
       container-gradle-smoke
+      container-jmeter-smoke
       container-k6-smoke
       container-maven-smoke
       container-postman-smoke
       container-playwright-smoke
+      container-soapui-smoke
       curl-smoke
       cypress-smoke
       ginkgo-smoke
@@ -390,10 +414,12 @@ main() {
       container-curl-smoke
       container-cypress-smoke
       container-gradle-smoke
+      container-jmeter-smoke
       container-k6-smoke
       container-maven-smoke
       container-postman-smoke
       container-playwright-smoke
+      container-soapui-smoke
       curl-smoke
       cypress-smoke
       ginkgo-smoke

diff --git a/test/suites/executor-container-cypress-smoke-tests.yaml b/test/suites/executor-container-cypress-smoke-tests.yaml
@@ -13,3 +13,6 @@ spec:
   - stopOnFailure: false
     execute:
     - test: container-executor-cypress-v12.7.0-smoke-git-dir
+  - stopOnFailure: false
+    execute:
+    - test: container-executor-cypress-v12.7.0-video-artifacts-only
diff --git a/test/suites/executor-container-jmeter-smoke-tests.yaml b/test/suites/executor-container-jmeter-smoke-tests.yaml
@@ -0,0 +1,12 @@
+apiVersion: tests.testkube.io/v3
+kind: TestSuite
+metadata:
+  name: executor-container-jmeter-smoke-tests
+  labels:
+    app: testkube
+spec:
+  description: "container executor jmeter smoke tests"
+  steps:
+  - stopOnFailure: false
+    execute:
+    - test: container-executor-jmeter-smoke
diff --git a/test/suites/executor-container-soapui-smoke-tests.yaml b/test/suites/executor-container-soapui-smoke-tests.yaml
@@ -0,0 +1,12 @@
+apiVersion: tests.testkube.io/v3
+kind: TestSuite
+metadata:
+  name: executor-container-soapui-smoke-tests
+  labels:
+    app: testkube
+spec:
+  description: "container executor soapui smoke tests"
+  steps:
+  - stopOnFailure: false
+    execute:
+    - test: container-executor-soapui-smoke