From ce86ab620639ac6324d432e14c9b6cee857f9acc Mon Sep 17 00:00:00 2001 From: Vladislav Sukhin Date: Wed, 2 Oct 2024 15:51:39 +0300 Subject: [PATCH] feat: [TKC-2623] testkube kind Docker image (#5867) * feat: testkube kind Docker image Signed-off-by: Vladislav Sukhin * fix: tune entrypoint Signed-off-by: Vladislav Sukhin * fix: removed Signed-off-by: Vladislav Sukhin * fix: remove files Signed-off-by: Vladislav Sukhin * fix: add logs Signed-off-by: Vladislav Sukhin * feat: cloud image Signed-off-by: Vladislav Sukhin * Update build/kind/oss/Dockerfile.kind Co-authored-by: Dawid Rusnak * fix: remove org and env id Signed-off-by: Vladislav Sukhin * fix: move folder Signed-off-by: Vladislav Sukhin * fix: remove example Signed-off-by: Vladislav Sukhin * fix: add docker container env Signed-off-by: Vladislav Sukhin --------- Signed-off-by: Vladislav Sukhin Co-authored-by: Dawid Rusnak --- build/kind/Dockerfile.kind | 30 +++++ build/kind/entrypoint.sh | 139 +++++++++++++++++++++ build/kind/k6.yaml | 239 +++++++++++++++++++++++++++++++++++++ 3 files changed, 408 insertions(+) create mode 100644 build/kind/Dockerfile.kind create mode 100644 build/kind/entrypoint.sh create mode 100644 build/kind/k6.yaml diff --git a/build/kind/Dockerfile.kind b/build/kind/Dockerfile.kind new file mode 100644 index 0000000000..30432dee86 --- /dev/null +++ b/build/kind/Dockerfile.kind @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile:1 +# Step 1: Use a base image with Docker installed +FROM docker:20.10.24-dind + +# Step 2: Install necessary dependencies (curl, bash, tini) +RUN apk add --no-cache bash curl tini + +# Step 3: Install Kind (Kubernetes in Docker) +RUN curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 && \ + chmod +x ./kind && \ + mv ./kind /usr/local/bin/kind + +# Step 4: Install kubectl (for interacting with the Kubernetes cluster) +RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \ + chmod +x ./kubectl && \ + mv ./kubectl /usr/local/bin/kubectl + +# Step 5: Install Helm (package manager for Kubernetes) +RUN curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + +# Step 6: Script to automatically create Kind cluster and install Testkube +COPY entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +# Step 7: Example K6 Test Workflow CRD and preload Kind images +COPY ./images /images +COPY k6.yaml /examples/k6.yaml + +# Step 8: Set Docker entry point for DIND (Docker-in-Docker) +ENTRYPOINT ["tini", "--", "/usr/local/bin/entrypoint.sh"] diff --git a/build/kind/entrypoint.sh b/build/kind/entrypoint.sh new file mode 100644 index 0000000000..e02303766f --- /dev/null +++ b/build/kind/entrypoint.sh @@ -0,0 +1,139 @@ +#!/bin/bash + +# Turn on bash's job control +set -m + +# Logging function to make it easier to debug +log() { + echo "[INFO] $1" +} + +# Check if agent key is provided +if [ -z "$AGENT_KEY" ]; then + log "Please provide AGENT_KEY env var" + exit 1 +fi + +# Check if cloud url is provided +if [ -z "$CLOUD_URL" ]; then + log "Please provide CLOUD_URL env var" + exit 1 +fi + +# Step 1: Start docker service in background +/usr/local/bin/dockerd-entrypoint.sh & + +# Step 2: Wait that the docker service is up +while ! docker info; do + log "Waiting docker for 5 seconds..." + sleep 5 +done + +# Step 3: Import pre-installed images +for file in /images/*.tar; do + log "Load docker image $file..." + docker load <$file +done + +# Step 4: Create Kind cluster using a specific Kubernetes version +log "Creating Kubernetes cluster using Kind (Kubernetes v1.31.0)..." +kind create cluster --name testkube-cluster --image kindest/node:v1.31.0 --wait 5m +if [ $? -ne 0 ]; then + log "Failed to create Kind cluster." + exit 1 +fi + +# Step 5: Verify kubectl is connected to the cluster +log "Verifying cluster is up..." +kubectl cluster-info +if [ $? -ne 0 ]; then + log "Failed to verify cluster." + exit 1 +fi + +# Step 6: Add the Testkube Helm repository +log "Adding Testkube Helm repository..." +helm repo add testkube https://kubeshop.github.io/helm-charts +helm repo update + +# Step 7: Install Testkube using Helm +log "Installing Testkube via Helm..." +helm install testkube testkube/testkube --namespace testkube --create-namespace --set testkube-api.cloud.key=$AGENT_KEY --set testkube-api.minio.enabled=false --set mongodb.enabled=false --set testkube-dashboard.enabled=false --set testkube-api.cloud.url=$CLOUD_URL --set testkube-api.containerEnv=docker +if [ $? -ne 0 ]; then + log "Testkube installation failed." + exit 1 +fi + +# Step 8: Verify Testkube is installed to the cluster +log "Verifying Testkube is up..." +counter=0 +log_pattern="starting Testkube API server" +while [ $counter -lt 15 ] +do + # Get all pod statuses in the Testkube namespace + pod_status=$(kubectl get pods -n testkube --no-headers) + + # Check if there are any pods in the Testkube namespace + if [ -z "$pod_status" ]; then + log "No pods found in Testkube namespace." + exit 1 + fi + + # Iterate through each pod, check status and log pattern + all_running=true + found_pattern=false + + log "Checking pods in Testkube namespace..." + while read -r line; do + pod_name=$(echo "$line" | awk '{print $1}') + status=$(echo "$line" | awk '{print $3}') + + if [ "$status" != "Running" ]; then + log "Pod $pod_name is not running. Status: $status." + all_running=false + break + else + log "Pod $pod_name is running." + fi + + if [[ $pod_name == *"testkube-api-server"* ]]; then + pod_logs=$(kubectl logs "$pod_name" -n testkube) + + # Check if logs contain the desired pattern + if echo "$pod_logs" | grep -q "$log_pattern"; then + log "Log pattern found: $log_pattern." + found_pattern=true + else + log "Log pattern not found: $log_pattern." + break + fi + fi + done <<< "$pod_status" + + if [ "$all_running" = true ] && [ "$found_pattern" = true ] ; then + log "Waiting Testkube API for 30 seconds..." + sleep 30 + break + else + log "Waiting Testkube for 30 seconds..." + sleep 30 + fi + + counter=$(( counter + 1 )) +done + +if [ $counter -eq 15 ]; then + log "Testkube validation failed." + exit 1 +fi +log "Testkube is up and running." + +# Step 9: Create Testkube k6 Test Workflow +log "Creating and running Testkube k6 Test Workflow..." +kubectl apply -f /examples/k6.yaml -n testkube + +log "Testkube installation successful!" +log "You can now use Testkube in your Kind Kubernetes cluster." + +# Step 10: Bring docker service back to foreground +fg %1 diff --git a/build/kind/k6.yaml b/build/kind/k6.yaml new file mode 100644 index 0000000000..ca3c637fef --- /dev/null +++ b/build/kind/k6.yaml @@ -0,0 +1,239 @@ +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: k6-workflow-smoke + labels: + core-tests: workflows +spec: + content: + git: + uri: https://github.com/kubeshop/testkube + revision: main + paths: + - test/k6/executor-tests/k6-smoke-test.js + container: + resources: + requests: + cpu: 128m + memory: 128Mi + workingDir: /data/repo/test/k6/executor-tests + steps: + - name: Run test + run: + image: grafana/k6:0.43.1 + args: + - run + - k6-smoke-test.js + - -e + - K6_ENV_FROM_PARAM=K6_ENV_FROM_PARAM_value + env: + - name: K6_SYSTEM_ENV + value: K6_SYSTEM_ENV_value +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: k6-workflow-smoke-template + labels: + core-tests: workflows +spec: + container: + resources: + requests: + cpu: 128m + memory: 128Mi + workingDir: /data/repo/test/k6/executor-tests + env: + - name: K6_SYSTEM_ENV # currently only possible on this level + value: K6_SYSTEM_ENV_value + steps: + - name: Run from template + workingDir: /data/repo/test/k6/executor-tests + content: + git: + uri: https://github.com/kubeshop/testkube + revision: main + paths: + - test/k6/executor-tests/k6-smoke-test.js + template: + name: official/k6/v1 + config: + version: 0.48.0 + run: "k6 run k6-smoke-test.js -e K6_ENV_FROM_PARAM=K6_ENV_FROM_PARAM_value" +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: k6-workflow-smoke-template-without-checkout-step + labels: + core-tests: workflows +spec: + content: + git: + uri: https://github.com/kubeshop/testkube + revision: main + paths: + - test/k6/executor-tests/k6-smoke-test.js + container: + resources: + requests: + cpu: 128m + memory: 128Mi + workingDir: /data/repo/test/k6/executor-tests + env: + - name: K6_SYSTEM_ENV # currently only possible on this level + value: K6_SYSTEM_ENV_value + steps: + - name: Run from template + workingDir: /data/repo/test/k6/executor-tests + template: + name: official/k6/v1 + config: + version: 0.48.0 + run: "k6 run k6-smoke-test.js -e K6_ENV_FROM_PARAM=K6_ENV_FROM_PARAM_value" +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: k6-workflow-smoke-artifacts + labels: + core-tests: workflows +spec: + content: + git: + uri: https://github.com/kubeshop/testkube + revision: main + paths: + - test/k6/executor-tests/k6-smoke-test.js + container: + resources: + requests: + cpu: 128m + memory: 128Mi + workingDir: /data/repo/test/k6/executor-tests + steps: + - name: Run test + container: + image: grafana/k6:0.49.0 + steps: + - shell: mkdir /data/artifacts + - run: + args: + - run + - k6-smoke-test.js + - -e + - K6_ENV_FROM_PARAM=K6_ENV_FROM_PARAM_value + env: + - name: K6_SYSTEM_ENV + value: K6_SYSTEM_ENV_value + - name: K6_WEB_DASHBOARD + value: "true" + - name: K6_WEB_DASHBOARD_EXPORT + value: "/data/artifacts/k6-test-report.html" + steps: + - name: Saving artifacts + workingDir: /data/artifacts + artifacts: + paths: + - '*' +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: distributed-k6-workflow-smoke + labels: + core-tests: workflows +spec: + config: + vus: {type: integer, default: 2} + duration: {type: string, default: '2s'} + workers: {type: integer, default: 3} + content: + git: + uri: https://github.com/kubeshop/testkube + revision: main + paths: + - test/k6/executor-tests/k6-smoke-test.js + steps: + - name: Run test + parallel: + count: 'config.workers' + transfer: + - from: /data/repo + use: + - name: distribute/evenly + container: + workingDir: /data/repo/test/k6/executor-tests + resources: + requests: + cpu: 128m + memory: 128Mi + env: + - name: K6_SYSTEM_ENV + value: K6_SYSTEM_ENV_value + paused: true # synchronize running all workers + run: + image: grafana/k6:0.49.0 + shell: | + k6 run k6-smoke-test.js \ + -e K6_ENV_FROM_PARAM=K6_ENV_FROM_PARAM_value \ + --vus {{ shellquote(config.vus) }} \ + --duration {{ shellquote(config.duration) }} \ + --execution-segment {{ index }}/{{ count }}:{{ index + 1 }}/{{ count }} +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: distributed-k6-workflow-smoke-artifacts + labels: + core-tests: workflows +spec: + config: + vus: {type: integer, default: 2} + duration: {type: string, default: '2s'} + workers: {type: integer, default: 3} + content: + git: + uri: https://github.com/kubeshop/testkube + revision: main + paths: + - test/k6/executor-tests/k6-smoke-test.js + steps: + - name: Run test + parallel: + count: 'config.workers' + transfer: + - from: /data/repo + use: + - name: distribute/evenly + container: + resources: + requests: + cpu: 128m + memory: 128Mi + paused: true # synchronise running all workers + run: + image: grafana/k6:0.49.0 + workingDir: /data/repo/test/k6/executor-tests + args: + - run + - k6-smoke-test.js + - -e + - K6_ENV_FROM_PARAM=K6_ENV_FROM_PARAM_value + - --vus + - '{{ config.vus }}' + - --duration + - '{{ config.duration }}' + - --execution-segment + - '{{ index }}/{{ count }}:{{ index + 1 }}/{{ count }}' + env: + - name: K6_SYSTEM_ENV + value: K6_SYSTEM_ENV_value + - name: K6_WEB_DASHBOARD + value: "true" + - name: K6_WEB_DASHBOARD_EXPORT + value: "/data/k6-test-report.html" + artifacts: + workingDir: /data + paths: + - '*.html'