From 25f2a1b5e46797fcbbf057127ec56376a32d93ef Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Mon, 27 Nov 2023 20:35:18 -0600 Subject: [PATCH 01/39] Add RHTAP build pipeline A few differences from the standard RHTAP build pipeline: * Use of 6GB buildah image to avoid OOMs during container build task * There is an extra step to fetch the vuln feed data. This is done outside of the build step to make hermetic builds easier. The vuln feed data script pulls the definitions from the Google storage location and writes them to the "source" folder of the "source" workspace which is shared by the build-container task. This task will build using `image/scanner/rhtap/Dockerfile`, which expects the vuln files to be in the buildah working directory. I decided to create a separate dockerfile for RHTAP to avoid any regressions with modifying the existing ones. The dockerfile was created using a combination of the upstream and downstream dockerfiles. Current RHTAP build trigger config: * PRs will only build in RHTAP when "rhtap" is in the branch name * Pushes to master will trigger an RHTAP build --- .containerignore | 9 + .github/workflows/style.yaml | 28 ++ .tekton/scanner-pull-request.yaml | 435 +++++++++++++++++++++++++++++ .tekton/scanner-push.yaml | 432 ++++++++++++++++++++++++++++ image/scanner/rhtap/Dockerfile | 62 ++++ scripts/ci/jobs/check-generated.sh | 31 ++ 6 files changed, 997 insertions(+) create mode 100644 .containerignore create mode 100644 .github/workflows/style.yaml create mode 100644 .tekton/scanner-pull-request.yaml create mode 100644 .tekton/scanner-push.yaml create mode 100644 image/scanner/rhtap/Dockerfile create mode 100755 scripts/ci/jobs/check-generated.sh diff --git a/.containerignore b/.containerignore new file mode 100644 index 000000000..d1cc8cef5 --- /dev/null +++ b/.containerignore @@ -0,0 +1,9 @@ +# This file is derived of the .dockerignore +# It is required to pass `.git` to the builder containers on RHTAP so we can bake build information into the binary. +# It is required to pass the `.gitignore` files in `image/bin/` and `bin` to avoid a dirty version. +# RHTAP uses Buildah which first looks up for `.containerignore`, then, if absent, `.dockerignore`. +# Having this file enables us to have different exclusion list for RHTAP builds than in GHA/OSCI. +# To regenerate, replace the below contents with the output of the following command (please keep these comments). +# grep -vF -e '/.git/' -e '/bin/' -e '/image/bin/' .dockerignore + +# currently empty! diff --git a/.github/workflows/style.yaml b/.github/workflows/style.yaml new file mode 100644 index 000000000..b12cd9af6 --- /dev/null +++ b/.github/workflows/style.yaml @@ -0,0 +1,28 @@ +name: Style +on: + push: + tags: + - '*' + branches: + - master + pull_request: + types: + - opened + - reopened + - synchronize +jobs: + check-generated-files: + runs-on: ubuntu-latest + container: + image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - uses: ./.github/actions/job-preamble + + - name: Check Generated + run: scripts/ci/jobs/check-generated.sh diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml new file mode 100644 index 000000000..2a0d85fe1 --- /dev/null +++ b/.tekton/scanner-pull-request.yaml @@ -0,0 +1,435 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/stackrox/scanner?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && source_branch.contains("rhtap") + creationTimestamp: null + labels: + appstudio.openshift.io/application: acs + appstudio.openshift.io/component: scanner + pipelines.appstudio.openshift.io/type: build + name: scanner-on-pull-request + namespace: rh-acs-tenant +spec: + params: + - name: dockerfile + value: image/scanner/rhtap/Dockerfile + - name: git-url + value: '{{repo_url}}' + - name: image-expires-after + value: 5d + - name: output-image + value: quay.io/redhat-user-workloads/rh-acs-tenant/acs/scanner:on-pr-{{revision}} + - name: path-context + value: . + - name: revision + value: '{{revision}}' + pipelineSpec: + finally: + - name: show-sbom + params: + - name: IMAGE_URL + value: $(tasks.build-container.results.IMAGE_URL) + taskRef: + params: + - name: name + value: show-sbom + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-show-sbom:0.1@sha256:202d3c3385120ea847d8f0a82bd8d9d5e873d67f981d6f8a51fb1706caaf6bef + - name: kind + value: task + resolver: bundles + - name: show-summary + params: + - name: pipelinerun-name + value: $(context.pipelineRun.name) + - name: git-url + value: $(tasks.clone-repository.results.url)?rev=$(tasks.clone-repository.results.commit) + - name: image-url + value: $(params.output-image) + - name: build-task-status + value: $(tasks.build-container.status) + taskRef: + params: + - name: name + value: summary + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-summary:0.1@sha256:f65a69aaf71cbab382eff685eee522ad35068a4d91d233e76cef7d42ff15a686 + - name: kind + value: task + resolver: bundles + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Force rebuild image + name: rebuild + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "true" + description: Skip optional checks, set false if you want to run optional checks + name: skip-optional + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched by Cachi2 + name: prefetch-input + type: string + - default: "false" + description: Java build + name: java + type: string + - default: "" + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + type: string + - default: "false" + description: Build a source image. + name: build-source-image + type: string + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-container.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-container.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + - description: "" + name: JAVA_COMMUNITY_DEPENDENCIES + value: $(tasks.build-container.results.JAVA_COMMUNITY_DEPENDENCIES) + tasks: + - name: init + params: + - name: image-url + value: $(params.output-image) + - name: rebuild + value: $(params.rebuild) + - name: skip-checks + value: $(params.skip-checks) + - name: skip-optional + value: $(params.skip-optional) + - name: pipelinerun-name + value: $(context.pipelineRun.name) + - name: pipelinerun-uid + value: $(context.pipelineRun.uid) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-init:0.1@sha256:c35cc37d6f40fef0f2ed614b502b058e13fe7af29c0888bc4799fd625b6f3374 + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + - name: fetchTags + value: true + - name: depth + value: 0 + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-git-clone:0.1@sha256:b8fddc2d36313a5cde93aba2491205f4a84e6853af6c34ede681f8339b147478 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: output + workspace: workspace + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-prefetch-dependencies:0.1@sha256:9630dd7d50002fdffb4a406fb0c538703ef98bf2f4318249ac3a2c229938dbea + - name: kind + value: task + resolver: bundles + when: + - input: $(params.hermetic) + operator: in + values: + - "true" + workspaces: + - name: source + workspace: workspace + - name: fetch-vuln-feed-data + runAfter: + - init + taskSpec: + steps: + - name: fetch-vuln-feed-data + image: registry.access.redhat.com/ubi8/ubi + script: | + #!/usr/bin/env bash + mkdir -p "$(workspaces.source.path)/source" + blobs=( + nvd-definitions.zip + k8s-definitions.zip + repo2cpe.zip + genesis_manifests.json + ) + for blob in "${blobs[@]}"; do + echo "https://storage.googleapis.com/definitions.stackrox.io/scanner-data/latest/${blob} > $(workspaces.source.path)/source/blob-${blob}" + curl --fail -s --show-error --retry 4 --retry-max-time 30 --retry-connrefused \ + --output "$(workspaces.source.path)/source/blob-${blob}" \ + "https://storage.googleapis.com/definitions.stackrox.io/scanner-data/latest/${blob}" + + if [ "$?" != "0" ]; then + echo "Failed to download" + exit 1 + fi + done + workspaces: + - name: source + workspace: workspace + - name: build-container + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + runAfter: + - prefetch-dependencies + - fetch-vuln-feed-data + taskRef: + params: + - name: name + value: buildah-6gb + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-buildah-6gb:0.1 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: source + workspace: workspace + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(params.output-image) + - name: BASE_IMAGES + value: $(tasks.build-container.results.BASE_IMAGES_DIGESTS) + runAfter: + - build-container + taskRef: + params: + - name: name + value: source-build + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-source-build:0.1@sha256:f8c5dec871fb5347eb2fc61d44754bcc101897aecf953b374ab3e8315e1a9804 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - input: $(params.build-source-image) + operator: in + values: + - "true" + workspaces: + - name: workspace + workspace: workspace + - name: deprecated-base-image-check + params: + - name: BASE_IMAGES_DIGESTS + value: $(tasks.build-container.results.BASE_IMAGES_DIGESTS) + runAfter: + - build-container + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-deprecated-image-check:0.3@sha256:d87f8c50a674f57527a0c4f3df6d9093941a2ae84739b55368b3c11702ce340c + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clair-scan + params: + - name: image-digest + value: $(tasks.build-container.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-container.results.IMAGE_URL) + runAfter: + - build-container + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-clair-scan:0.1@sha256:fbe1ab58531d856fba360060d3884a0606310a966e2d01ba9305da9eb01ab916 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: sast-snyk-check + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-sast-snyk-check:0.1@sha256:eee508768b14655275fbcc2f42f9da1ab553b872dcbe113b0896aa9bcf7e1adf + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: workspace + workspace: workspace + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-container.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-container.results.IMAGE_URL) + runAfter: + - build-container + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:d72cb58db88289559676676c3db43906718028e07279f70ddb12ed8bdc8e2860 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sbom-json-check + params: + - name: IMAGE_URL + value: $(tasks.build-container.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: sbom-json-check + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-sbom-json-check:0.1@sha256:717e6e33f02dbe1a28fb743f32699e002c944680c251a50b644f27becb9208e9 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: workspace + - name: git-auth + optional: true + taskRunTemplate: {} + workspaces: + - name: workspace + volumeClaimTemplate: + metadata: + creationTimestamp: null + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + status: {} + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/.tekton/scanner-push.yaml b/.tekton/scanner-push.yaml new file mode 100644 index 000000000..dc021a2b1 --- /dev/null +++ b/.tekton/scanner-push.yaml @@ -0,0 +1,432 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/stackrox/scanner?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch == "master" + creationTimestamp: null + labels: + appstudio.openshift.io/application: acs + appstudio.openshift.io/component: scanner + pipelines.appstudio.openshift.io/type: build + name: scanner-on-push + namespace: rh-acs-tenant +spec: + params: + - name: dockerfile + value: image/scanner/rhtap/Dockerfile + - name: git-url + value: '{{repo_url}}' + - name: output-image + value: quay.io/redhat-user-workloads/rh-acs-tenant/acs/scanner:{{revision}} + - name: path-context + value: . + - name: revision + value: '{{revision}}' + pipelineSpec: + finally: + - name: show-sbom + params: + - name: IMAGE_URL + value: $(tasks.build-container.results.IMAGE_URL) + taskRef: + params: + - name: name + value: show-sbom + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-show-sbom:0.1@sha256:202d3c3385120ea847d8f0a82bd8d9d5e873d67f981d6f8a51fb1706caaf6bef + - name: kind + value: task + resolver: bundles + - name: show-summary + params: + - name: pipelinerun-name + value: $(context.pipelineRun.name) + - name: git-url + value: $(tasks.clone-repository.results.url)?rev=$(tasks.clone-repository.results.commit) + - name: image-url + value: $(params.output-image) + - name: build-task-status + value: $(tasks.build-container.status) + taskRef: + params: + - name: name + value: summary + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-summary:0.1@sha256:f65a69aaf71cbab382eff685eee522ad35068a4d91d233e76cef7d42ff15a686 + - name: kind + value: task + resolver: bundles + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Force rebuild image + name: rebuild + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "true" + description: Skip optional checks, set false if you want to run optional checks + name: skip-optional + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched by Cachi2 + name: prefetch-input + type: string + - default: "false" + description: Java build + name: java + type: string + - default: "" + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + type: string + - default: "false" + description: Build a source image. + name: build-source-image + type: string + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-container.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-container.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + - description: "" + name: JAVA_COMMUNITY_DEPENDENCIES + value: $(tasks.build-container.results.JAVA_COMMUNITY_DEPENDENCIES) + tasks: + - name: init + params: + - name: image-url + value: $(params.output-image) + - name: rebuild + value: $(params.rebuild) + - name: skip-checks + value: $(params.skip-checks) + - name: skip-optional + value: $(params.skip-optional) + - name: pipelinerun-name + value: $(context.pipelineRun.name) + - name: pipelinerun-uid + value: $(context.pipelineRun.uid) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-init:0.1@sha256:c35cc37d6f40fef0f2ed614b502b058e13fe7af29c0888bc4799fd625b6f3374 + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + - name: fetchTags + value: true + - name: depth + value: 0 + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-git-clone:0.1@sha256:b8fddc2d36313a5cde93aba2491205f4a84e6853af6c34ede681f8339b147478 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: output + workspace: workspace + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-prefetch-dependencies:0.1@sha256:9630dd7d50002fdffb4a406fb0c538703ef98bf2f4318249ac3a2c229938dbea + - name: kind + value: task + resolver: bundles + when: + - input: $(params.hermetic) + operator: in + values: + - "true" + workspaces: + - name: source + workspace: workspace + - name: fetch-vuln-feed-data + runAfter: + - init + taskSpec: + steps: + - name: fetch-vuln-feed-data + image: registry.access.redhat.com/ubi8/ubi + script: | + #!/usr/bin/env bash + mkdir -p "$(workspaces.source.path)/source" + blobs=( + nvd-definitions.zip + k8s-definitions.zip + repo2cpe.zip + genesis_manifests.json + ) + for blob in "${blobs[@]}"; do + echo "https://storage.googleapis.com/definitions.stackrox.io/scanner-data/latest/${blob} > $(workspaces.source.path)/source/blob-${blob}" + curl --fail -s --show-error --retry 4 --retry-max-time 30 --retry-connrefused \ + --output "$(workspaces.source.path)/source/blob-${blob}" \ + "https://storage.googleapis.com/definitions.stackrox.io/scanner-data/latest/${blob}" + + if [ "$?" != "0" ]; then + echo "Failed to download" + exit 1 + fi + done + workspaces: + - name: source + workspace: workspace + - name: build-container + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + runAfter: + - prefetch-dependencies + - fetch-vuln-feed-data + taskRef: + params: + - name: name + value: buildah-6gb + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-buildah-6gb:0.1 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: source + workspace: workspace + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(params.output-image) + - name: BASE_IMAGES + value: $(tasks.build-container.results.BASE_IMAGES_DIGESTS) + runAfter: + - build-container + taskRef: + params: + - name: name + value: source-build + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-source-build:0.1@sha256:f8c5dec871fb5347eb2fc61d44754bcc101897aecf953b374ab3e8315e1a9804 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - input: $(params.build-source-image) + operator: in + values: + - "true" + workspaces: + - name: workspace + workspace: workspace + - name: deprecated-base-image-check + params: + - name: BASE_IMAGES_DIGESTS + value: $(tasks.build-container.results.BASE_IMAGES_DIGESTS) + runAfter: + - build-container + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-deprecated-image-check:0.3@sha256:d87f8c50a674f57527a0c4f3df6d9093941a2ae84739b55368b3c11702ce340c + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clair-scan + params: + - name: image-digest + value: $(tasks.build-container.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-container.results.IMAGE_URL) + runAfter: + - build-container + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-clair-scan:0.1@sha256:fbe1ab58531d856fba360060d3884a0606310a966e2d01ba9305da9eb01ab916 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: sast-snyk-check + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-sast-snyk-check:0.1@sha256:eee508768b14655275fbcc2f42f9da1ab553b872dcbe113b0896aa9bcf7e1adf + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: workspace + workspace: workspace + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-container.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-container.results.IMAGE_URL) + runAfter: + - build-container + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:d72cb58db88289559676676c3db43906718028e07279f70ddb12ed8bdc8e2860 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sbom-json-check + params: + - name: IMAGE_URL + value: $(tasks.build-container.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: sbom-json-check + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-sbom-json-check:0.1@sha256:717e6e33f02dbe1a28fb743f32699e002c944680c251a50b644f27becb9208e9 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: workspace + - name: git-auth + optional: true + taskRunTemplate: {} + workspaces: + - name: workspace + volumeClaimTemplate: + metadata: + creationTimestamp: null + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + status: {} + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/image/scanner/rhtap/Dockerfile b/image/scanner/rhtap/Dockerfile new file mode 100644 index 000000000..65612b5d5 --- /dev/null +++ b/image/scanner/rhtap/Dockerfile @@ -0,0 +1,62 @@ +ARG BASE_REGISTRY=registry.access.redhat.com +ARG BASE_IMAGE=ubi8-minimal +ARG BASE_TAG=latest + +FROM brew.registry.redhat.io/rh-osbs/openshift-golang-builder:rhel_8_1.20 as builder + +ENV CGO_ENABLED=1 +ENV GOFLAGS="" +ENV CI=1 + +COPY . /src +WORKDIR /src + +RUN unzip -j blob-repo2cpe.zip -d image/scanner/dump/repo2cpe && \ + unzip -j blob-k8s-definitions.zip -d image/scanner/dump/k8s_definitions && \ + unzip -j blob-nvd-definitions.zip -d image/scanner/dump/nvd_definitions + +RUN echo "version: $(git describe --tags --abbrev=10 --dirty --long)"; git status; go build -trimpath -ldflags="-X github.com/stackrox/scanner/pkg/version.Version=$(git describe --tags --abbrev=10 --dirty --long)" -o image/scanner/bin/scanner ./cmd/clair + +# Replace genesis manifests file in the source code with the one generated at +# the point when the dump was taken. This is to avoid discrepancy between other +# files of the dump and the manifest. +COPY ./blob-genesis_manifests.json image/scanner/dump/genesis_manifests.json + +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} + +LABEL name="scanner" \ + vendor="StackRox" \ + maintainer="support@stackrox.com" \ + summary="Image scanner for the StackRox Kubernetes Security Platform" \ + description="This image supports image scanning in the StackRox Kubernetes Security Platform." + +ENV NVD_DEFINITIONS_DIR="/nvd_definitions" +ENV K8S_DEFINITIONS_DIR="/k8s_definitions" +ENV REPO_TO_CPE_DIR="/repo2cpe" + +COPY --from=builder /src/image/scanner/scripts / +COPY --from=builder /src/image/scanner/bin/scanner ./ +COPY --chown=65534:65534 --from=builder "/src/image/scanner/dump${NVD_DEFINITIONS_DIR}/" ".${NVD_DEFINITIONS_DIR}/" +COPY --chown=65534:65534 --from=builder "/src/image/scanner/dump${K8S_DEFINITIONS_DIR}/" ".${K8S_DEFINITIONS_DIR}/" +COPY --chown=65534:65534 --from=builder "/src/image/scanner/dump${REPO_TO_CPE_DIR}/" ".${REPO_TO_CPE_DIR}/" +COPY --chown=65534:65534 --from=builder /src/image/scanner/dump/genesis_manifests.json ./ + +RUN microdnf upgrade --nobest && \ + microdnf install xz && \ + microdnf clean all && \ + # (Optional) Remove line below to keep package management utilities + # We don't uninstall rpm because scanner uses it to get packages installed in scanned images. + rpm -e --nodeps $(rpm -qa curl '*dnf*' '*libsolv*' '*hawkey*' 'yum*') && \ + rm -rf /var/cache/dnf /var/cache/yum && \ + chown -R 65534:65534 /tmp && \ + # The contents of paths mounted as emptyDir volumes in Kubernetes are saved + # by the script `save-dir-contents` during the image build. The directory + # contents are then restored by the script `restore-all-dir-contents` + # during the container start. + chown -R 65534:65534 /etc/pki /etc/ssl && /save-dir-contents /etc/pki/ca-trust /etc/ssl && \ + chmod +rx /scanner + +# This is equivalent to nobody:nobody. +USER 65534:65534 + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/scripts/ci/jobs/check-generated.sh b/scripts/ci/jobs/check-generated.sh new file mode 100755 index 000000000..e99d35945 --- /dev/null +++ b/scripts/ci/jobs/check-generated.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../../.. && pwd)" +# shellcheck source=../../../scripts/ci/lib.sh +source "$ROOT/scripts/ci/lib.sh" + +set -euo pipefail + +FAIL_FLAG="/tmp/fail" + +info 'Check .containerignore file is in sync with .dockerignore (If this fails, follow instructions in .containerignore to update it.)' +function check-containerignore-is-in-sync() { + diff \ + --unified \ + --ignore-blank-lines \ + <(grep -v -e '^#' .containerignore) \ + <(grep -vF -e '/.git/' -e '/image/' -e '/bin/' .dockerignore) \ + > diff.txt +} +check-containerignore-is-in-sync || { + info ".containerignore file is not in sync with .dockerignore" + info "$(cat diff.txt)" + git reset --hard HEAD + echo check-containerignore-is-in-sync >> "$FAIL_FLAG" +} + +if [[ -e "$FAIL_FLAG" ]]; then + echo "ERROR: Some generated file checks failed:" + cat "$FAIL_FLAG" + exit 1 +fi From 8aa397dd68a4da119880eca216ad56409a84ffca Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Thu, 11 Jan 2024 11:25:46 -0600 Subject: [PATCH 02/39] Increase cpu request for clamav task --- .tekton/scanner-pull-request.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 2a0d85fe1..4e2c38880 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -417,6 +417,11 @@ spec: - name: git-auth optional: true taskRunTemplate: {} + taskRunSpecs: + - pipelineTaskName: clamav-scan + computeResources: + requests: + cpu: 1 workspaces: - name: workspace volumeClaimTemplate: From 1130867f9541e847857e085fd8e1742ae236f20e Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Thu, 11 Jan 2024 12:12:23 -0600 Subject: [PATCH 03/39] Move cpu request spec --- .tekton/scanner-pull-request.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 4e2c38880..8e0f47f4d 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -369,6 +369,9 @@ spec: - name: workspace workspace: workspace - name: clamav-scan + computeResources: + requests: + cpu: 1 params: - name: image-digest value: $(tasks.build-container.results.IMAGE_DIGEST) @@ -417,11 +420,6 @@ spec: - name: git-auth optional: true taskRunTemplate: {} - taskRunSpecs: - - pipelineTaskName: clamav-scan - computeResources: - requests: - cpu: 1 workspaces: - name: workspace volumeClaimTemplate: From 7807325f823d36fed354ec1800e14f63785edfc0 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Thu, 11 Jan 2024 16:33:13 -0600 Subject: [PATCH 04/39] Add debug to vuln feed download issue --- .tekton/scanner-pull-request.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 8e0f47f4d..4a1f60661 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -237,6 +237,8 @@ spec: echo "Failed to download" exit 1 fi + + ls -lh $(workspaces.source.path)/source done workspaces: - name: source From 02f24a75cee4fd1aae508db4690917c4c943488a Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Fri, 12 Jan 2024 08:19:16 -0600 Subject: [PATCH 05/39] Ensure fetch-vuln-feed-data runs after clone-repository This will hopefully fix an intermittent issue where the vuln feed zip files disappear from the workspace before they make it to the build step. --- .tekton/scanner-pull-request.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 4a1f60661..b4cc924e2 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -214,6 +214,7 @@ spec: - name: fetch-vuln-feed-data runAfter: - init + - clone-repository taskSpec: steps: - name: fetch-vuln-feed-data From 8886cdc10cb1ecd67348087c06310bcdfb364fff Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Mon, 15 Jan 2024 08:52:27 -0600 Subject: [PATCH 06/39] increase timeout to 2h --- .tekton/scanner-pull-request.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index b4cc924e2..eb9b01814 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -423,6 +423,10 @@ spec: - name: git-auth optional: true taskRunTemplate: {} + timeouts: + pipeline: "2h0m0s" + tasks: "2h0m0s" + finally: "2h0m0s" workspaces: - name: workspace volumeClaimTemplate: From 171e05950e5bc22dbd4beca1ec252689bb4ece61 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Mon, 15 Jan 2024 11:20:43 -0600 Subject: [PATCH 07/39] Update timeouts to satisfy validation --- .tekton/scanner-pull-request.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index eb9b01814..7a6a60021 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -424,9 +424,9 @@ spec: optional: true taskRunTemplate: {} timeouts: - pipeline: "2h0m0s" + pipeline: "3h0m0s" tasks: "2h0m0s" - finally: "2h0m0s" + finally: "1h0m0s" workspaces: - name: workspace volumeClaimTemplate: From c25954aafa82917d445190cf34044f046966b713 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Tue, 16 Jan 2024 09:43:06 -0600 Subject: [PATCH 08/39] Bump clamav task timeout to 2h --- .tekton/scanner-pull-request.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 7a6a60021..b2d9afbf3 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -382,6 +382,7 @@ spec: value: $(tasks.build-container.results.IMAGE_URL) runAfter: - build-container + timeout: 2h taskRef: params: - name: name From ea0aee02cb6a2baa4f6b1057532f0eadc6fb2c5e Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Tue, 16 Jan 2024 12:27:10 -0600 Subject: [PATCH 09/39] Bump timeouts again --- .tekton/scanner-pull-request.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index b2d9afbf3..92c8da3ab 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -382,7 +382,7 @@ spec: value: $(tasks.build-container.results.IMAGE_URL) runAfter: - build-container - timeout: 2h + timeout: 3h taskRef: params: - name: name @@ -425,8 +425,8 @@ spec: optional: true taskRunTemplate: {} timeouts: - pipeline: "3h0m0s" - tasks: "2h0m0s" + pipeline: "4h0m0s" + tasks: "3h0m0s" finally: "1h0m0s" workspaces: - name: workspace From 39cf8cafb8bc1cf86ebd1dcef8f1e92041d32754 Mon Sep 17 00:00:00 2001 From: "red-hat-trusted-app-pipeline[bot]" <126015336+red-hat-trusted-app-pipeline[bot]@users.noreply.github.com> Date: Thu, 25 Jan 2024 11:51:45 -0600 Subject: [PATCH 10/39] chore(deps): update rhtap references (#1370) Co-authored-by: red-hat-trusted-app-pipeline <123456+red-hat-trusted-app-pipeline[bot]@users.noreply.github.com> --- .tekton/scanner-pull-request.yaml | 16 ++++++++-------- .tekton/scanner-push.yaml | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 92c8da3ab..82388eb3e 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -40,7 +40,7 @@ spec: - name: name value: show-sbom - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-show-sbom:0.1@sha256:202d3c3385120ea847d8f0a82bd8d9d5e873d67f981d6f8a51fb1706caaf6bef + value: quay.io/redhat-appstudio-tekton-catalog/task-show-sbom:0.1@sha256:82737c8d365c620295fa526d21a481d4614f657800175ddc0ccd7846c54207f8 - name: kind value: task resolver: bundles @@ -153,7 +153,7 @@ spec: - name: name value: init - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-init:0.1@sha256:c35cc37d6f40fef0f2ed614b502b058e13fe7af29c0888bc4799fd625b6f3374 + value: quay.io/redhat-appstudio-tekton-catalog/task-init:0.2@sha256:3d8f01fa59596a998d30dc700fcf7377f09d60008337290eebaeaf604512ce2b - name: kind value: task resolver: bundles @@ -199,7 +199,7 @@ spec: - name: name value: prefetch-dependencies - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-prefetch-dependencies:0.1@sha256:9630dd7d50002fdffb4a406fb0c538703ef98bf2f4318249ac3a2c229938dbea + value: quay.io/redhat-appstudio-tekton-catalog/task-prefetch-dependencies:0.1@sha256:0b7bec23b6c08f37138a86e569835842763b3aa42f4455fd70ba3986350e07c7 - name: kind value: task resolver: bundles @@ -320,7 +320,7 @@ spec: - name: name value: deprecated-image-check - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-deprecated-image-check:0.3@sha256:d87f8c50a674f57527a0c4f3df6d9093941a2ae84739b55368b3c11702ce340c + value: quay.io/redhat-appstudio-tekton-catalog/task-deprecated-image-check:0.3@sha256:a299ff57d97f3924020634625dfb9bbc66547124ca23a3396e338c645f7b4a8e - name: kind value: task resolver: bundles @@ -342,7 +342,7 @@ spec: - name: name value: clair-scan - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-clair-scan:0.1@sha256:fbe1ab58531d856fba360060d3884a0606310a966e2d01ba9305da9eb01ab916 + value: quay.io/redhat-appstudio-tekton-catalog/task-clair-scan:0.1@sha256:63b42c0fc23d05e26776a0e7c4f0ab00750096ebfe1eed9a7ba96f8b27713fbf - name: kind value: task resolver: bundles @@ -359,7 +359,7 @@ spec: - name: name value: sast-snyk-check - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-sast-snyk-check:0.1@sha256:eee508768b14655275fbcc2f42f9da1ab553b872dcbe113b0896aa9bcf7e1adf + value: quay.io/redhat-appstudio-tekton-catalog/task-sast-snyk-check:0.1@sha256:47515cb119225bba55c593876610bd890f8efcbb66bb57fb0c0881ddd47ce558 - name: kind value: task resolver: bundles @@ -388,7 +388,7 @@ spec: - name: name value: clamav-scan - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:d72cb58db88289559676676c3db43906718028e07279f70ddb12ed8bdc8e2860 + value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:353fa2cda9855217cfcec3303973b666a10f384795630cf0eb13b874c24b0f7a - name: kind value: task resolver: bundles @@ -410,7 +410,7 @@ spec: - name: name value: sbom-json-check - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-sbom-json-check:0.1@sha256:717e6e33f02dbe1a28fb743f32699e002c944680c251a50b644f27becb9208e9 + value: quay.io/redhat-appstudio-tekton-catalog/task-sbom-json-check:0.1@sha256:bf49861b3bbee2129e8d1b5966fc2a7c3f259d96a5fcef5674d05c9cb21ab540 - name: kind value: task resolver: bundles diff --git a/.tekton/scanner-push.yaml b/.tekton/scanner-push.yaml index dc021a2b1..3f07e5c51 100644 --- a/.tekton/scanner-push.yaml +++ b/.tekton/scanner-push.yaml @@ -37,7 +37,7 @@ spec: - name: name value: show-sbom - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-show-sbom:0.1@sha256:202d3c3385120ea847d8f0a82bd8d9d5e873d67f981d6f8a51fb1706caaf6bef + value: quay.io/redhat-appstudio-tekton-catalog/task-show-sbom:0.1@sha256:82737c8d365c620295fa526d21a481d4614f657800175ddc0ccd7846c54207f8 - name: kind value: task resolver: bundles @@ -150,7 +150,7 @@ spec: - name: name value: init - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-init:0.1@sha256:c35cc37d6f40fef0f2ed614b502b058e13fe7af29c0888bc4799fd625b6f3374 + value: quay.io/redhat-appstudio-tekton-catalog/task-init:0.2@sha256:3d8f01fa59596a998d30dc700fcf7377f09d60008337290eebaeaf604512ce2b - name: kind value: task resolver: bundles @@ -196,7 +196,7 @@ spec: - name: name value: prefetch-dependencies - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-prefetch-dependencies:0.1@sha256:9630dd7d50002fdffb4a406fb0c538703ef98bf2f4318249ac3a2c229938dbea + value: quay.io/redhat-appstudio-tekton-catalog/task-prefetch-dependencies:0.1@sha256:0b7bec23b6c08f37138a86e569835842763b3aa42f4455fd70ba3986350e07c7 - name: kind value: task resolver: bundles @@ -314,7 +314,7 @@ spec: - name: name value: deprecated-image-check - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-deprecated-image-check:0.3@sha256:d87f8c50a674f57527a0c4f3df6d9093941a2ae84739b55368b3c11702ce340c + value: quay.io/redhat-appstudio-tekton-catalog/task-deprecated-image-check:0.3@sha256:a299ff57d97f3924020634625dfb9bbc66547124ca23a3396e338c645f7b4a8e - name: kind value: task resolver: bundles @@ -336,7 +336,7 @@ spec: - name: name value: clair-scan - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-clair-scan:0.1@sha256:fbe1ab58531d856fba360060d3884a0606310a966e2d01ba9305da9eb01ab916 + value: quay.io/redhat-appstudio-tekton-catalog/task-clair-scan:0.1@sha256:63b42c0fc23d05e26776a0e7c4f0ab00750096ebfe1eed9a7ba96f8b27713fbf - name: kind value: task resolver: bundles @@ -353,7 +353,7 @@ spec: - name: name value: sast-snyk-check - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-sast-snyk-check:0.1@sha256:eee508768b14655275fbcc2f42f9da1ab553b872dcbe113b0896aa9bcf7e1adf + value: quay.io/redhat-appstudio-tekton-catalog/task-sast-snyk-check:0.1@sha256:47515cb119225bba55c593876610bd890f8efcbb66bb57fb0c0881ddd47ce558 - name: kind value: task resolver: bundles @@ -378,7 +378,7 @@ spec: - name: name value: clamav-scan - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:d72cb58db88289559676676c3db43906718028e07279f70ddb12ed8bdc8e2860 + value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:353fa2cda9855217cfcec3303973b666a10f384795630cf0eb13b874c24b0f7a - name: kind value: task resolver: bundles @@ -400,7 +400,7 @@ spec: - name: name value: sbom-json-check - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-sbom-json-check:0.1@sha256:717e6e33f02dbe1a28fb743f32699e002c944680c251a50b644f27becb9208e9 + value: quay.io/redhat-appstudio-tekton-catalog/task-sbom-json-check:0.1@sha256:bf49861b3bbee2129e8d1b5966fc2a7c3f259d96a5fcef5674d05c9cb21ab540 - name: kind value: task resolver: bundles From a59ab95e13638a6ab74308a3f3145f15450ffa98 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Thu, 25 Jan 2024 12:07:07 -0600 Subject: [PATCH 11/39] comment out clamav scan for now there is a problem with the scan where it takes over two hours to complete on an openshift cluster, but runs in ~1m on my local box. --- .tekton/scanner-pull-request.yaml | 52 +++++++++++++++---------------- .tekton/scanner-push.yaml | 44 +++++++++++++------------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 82388eb3e..b1829a85c 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -371,32 +371,32 @@ spec: workspaces: - name: workspace workspace: workspace - - name: clamav-scan - computeResources: - requests: - cpu: 1 - params: - - name: image-digest - value: $(tasks.build-container.results.IMAGE_DIGEST) - - name: image-url - value: $(tasks.build-container.results.IMAGE_URL) - runAfter: - - build-container - timeout: 3h - taskRef: - params: - - name: name - value: clamav-scan - - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:353fa2cda9855217cfcec3303973b666a10f384795630cf0eb13b874c24b0f7a - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" + # - name: clamav-scan + # computeResources: + # requests: + # cpu: 1 + # params: + # - name: image-digest + # value: $(tasks.build-container.results.IMAGE_DIGEST) + # - name: image-url + # value: $(tasks.build-container.results.IMAGE_URL) + # runAfter: + # - build-container + # timeout: 3h + # taskRef: + # params: + # - name: name + # value: clamav-scan + # - name: bundle + # value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:353fa2cda9855217cfcec3303973b666a10f384795630cf0eb13b874c24b0f7a + # - name: kind + # value: task + # resolver: bundles + # when: + # - input: $(params.skip-checks) + # operator: in + # values: + # - "false" - name: sbom-json-check params: - name: IMAGE_URL diff --git a/.tekton/scanner-push.yaml b/.tekton/scanner-push.yaml index 3f07e5c51..ed5a3d1e9 100644 --- a/.tekton/scanner-push.yaml +++ b/.tekton/scanner-push.yaml @@ -365,28 +365,28 @@ spec: workspaces: - name: workspace workspace: workspace - - name: clamav-scan - params: - - name: image-digest - value: $(tasks.build-container.results.IMAGE_DIGEST) - - name: image-url - value: $(tasks.build-container.results.IMAGE_URL) - runAfter: - - build-container - taskRef: - params: - - name: name - value: clamav-scan - - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:353fa2cda9855217cfcec3303973b666a10f384795630cf0eb13b874c24b0f7a - - name: kind - value: task - resolver: bundles - when: - - input: $(params.skip-checks) - operator: in - values: - - "false" + # - name: clamav-scan + # params: + # - name: image-digest + # value: $(tasks.build-container.results.IMAGE_DIGEST) + # - name: image-url + # value: $(tasks.build-container.results.IMAGE_URL) + # runAfter: + # - build-container + # taskRef: + # params: + # - name: name + # value: clamav-scan + # - name: bundle + # value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:353fa2cda9855217cfcec3303973b666a10f384795630cf0eb13b874c24b0f7a + # - name: kind + # value: task + # resolver: bundles + # when: + # - input: $(params.skip-checks) + # operator: in + # values: + # - "false" - name: sbom-json-check params: - name: IMAGE_URL From 734e8560ed2c102aa9769e62570a1d6343cf9d3f Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Fri, 26 Jan 2024 12:14:19 -0600 Subject: [PATCH 12/39] Rename rhtap folder to konflux --- .tekton/scanner-pull-request.yaml | 2 +- .tekton/scanner-push.yaml | 2 +- image/scanner/{rhtap => konflux}/Dockerfile | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename image/scanner/{rhtap => konflux}/Dockerfile (100%) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index b1829a85c..9f8e82dee 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -18,7 +18,7 @@ metadata: spec: params: - name: dockerfile - value: image/scanner/rhtap/Dockerfile + value: image/scanner/konflux/Dockerfile - name: git-url value: '{{repo_url}}' - name: image-expires-after diff --git a/.tekton/scanner-push.yaml b/.tekton/scanner-push.yaml index ed5a3d1e9..8a49c551a 100644 --- a/.tekton/scanner-push.yaml +++ b/.tekton/scanner-push.yaml @@ -17,7 +17,7 @@ metadata: spec: params: - name: dockerfile - value: image/scanner/rhtap/Dockerfile + value: image/scanner/konflux/Dockerfile - name: git-url value: '{{repo_url}}' - name: output-image diff --git a/image/scanner/rhtap/Dockerfile b/image/scanner/konflux/Dockerfile similarity index 100% rename from image/scanner/rhtap/Dockerfile rename to image/scanner/konflux/Dockerfile From adfa9a71f060dabcbe7fdb149e30c8cb142deb7d Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Fri, 26 Jan 2024 14:44:21 -0600 Subject: [PATCH 13/39] Trigger build From 1ec867a9bd399d0254cbb313e708afcd56a738da Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Mon, 29 Jan 2024 16:40:49 -0600 Subject: [PATCH 14/39] Remove CI checks to ensure containerignore/dockerignore are in sync --- .containerignore | 16 ++++++------ .dockerignore | 3 +++ .github/actions/job-preamble/action.yaml | 17 ------------- .github/workflows/style.yaml | 28 --------------------- scripts/ci/jobs/check-generated.sh | 31 ------------------------ 5 files changed, 11 insertions(+), 84 deletions(-) delete mode 100644 .github/actions/job-preamble/action.yaml delete mode 100644 .github/workflows/style.yaml delete mode 100755 scripts/ci/jobs/check-generated.sh diff --git a/.containerignore b/.containerignore index d1cc8cef5..b6150adb9 100644 --- a/.containerignore +++ b/.containerignore @@ -1,9 +1,9 @@ -# This file is derived of the .dockerignore -# It is required to pass `.git` to the builder containers on RHTAP so we can bake build information into the binary. -# It is required to pass the `.gitignore` files in `image/bin/` and `bin` to avoid a dirty version. -# RHTAP uses Buildah which first looks up for `.containerignore`, then, if absent, `.dockerignore`. -# Having this file enables us to have different exclusion list for RHTAP builds than in GHA/OSCI. -# To regenerate, replace the below contents with the output of the following command (please keep these comments). -# grep -vF -e '/.git/' -e '/bin/' -e '/image/bin/' .dockerignore +# Konflux uses buildah which first looks up `.containerignore` then, if absent, `.dockerignore`. +# Having this file enables us to have different exclusion list for Konflux builds than in GHA/OSCI. -# currently empty! +# This file is empty because all the items that are ignored in GHA/OSCI need to be present for Konflux builds: +# * `.git` is needed so we can bake build information into the binary. +# * `.gitignore`, `image/bin/` and `bin` are required to avoid a dirty version. + +# If content ever needs to be added here, be sure to borrow the check-generated.sh template from the +# stackrox/stackrox repo to ensure .dockerfile and .containerignore stay in sync. diff --git a/.dockerignore b/.dockerignore index dc8214ccf..311cf2266 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,6 @@ +# NOTE: if adding items here, please implement the check-generated.sh script +# from stackrox/stackrox to keep this file and .containeringore in sync + /.git/ /bin/ /image/bin/ diff --git a/.github/actions/job-preamble/action.yaml b/.github/actions/job-preamble/action.yaml deleted file mode 100644 index 42f59f243..000000000 --- a/.github/actions/job-preamble/action.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: Job Preamble -description: Common steps for most jobs -runs: - using: composite - steps: - - name: Recover docker image cache space - run: | - df --si / - docker system prune --force --all - df --si / - shell: bash - - - name: Ignore dubious repository ownership - run: | - # Prevent fatal error "detected dubious ownership in repository" from recent git. - git config --global --add safe.directory "$(pwd)" - shell: bash diff --git a/.github/workflows/style.yaml b/.github/workflows/style.yaml deleted file mode 100644 index b12cd9af6..000000000 --- a/.github/workflows/style.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: Style -on: - push: - tags: - - '*' - branches: - - master - pull_request: - types: - - opened - - reopened - - synchronize -jobs: - check-generated-files: - runs-on: ubuntu-latest - container: - image: quay.io/stackrox-io/apollo-ci:scanner-test-0.3.61 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - - uses: ./.github/actions/job-preamble - - - name: Check Generated - run: scripts/ci/jobs/check-generated.sh diff --git a/scripts/ci/jobs/check-generated.sh b/scripts/ci/jobs/check-generated.sh deleted file mode 100755 index e99d35945..000000000 --- a/scripts/ci/jobs/check-generated.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../../.. && pwd)" -# shellcheck source=../../../scripts/ci/lib.sh -source "$ROOT/scripts/ci/lib.sh" - -set -euo pipefail - -FAIL_FLAG="/tmp/fail" - -info 'Check .containerignore file is in sync with .dockerignore (If this fails, follow instructions in .containerignore to update it.)' -function check-containerignore-is-in-sync() { - diff \ - --unified \ - --ignore-blank-lines \ - <(grep -v -e '^#' .containerignore) \ - <(grep -vF -e '/.git/' -e '/image/' -e '/bin/' .dockerignore) \ - > diff.txt -} -check-containerignore-is-in-sync || { - info ".containerignore file is not in sync with .dockerignore" - info "$(cat diff.txt)" - git reset --hard HEAD - echo check-containerignore-is-in-sync >> "$FAIL_FLAG" -} - -if [[ -e "$FAIL_FLAG" ]]; then - echo "ERROR: Some generated file checks failed:" - cat "$FAIL_FLAG" - exit 1 -fi From 8f232d6b7af18d1924415136501b84dd3316bcbb Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Mon, 29 Jan 2024 16:52:15 -0600 Subject: [PATCH 15/39] Fail builds if git is dirty --- image/scanner/konflux/Dockerfile | 6 +++- scripts/konflux/fail-build-if-git-is-dirty.sh | 34 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100755 scripts/konflux/fail-build-if-git-is-dirty.sh diff --git a/image/scanner/konflux/Dockerfile b/image/scanner/konflux/Dockerfile index 65612b5d5..fce035797 100644 --- a/image/scanner/konflux/Dockerfile +++ b/image/scanner/konflux/Dockerfile @@ -11,11 +11,15 @@ ENV CI=1 COPY . /src WORKDIR /src +RUN scripts/konflux/fail-build-if-git-is-dirty.sh + RUN unzip -j blob-repo2cpe.zip -d image/scanner/dump/repo2cpe && \ unzip -j blob-k8s-definitions.zip -d image/scanner/dump/k8s_definitions && \ unzip -j blob-nvd-definitions.zip -d image/scanner/dump/nvd_definitions -RUN echo "version: $(git describe --tags --abbrev=10 --dirty --long)"; git status; go build -trimpath -ldflags="-X github.com/stackrox/scanner/pkg/version.Version=$(git describe --tags --abbrev=10 --dirty --long)" -o image/scanner/bin/scanner ./cmd/clair +RUN echo "version: $(git describe --tags --abbrev=10 --dirty --long)" && \ + git status && \ + go build -trimpath -ldflags="-X github.com/stackrox/scanner/pkg/version.Version=$(git describe --tags --abbrev=10 --dirty --long)" -o image/scanner/bin/scanner ./cmd/clair # Replace genesis manifests file in the source code with the one generated at # the point when the dump was taken. This is to avoid discrepancy between other diff --git a/scripts/konflux/fail-build-if-git-is-dirty.sh b/scripts/konflux/fail-build-if-git-is-dirty.sh new file mode 100755 index 000000000..9bb5161c1 --- /dev/null +++ b/scripts/konflux/fail-build-if-git-is-dirty.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# This script works around the fact that RHTAP modifies Dockerfiles provided to it when prefetching dependencies is on. +# RHTAP changes should stop happening after https://issues.redhat.com/browse/STONEBLD-1847 +# Additionally, the script returns no-zero if it detects any other changes to the git repo. +# +# If this script is not called and does not fail the build, things like `make tag` will produce `-dirty` suffix +# (as in `4.3.x-63-g09e5188ab9-dirty`) which gets embedded as the version attribute in built binaries. +# +# The script MUST be executed only from within the Dockerfile (not outside of it) because binaries are built inside. + +set -euo pipefail + +# When executing in RHTAP (as opposed to the script ran directly), we undo RHTAP changes to Dockerfiles. +# I found no better way to detect RHTAP than by checking the presence of cachi2.env file. +if [[ -f /cachi2/cachi2.env ]]; then + # We can safely restore dockerfiles because the modified version of dockerfile interpreted by docker/buildah stays + # outside, and these are local copies inside of the build context. + git restore "**/konflux/Dockerfile" +fi + +# Next, make sure no other things that make it `-dirty` slipped through. If they did, fail the build. + +echo "Checking that files in git repo are not modified." +echo "If this command fails, you should see the list of modified files below." +echo "You need to find the reason and prevent it because otherwise the build results will be inconsistent." +echo "" + +if git status --porcelain | grep '.' >&2 ; then + >&2 echo "ERROR: Modified files found." + exit 2 +else + echo "No modifications to git repo detected." +fi From 05dd79bbf8a53c3cf4cb96fc800a11614611789a Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Tue, 30 Jan 2024 17:10:18 -0600 Subject: [PATCH 16/39] Filter out output from git status that doesn't make a tag dirty --- scripts/konflux/fail-build-if-git-is-dirty.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/konflux/fail-build-if-git-is-dirty.sh b/scripts/konflux/fail-build-if-git-is-dirty.sh index 9bb5161c1..423907b60 100755 --- a/scripts/konflux/fail-build-if-git-is-dirty.sh +++ b/scripts/konflux/fail-build-if-git-is-dirty.sh @@ -26,7 +26,7 @@ echo "If this command fails, you should see the list of modified files below." echo "You need to find the reason and prevent it because otherwise the build results will be inconsistent." echo "" -if git status --porcelain | grep '.' >&2 ; then +if git status --porcelain | grep -v "^??" | grep '.' >&2 ; then >&2 echo "ERROR: Modified files found." exit 2 else From 8954b5f70e2ea66a2ef3504b464588a866b7d9e0 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Tue, 30 Jan 2024 20:59:35 -0600 Subject: [PATCH 17/39] Add back the job-preamble gitub action --- .github/actions/job-preamble/action.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/actions/job-preamble/action.yaml diff --git a/.github/actions/job-preamble/action.yaml b/.github/actions/job-preamble/action.yaml new file mode 100644 index 000000000..42f59f243 --- /dev/null +++ b/.github/actions/job-preamble/action.yaml @@ -0,0 +1,17 @@ +name: Job Preamble +description: Common steps for most jobs +runs: + using: composite + steps: + - name: Recover docker image cache space + run: | + df --si / + docker system prune --force --all + df --si / + shell: bash + + - name: Ignore dubious repository ownership + run: | + # Prevent fatal error "detected dubious ownership in repository" from recent git. + git config --global --add safe.directory "$(pwd)" + shell: bash From 1371932458919648a5fd4d34091b8a5bdc3e76ae Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Tue, 30 Jan 2024 21:44:59 -0600 Subject: [PATCH 18/39] Set the correct version string in the compiled binary --- image/scanner/konflux/Dockerfile | 2 +- scripts/konflux/version.sh | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100755 scripts/konflux/version.sh diff --git a/image/scanner/konflux/Dockerfile b/image/scanner/konflux/Dockerfile index fce035797..77321884b 100644 --- a/image/scanner/konflux/Dockerfile +++ b/image/scanner/konflux/Dockerfile @@ -19,7 +19,7 @@ RUN unzip -j blob-repo2cpe.zip -d image/scanner/dump/repo2cpe && \ RUN echo "version: $(git describe --tags --abbrev=10 --dirty --long)" && \ git status && \ - go build -trimpath -ldflags="-X github.com/stackrox/scanner/pkg/version.Version=$(git describe --tags --abbrev=10 --dirty --long)" -o image/scanner/bin/scanner ./cmd/clair + go build -trimpath -ldflags="-X github.com/stackrox/scanner/pkg/version.Version=$(scripts/konflux/version.sh)" -o image/scanner/bin/scanner ./cmd/clair # Replace genesis manifests file in the source code with the one generated at # the point when the dump was taken. This is to avoid discrepancy between other diff --git a/scripts/konflux/version.sh b/scripts/konflux/version.sh new file mode 100755 index 000000000..eb1e3fcd6 --- /dev/null +++ b/scripts/konflux/version.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# This script is used by the Konflux dockerfile to get the correct version to +# compile in to the binary. If HEAD points to a git tag, use that; otherwise +# use `git describe ...`. + +if [[ -n "$(git tag --contains)" ]]; then + git tag --sort=creatordate --contains | tail -1 +else + git describe --tags --abbrev=10 --dirty --long +fi From af81ebe5e9280dd5a7aec3ff27e4d919f6ba8973 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Wed, 31 Jan 2024 12:14:19 -0600 Subject: [PATCH 19/39] Add labels to match other konflux builds --- image/scanner/konflux/Dockerfile | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/image/scanner/konflux/Dockerfile b/image/scanner/konflux/Dockerfile index 77321884b..c49c489e6 100644 --- a/image/scanner/konflux/Dockerfile +++ b/image/scanner/konflux/Dockerfile @@ -28,11 +28,21 @@ COPY ./blob-genesis_manifests.json image/scanner/dump/genesis_manifests.json FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} -LABEL name="scanner" \ - vendor="StackRox" \ - maintainer="support@stackrox.com" \ - summary="Image scanner for the StackRox Kubernetes Security Platform" \ - description="This image supports image scanning in the StackRox Kubernetes Security Platform." +LABEL \ + com.redhat.component="rhacs-scanner-container" \ + com.redhat.license_terms="https://www.redhat.com/agreements" \ + description="This image supports image scanning for RHACS" \ + io.k8s.description="This image supports image scanning for RHACS" \ + io.k8s.display-name="scanner" \ + io.openshift.tags="rhacs,scanner,stackrox" \ + maintainer="Red Hat, Inc." \ + name="rhacs-scanner-rhel8" \ + source-location="https://github.com/stackrox/scanner" \ + summary="The image scanner for RHACS" \ + url="https://catalog.redhat.com/software/container-stacks/detail/60eefc88ee05ae7c5b8f041c" \ + # We must set version label to prevent inheriting value set in the base stage. + # TODO(ROX-20236): configure injection of dynamic version value when it becomes possible. + version="0.0.1-todo" ENV NVD_DEFINITIONS_DIR="/nvd_definitions" ENV K8S_DEFINITIONS_DIR="/k8s_definitions" From 60685202bfdde41fb73d2d2fd90b06d75a005f28 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Wed, 31 Jan 2024 12:30:21 -0600 Subject: [PATCH 20/39] Try a build without chmod +rx /scanner --- image/scanner/konflux/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/image/scanner/konflux/Dockerfile b/image/scanner/konflux/Dockerfile index c49c489e6..3cdd54f0f 100644 --- a/image/scanner/konflux/Dockerfile +++ b/image/scanner/konflux/Dockerfile @@ -67,8 +67,7 @@ RUN microdnf upgrade --nobest && \ # by the script `save-dir-contents` during the image build. The directory # contents are then restored by the script `restore-all-dir-contents` # during the container start. - chown -R 65534:65534 /etc/pki /etc/ssl && /save-dir-contents /etc/pki/ca-trust /etc/ssl && \ - chmod +rx /scanner + chown -R 65534:65534 /etc/pki /etc/ssl && /save-dir-contents /etc/pki/ca-trust /etc/ssl # This is equivalent to nobody:nobody. USER 65534:65534 From d77527ab0239ce50539ccb5edc7d6d2ac3d44063 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Wed, 31 Jan 2024 12:55:32 -0600 Subject: [PATCH 21/39] Add SHELL line to konflux dockerfile; fix nit --- image/scanner/konflux/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/image/scanner/konflux/Dockerfile b/image/scanner/konflux/Dockerfile index 3cdd54f0f..7137c76f0 100644 --- a/image/scanner/konflux/Dockerfile +++ b/image/scanner/konflux/Dockerfile @@ -44,6 +44,8 @@ LABEL \ # TODO(ROX-20236): configure injection of dynamic version value when it becomes possible. version="0.0.1-todo" +SHELL ["/bin/sh", "-o", "pipefail", "-c"] + ENV NVD_DEFINITIONS_DIR="/nvd_definitions" ENV K8S_DEFINITIONS_DIR="/k8s_definitions" ENV REPO_TO_CPE_DIR="/repo2cpe" @@ -67,7 +69,8 @@ RUN microdnf upgrade --nobest && \ # by the script `save-dir-contents` during the image build. The directory # contents are then restored by the script `restore-all-dir-contents` # during the container start. - chown -R 65534:65534 /etc/pki /etc/ssl && /save-dir-contents /etc/pki/ca-trust /etc/ssl + chown -R 65534:65534 /etc/pki /etc/ssl && \ + /save-dir-contents /etc/pki/ca-trust /etc/ssl # This is equivalent to nobody:nobody. USER 65534:65534 From 67dd3cfece189c40b179bcc6ef89e467ef2bd4d3 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Wed, 31 Jan 2024 13:16:17 -0600 Subject: [PATCH 22/39] Save konflux-built images for longer --- .tekton/scanner-pull-request.yaml | 5 +++-- .tekton/scanner-push.yaml | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 9f8e82dee..87d1940cd 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -6,7 +6,8 @@ metadata: build.appstudio.redhat.com/commit_sha: '{{revision}}' build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' - pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/max-keep-runs: "500" + # TODO(ROX-21073): re-enable for all PR branches pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && source_branch.contains("rhtap") creationTimestamp: null labels: @@ -22,7 +23,7 @@ spec: - name: git-url value: '{{repo_url}}' - name: image-expires-after - value: 5d + value: '13w' - name: output-image value: quay.io/redhat-user-workloads/rh-acs-tenant/acs/scanner:on-pr-{{revision}} - name: path-context diff --git a/.tekton/scanner-push.yaml b/.tekton/scanner-push.yaml index 8a49c551a..3f63048e5 100644 --- a/.tekton/scanner-push.yaml +++ b/.tekton/scanner-push.yaml @@ -5,7 +5,7 @@ metadata: build.appstudio.openshift.io/repo: https://github.com/stackrox/scanner?rev={{revision}} build.appstudio.redhat.com/commit_sha: '{{revision}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' - pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/max-keep-runs: "500" pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch == "master" creationTimestamp: null labels: @@ -20,6 +20,9 @@ spec: value: image/scanner/konflux/Dockerfile - name: git-url value: '{{repo_url}}' + - name: image-expires-after + # TODO(ROX-20230): make release images not expire. + value: '13w' - name: output-image value: quay.io/redhat-user-workloads/rh-acs-tenant/acs/scanner:{{revision}} - name: path-context From 225640345618fc84bce0545d864fea2fcf37bbfb Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Wed, 31 Jan 2024 13:35:48 -0600 Subject: [PATCH 23/39] Add spaces between sections and reorder tekton config --- .tekton/scanner-pull-request.yaml | 74 +++++++++++++++++-------------- .tekton/scanner-push.yaml | 1 + 2 files changed, 42 insertions(+), 33 deletions(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 87d1940cd..bdcf6dd04 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -17,6 +17,7 @@ metadata: name: scanner-on-pull-request namespace: rh-acs-tenant spec: + params: - name: dockerfile value: image/scanner/konflux/Dockerfile @@ -30,7 +31,25 @@ spec: value: . - name: revision value: '{{revision}}' + + workspaces: + - name: workspace + volumeClaimTemplate: + metadata: + creationTimestamp: null + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + status: {} + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' + pipelineSpec: + finally: - name: show-sbom params: @@ -64,6 +83,7 @@ spec: - name: kind value: task resolver: bundles + params: - description: Source Repository URL name: git-url @@ -93,10 +113,6 @@ spec: description: Skip checks against built image name: skip-checks type: string - - default: "true" - description: Skip optional checks, set false if you want to run optional checks - name: skip-optional - type: string - default: "false" description: Execute the build with network isolation name: hermetic @@ -118,6 +134,7 @@ spec: description: Build a source image. name: build-source-image type: string + results: - description: "" name: IMAGE_URL @@ -134,7 +151,13 @@ spec: - description: "" name: JAVA_COMMUNITY_DEPENDENCIES value: $(tasks.build-container.results.JAVA_COMMUNITY_DEPENDENCIES) + + workspaces: + - name: workspace + - name: git-auth + tasks: + - name: init params: - name: image-url @@ -143,12 +166,6 @@ spec: value: $(params.rebuild) - name: skip-checks value: $(params.skip-checks) - - name: skip-optional - value: $(params.skip-optional) - - name: pipelinerun-name - value: $(context.pipelineRun.name) - - name: pipelinerun-uid - value: $(context.pipelineRun.uid) taskRef: params: - name: name @@ -158,16 +175,17 @@ spec: - name: kind value: task resolver: bundles + - name: clone-repository params: - name: url value: $(params.git-url) - name: revision value: $(params.revision) - - name: fetchTags - value: true - name: depth value: 0 + - name: fetchTags + value: true runAfter: - init taskRef: @@ -189,6 +207,7 @@ spec: workspace: workspace - name: basic-auth workspace: git-auth + - name: prefetch-dependencies params: - name: input @@ -245,6 +264,7 @@ spec: workspaces: - name: source workspace: workspace + - name: build-container params: - name: IMAGE @@ -281,6 +301,7 @@ spec: workspaces: - name: source workspace: workspace + - name: build-source-image params: - name: BINARY_IMAGE @@ -310,6 +331,7 @@ spec: workspaces: - name: workspace workspace: workspace + - name: deprecated-base-image-check params: - name: BASE_IMAGES_DIGESTS @@ -330,6 +352,7 @@ spec: operator: in values: - "false" + - name: clair-scan params: - name: image-digest @@ -352,6 +375,7 @@ spec: operator: in values: - "false" + - name: sast-snyk-check runAfter: - clone-repository @@ -372,6 +396,7 @@ spec: workspaces: - name: workspace workspace: workspace + # - name: clamav-scan # computeResources: # requests: @@ -398,6 +423,7 @@ spec: # operator: in # values: # - "false" + - name: sbom-json-check params: - name: IMAGE_URL @@ -420,28 +446,10 @@ spec: operator: in values: - "false" - workspaces: - - name: workspace - - name: git-auth - optional: true - taskRunTemplate: {} timeouts: pipeline: "4h0m0s" tasks: "3h0m0s" finally: "1h0m0s" - workspaces: - - name: workspace - volumeClaimTemplate: - metadata: - creationTimestamp: null - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - status: {} - - name: git-auth - secret: - secretName: '{{ git_auth_secret }}' -status: {} + +status: { } +taskRunTemplate: { } diff --git a/.tekton/scanner-push.yaml b/.tekton/scanner-push.yaml index 3f63048e5..e134d8e34 100644 --- a/.tekton/scanner-push.yaml +++ b/.tekton/scanner-push.yaml @@ -15,6 +15,7 @@ metadata: name: scanner-on-push namespace: rh-acs-tenant spec: + params: - name: dockerfile value: image/scanner/konflux/Dockerfile From d5c7039820c127e867e31865d8f3dae6e47af8e3 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Wed, 31 Jan 2024 13:36:11 -0600 Subject: [PATCH 24/39] Set rebuild=true --- .tekton/scanner-pull-request.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index bdcf6dd04..99c9d9916 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -31,6 +31,8 @@ spec: value: . - name: revision value: '{{revision}}' + - name: rebuild + value: 'true' workspaces: - name: workspace From 8580b714da12a94593fc5e2bf782ec79cc670904 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Wed, 31 Jan 2024 15:43:23 -0600 Subject: [PATCH 25/39] Enhance tekton task script logging --- .tekton/scanner-pull-request.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 99c9d9916..5e3b27ae0 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -251,7 +251,7 @@ spec: genesis_manifests.json ) for blob in "${blobs[@]}"; do - echo "https://storage.googleapis.com/definitions.stackrox.io/scanner-data/latest/${blob} > $(workspaces.source.path)/source/blob-${blob}" + echo "Downloading https://storage.googleapis.com/definitions.stackrox.io/scanner-data/latest/${blob} > $(workspaces.source.path)/source/blob-${blob}" curl --fail -s --show-error --retry 4 --retry-max-time 30 --retry-connrefused \ --output "$(workspaces.source.path)/source/blob-${blob}" \ "https://storage.googleapis.com/definitions.stackrox.io/scanner-data/latest/${blob}" From ffe1f37a432a77b4de94ab0ae063d42d0adcd2a3 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Wed, 31 Jan 2024 15:43:52 -0600 Subject: [PATCH 26/39] Try using standard buildah image (w/o higher mem limit) --- .tekton/scanner-pull-request.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 5e3b27ae0..b3111fcca 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -289,9 +289,9 @@ spec: taskRef: params: - name: name - value: buildah-6gb + value: buildah - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-buildah-6gb:0.1 + value: quay.io/redhat-appstudio-tekton-catalog/task-buildah:0.1@sha256:e45cf454d90b81d0e23107856c009083c279cbe07f9bc48538b1977d69713f35 - name: kind value: task resolver: bundles From c6d301686b30640b325ea8fbbffdd79f9f0fd8bb Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Wed, 31 Jan 2024 15:59:52 -0600 Subject: [PATCH 27/39] Switch vuln dump task image to ubi8-minimal --- .tekton/scanner-pull-request.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index b3111fcca..598b11e6c 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -240,7 +240,7 @@ spec: taskSpec: steps: - name: fetch-vuln-feed-data - image: registry.access.redhat.com/ubi8/ubi + image: registry.access.redhat.com/ubi8-minimal:latest script: | #!/usr/bin/env bash mkdir -p "$(workspaces.source.path)/source" From 6e1c57994b1246f952cb58cda599d14fb178e8de Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Wed, 31 Jan 2024 16:37:11 -0600 Subject: [PATCH 28/39] Externalize fetch-vuln-feed-data script; couple other small fixes --- .tekton/scanner-pull-request.yaml | 30 +++---------------------- scripts/konflux/fetch-vuln-feed-data.sh | 29 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 27 deletions(-) create mode 100755 scripts/konflux/fetch-vuln-feed-data.sh diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 598b11e6c..f03a8e73c 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -185,9 +185,9 @@ spec: - name: revision value: $(params.revision) - name: depth - value: 0 + value: "0" - name: fetchTags - value: true + value: "true" runAfter: - init taskRef: @@ -225,11 +225,6 @@ spec: - name: kind value: task resolver: bundles - when: - - input: $(params.hermetic) - operator: in - values: - - "true" workspaces: - name: source workspace: workspace @@ -243,26 +238,7 @@ spec: image: registry.access.redhat.com/ubi8-minimal:latest script: | #!/usr/bin/env bash - mkdir -p "$(workspaces.source.path)/source" - blobs=( - nvd-definitions.zip - k8s-definitions.zip - repo2cpe.zip - genesis_manifests.json - ) - for blob in "${blobs[@]}"; do - echo "Downloading https://storage.googleapis.com/definitions.stackrox.io/scanner-data/latest/${blob} > $(workspaces.source.path)/source/blob-${blob}" - curl --fail -s --show-error --retry 4 --retry-max-time 30 --retry-connrefused \ - --output "$(workspaces.source.path)/source/blob-${blob}" \ - "https://storage.googleapis.com/definitions.stackrox.io/scanner-data/latest/${blob}" - - if [ "$?" != "0" ]; then - echo "Failed to download" - exit 1 - fi - - ls -lh $(workspaces.source.path)/source - done + $(workspaces.source.path)/source/scanner/scripts/konflux/fetch-vuln-feed-data.sh "$(workspaces.source.path)/source" workspaces: - name: source workspace: workspace diff --git a/scripts/konflux/fetch-vuln-feed-data.sh b/scripts/konflux/fetch-vuln-feed-data.sh new file mode 100755 index 000000000..24b37881b --- /dev/null +++ b/scripts/konflux/fetch-vuln-feed-data.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ "$#" < 1 ]]; then + echo "Please pass target directory." + exit 1 +fi + +TARGET_DIR=$1 + +if [[ ! -d "$TARGET_DIR" ]]; then + echo "$TARGET_DIR is not a valid directory" + exit 1 +fi + +blobs=( + nvd-definitions.zip + k8s-definitions.zip + repo2cpe.zip + genesis_manifests.json +) + +for blob in "${blobs[@]}"; do + echo "Downloading https://storage.googleapis.com/definitions.stackrox.io/scanner-data/latest/${blob} > $TARGET_DIR/blob-${blob}" + curl --fail -s --show-error --retry 4 --retry-max-time 30 --retry-connrefused \ + --output "$TARGET_DIR/blob-${blob}" \ + "https://storage.googleapis.com/definitions.stackrox.io/scanner-data/latest/${blob}" +done From 439704c852fe80a0c18553ffb9288070c51f9f1a Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Wed, 31 Jan 2024 16:54:22 -0600 Subject: [PATCH 29/39] Fix path to script --- .tekton/scanner-pull-request.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index f03a8e73c..4b40f7030 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -238,7 +238,7 @@ spec: image: registry.access.redhat.com/ubi8-minimal:latest script: | #!/usr/bin/env bash - $(workspaces.source.path)/source/scanner/scripts/konflux/fetch-vuln-feed-data.sh "$(workspaces.source.path)/source" + $(workspaces.source.path)/source/scripts/konflux/fetch-vuln-feed-data.sh "$(workspaces.source.path)/source" workspaces: - name: source workspace: workspace From a3091e84fbc77a13e7db781d345bb7fc9f2106e3 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Wed, 31 Jan 2024 17:03:11 -0600 Subject: [PATCH 30/39] Enable prefetch and source image builds --- .tekton/scanner-pull-request.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 4b40f7030..ea2c84d08 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -33,6 +33,10 @@ spec: value: '{{revision}}' - name: rebuild value: 'true' + - name: prefetch-input + value: '{"type": "gomod", "path": "."}' + - name: build-source-image + value: 'true' workspaces: - name: workspace From 871d773582ea080a3ef78e21aeec922b996a58c6 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Wed, 31 Jan 2024 17:13:13 -0600 Subject: [PATCH 31/39] Clean up version logging in dockerfile --- image/scanner/konflux/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/image/scanner/konflux/Dockerfile b/image/scanner/konflux/Dockerfile index 7137c76f0..959709c0b 100644 --- a/image/scanner/konflux/Dockerfile +++ b/image/scanner/konflux/Dockerfile @@ -17,8 +17,7 @@ RUN unzip -j blob-repo2cpe.zip -d image/scanner/dump/repo2cpe && \ unzip -j blob-k8s-definitions.zip -d image/scanner/dump/k8s_definitions && \ unzip -j blob-nvd-definitions.zip -d image/scanner/dump/nvd_definitions -RUN echo "version: $(git describe --tags --abbrev=10 --dirty --long)" && \ - git status && \ +RUN echo "version: $(scripts/konflux/version.sh)" && \ go build -trimpath -ldflags="-X github.com/stackrox/scanner/pkg/version.Version=$(scripts/konflux/version.sh)" -o image/scanner/bin/scanner ./cmd/clair # Replace genesis manifests file in the source code with the one generated at From a8d2ec41ccf84458ddb9523727708e570388f94e Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Wed, 31 Jan 2024 22:33:12 -0600 Subject: [PATCH 32/39] sync tekton pipeline files --- .tekton/scanner-pull-request.yaml | 4 +- .tekton/scanner-push.yaml | 117 +++++++++++++++--------------- 2 files changed, 58 insertions(+), 63 deletions(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index ea2c84d08..45456d74c 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -24,6 +24,7 @@ spec: - name: git-url value: '{{repo_url}}' - name: image-expires-after + # TODO(ROX-20230): make release images not expire. value: '13w' - name: output-image value: quay.io/redhat-user-workloads/rh-acs-tenant/acs/scanner:on-pr-{{revision}} @@ -380,9 +381,6 @@ spec: workspace: workspace # - name: clamav-scan - # computeResources: - # requests: - # cpu: 1 # params: # - name: image-digest # value: $(tasks.build-container.results.IMAGE_DIGEST) diff --git a/.tekton/scanner-push.yaml b/.tekton/scanner-push.yaml index e134d8e34..2d8f9c2ec 100644 --- a/.tekton/scanner-push.yaml +++ b/.tekton/scanner-push.yaml @@ -30,7 +30,31 @@ spec: value: . - name: revision value: '{{revision}}' + - name: rebuild + value: 'true' + - name: prefetch-input + value: '{"type": "gomod", "path": "."}' + - name: build-source-image + value: 'true' + + workspaces: + - name: workspace + volumeClaimTemplate: + metadata: + creationTimestamp: null + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + status: {} + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' + pipelineSpec: + finally: - name: show-sbom params: @@ -64,6 +88,7 @@ spec: - name: kind value: task resolver: bundles + params: - description: Source Repository URL name: git-url @@ -93,10 +118,6 @@ spec: description: Skip checks against built image name: skip-checks type: string - - default: "true" - description: Skip optional checks, set false if you want to run optional checks - name: skip-optional - type: string - default: "false" description: Execute the build with network isolation name: hermetic @@ -118,6 +139,7 @@ spec: description: Build a source image. name: build-source-image type: string + results: - description: "" name: IMAGE_URL @@ -134,7 +156,13 @@ spec: - description: "" name: JAVA_COMMUNITY_DEPENDENCIES value: $(tasks.build-container.results.JAVA_COMMUNITY_DEPENDENCIES) + + workspaces: + - name: workspace + - name: git-auth + tasks: + - name: init params: - name: image-url @@ -143,12 +171,6 @@ spec: value: $(params.rebuild) - name: skip-checks value: $(params.skip-checks) - - name: skip-optional - value: $(params.skip-optional) - - name: pipelinerun-name - value: $(context.pipelineRun.name) - - name: pipelinerun-uid - value: $(context.pipelineRun.uid) taskRef: params: - name: name @@ -158,16 +180,17 @@ spec: - name: kind value: task resolver: bundles + - name: clone-repository params: - name: url value: $(params.git-url) - name: revision value: $(params.revision) - - name: fetchTags - value: true - name: depth - value: 0 + value: "0" + - name: fetchTags + value: "true" runAfter: - init taskRef: @@ -189,6 +212,7 @@ spec: workspace: workspace - name: basic-auth workspace: git-auth + - name: prefetch-dependencies params: - name: input @@ -204,44 +228,24 @@ spec: - name: kind value: task resolver: bundles - when: - - input: $(params.hermetic) - operator: in - values: - - "true" workspaces: - name: source workspace: workspace - name: fetch-vuln-feed-data runAfter: - init + - clone-repository taskSpec: steps: - name: fetch-vuln-feed-data - image: registry.access.redhat.com/ubi8/ubi + image: registry.access.redhat.com/ubi8-minimal:latest script: | #!/usr/bin/env bash - mkdir -p "$(workspaces.source.path)/source" - blobs=( - nvd-definitions.zip - k8s-definitions.zip - repo2cpe.zip - genesis_manifests.json - ) - for blob in "${blobs[@]}"; do - echo "https://storage.googleapis.com/definitions.stackrox.io/scanner-data/latest/${blob} > $(workspaces.source.path)/source/blob-${blob}" - curl --fail -s --show-error --retry 4 --retry-max-time 30 --retry-connrefused \ - --output "$(workspaces.source.path)/source/blob-${blob}" \ - "https://storage.googleapis.com/definitions.stackrox.io/scanner-data/latest/${blob}" - - if [ "$?" != "0" ]; then - echo "Failed to download" - exit 1 - fi - done + $(workspaces.source.path)/source/scripts/konflux/fetch-vuln-feed-data.sh "$(workspaces.source.path)/source" workspaces: - name: source workspace: workspace + - name: build-container params: - name: IMAGE @@ -264,9 +268,9 @@ spec: taskRef: params: - name: name - value: buildah-6gb + value: buildah - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-buildah-6gb:0.1 + value: quay.io/redhat-appstudio-tekton-catalog/task-buildah:0.1@sha256:e45cf454d90b81d0e23107856c009083c279cbe07f9bc48538b1977d69713f35 - name: kind value: task resolver: bundles @@ -278,6 +282,7 @@ spec: workspaces: - name: source workspace: workspace + - name: build-source-image params: - name: BINARY_IMAGE @@ -307,6 +312,7 @@ spec: workspaces: - name: workspace workspace: workspace + - name: deprecated-base-image-check params: - name: BASE_IMAGES_DIGESTS @@ -327,6 +333,7 @@ spec: operator: in values: - "false" + - name: clair-scan params: - name: image-digest @@ -349,6 +356,7 @@ spec: operator: in values: - "false" + - name: sast-snyk-check runAfter: - clone-repository @@ -369,6 +377,7 @@ spec: workspaces: - name: workspace workspace: workspace + # - name: clamav-scan # params: # - name: image-digest @@ -377,6 +386,7 @@ spec: # value: $(tasks.build-container.results.IMAGE_URL) # runAfter: # - build-container + # timeout: 3h # taskRef: # params: # - name: name @@ -391,6 +401,7 @@ spec: # operator: in # values: # - "false" + - name: sbom-json-check params: - name: IMAGE_URL @@ -413,24 +424,10 @@ spec: operator: in values: - "false" - workspaces: - - name: workspace - - name: git-auth - optional: true - taskRunTemplate: {} - workspaces: - - name: workspace - volumeClaimTemplate: - metadata: - creationTimestamp: null - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - status: {} - - name: git-auth - secret: - secretName: '{{ git_auth_secret }}' -status: {} + timeouts: + pipeline: "4h0m0s" + tasks: "3h0m0s" + finally: "1h0m0s" + +status: { } +taskRunTemplate: { } From aff4a23b42140268961463db904b52a373471fbc Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Thu, 1 Feb 2024 21:20:14 -0600 Subject: [PATCH 33/39] Addressing PR comments --- .tekton/scanner-pull-request.yaml | 6 +++--- .tekton/scanner-push.yaml | 3 ++- image/scanner/konflux/Dockerfile | 2 +- scripts/konflux/fetch-vuln-feed-data.sh | 3 ++- scripts/konflux/version.sh | 4 +++- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 45456d74c..0ceedcff8 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -8,7 +8,7 @@ metadata: build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "500" # TODO(ROX-21073): re-enable for all PR branches - pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && source_branch.contains("rhtap") + pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && (source_branch.contains("rhtap") || source_branch.contains("konflux")) creationTimestamp: null labels: appstudio.openshift.io/application: acs @@ -24,7 +24,6 @@ spec: - name: git-url value: '{{repo_url}}' - name: image-expires-after - # TODO(ROX-20230): make release images not expire. value: '13w' - name: output-image value: quay.io/redhat-user-workloads/rh-acs-tenant/acs/scanner:on-pr-{{revision}} @@ -233,6 +232,7 @@ spec: workspaces: - name: source workspace: workspace + - name: fetch-vuln-feed-data runAfter: - init @@ -243,7 +243,7 @@ spec: image: registry.access.redhat.com/ubi8-minimal:latest script: | #!/usr/bin/env bash - $(workspaces.source.path)/source/scripts/konflux/fetch-vuln-feed-data.sh "$(workspaces.source.path)/source" + "$(workspaces.source.path)/source/scripts/konflux/fetch-vuln-feed-data.sh" "$(workspaces.source.path)/source" workspaces: - name: source workspace: workspace diff --git a/.tekton/scanner-push.yaml b/.tekton/scanner-push.yaml index 2d8f9c2ec..f31160228 100644 --- a/.tekton/scanner-push.yaml +++ b/.tekton/scanner-push.yaml @@ -231,6 +231,7 @@ spec: workspaces: - name: source workspace: workspace + - name: fetch-vuln-feed-data runAfter: - init @@ -241,7 +242,7 @@ spec: image: registry.access.redhat.com/ubi8-minimal:latest script: | #!/usr/bin/env bash - $(workspaces.source.path)/source/scripts/konflux/fetch-vuln-feed-data.sh "$(workspaces.source.path)/source" + "$(workspaces.source.path)/source/scripts/konflux/fetch-vuln-feed-data.sh" "$(workspaces.source.path)/source" workspaces: - name: source workspace: workspace diff --git a/image/scanner/konflux/Dockerfile b/image/scanner/konflux/Dockerfile index 959709c0b..066fb6dfd 100644 --- a/image/scanner/konflux/Dockerfile +++ b/image/scanner/konflux/Dockerfile @@ -17,7 +17,7 @@ RUN unzip -j blob-repo2cpe.zip -d image/scanner/dump/repo2cpe && \ unzip -j blob-k8s-definitions.zip -d image/scanner/dump/k8s_definitions && \ unzip -j blob-nvd-definitions.zip -d image/scanner/dump/nvd_definitions -RUN echo "version: $(scripts/konflux/version.sh)" && \ +RUN echo -n "version: " && scripts/konflux/version.sh && \ go build -trimpath -ldflags="-X github.com/stackrox/scanner/pkg/version.Version=$(scripts/konflux/version.sh)" -o image/scanner/bin/scanner ./cmd/clair # Replace genesis manifests file in the source code with the one generated at diff --git a/scripts/konflux/fetch-vuln-feed-data.sh b/scripts/konflux/fetch-vuln-feed-data.sh index 24b37881b..634fea7bc 100755 --- a/scripts/konflux/fetch-vuln-feed-data.sh +++ b/scripts/konflux/fetch-vuln-feed-data.sh @@ -7,7 +7,7 @@ if [[ "$#" < 1 ]]; then exit 1 fi -TARGET_DIR=$1 +TARGET_DIR="$1" if [[ ! -d "$TARGET_DIR" ]]; then echo "$TARGET_DIR is not a valid directory" @@ -23,6 +23,7 @@ blobs=( for blob in "${blobs[@]}"; do echo "Downloading https://storage.googleapis.com/definitions.stackrox.io/scanner-data/latest/${blob} > $TARGET_DIR/blob-${blob}" + # TODO(ROX-22130): Assign proper suffix for tagged commits instead of /latest/. curl --fail -s --show-error --retry 4 --retry-max-time 30 --retry-connrefused \ --output "$TARGET_DIR/blob-${blob}" \ "https://storage.googleapis.com/definitions.stackrox.io/scanner-data/latest/${blob}" diff --git a/scripts/konflux/version.sh b/scripts/konflux/version.sh index eb1e3fcd6..0a0363753 100755 --- a/scripts/konflux/version.sh +++ b/scripts/konflux/version.sh @@ -1,9 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash # This script is used by the Konflux dockerfile to get the correct version to # compile in to the binary. If HEAD points to a git tag, use that; otherwise # use `git describe ...`. +set -euo pipefail + if [[ -n "$(git tag --contains)" ]]; then git tag --sort=creatordate --contains | tail -1 else From bbdcf1f66f13b343e2d68eef90957acf0d09aafd Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Fri, 2 Feb 2024 20:47:30 -0600 Subject: [PATCH 34/39] Re-enable clamav with new task image --- .tekton/scanner-pull-request.yaml | 46 +++++++++++++++---------------- .tekton/scanner-push.yaml | 46 +++++++++++++++---------------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 0ceedcff8..cd18b7239 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -380,29 +380,29 @@ spec: - name: workspace workspace: workspace - # - name: clamav-scan - # params: - # - name: image-digest - # value: $(tasks.build-container.results.IMAGE_DIGEST) - # - name: image-url - # value: $(tasks.build-container.results.IMAGE_URL) - # runAfter: - # - build-container - # timeout: 3h - # taskRef: - # params: - # - name: name - # value: clamav-scan - # - name: bundle - # value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:353fa2cda9855217cfcec3303973b666a10f384795630cf0eb13b874c24b0f7a - # - name: kind - # value: task - # resolver: bundles - # when: - # - input: $(params.skip-checks) - # operator: in - # values: - # - "false" + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-container.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-container.results.IMAGE_URL) + runAfter: + - build-container + timeout: 3h + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:3a546fe7c07fc24ecf3bddef88a7a9a66b525c4dd53acc8b5ea13069f60c086d + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" - name: sbom-json-check params: diff --git a/.tekton/scanner-push.yaml b/.tekton/scanner-push.yaml index f31160228..0217a54f8 100644 --- a/.tekton/scanner-push.yaml +++ b/.tekton/scanner-push.yaml @@ -379,29 +379,29 @@ spec: - name: workspace workspace: workspace - # - name: clamav-scan - # params: - # - name: image-digest - # value: $(tasks.build-container.results.IMAGE_DIGEST) - # - name: image-url - # value: $(tasks.build-container.results.IMAGE_URL) - # runAfter: - # - build-container - # timeout: 3h - # taskRef: - # params: - # - name: name - # value: clamav-scan - # - name: bundle - # value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:353fa2cda9855217cfcec3303973b666a10f384795630cf0eb13b874c24b0f7a - # - name: kind - # value: task - # resolver: bundles - # when: - # - input: $(params.skip-checks) - # operator: in - # values: - # - "false" + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-container.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-container.results.IMAGE_URL) + runAfter: + - build-container + timeout: 3h + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:3a546fe7c07fc24ecf3bddef88a7a9a66b525c4dd53acc8b5ea13069f60c086d + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" - name: sbom-json-check params: From 33aa8281766370ca30347bb20949946b3a7ff7f5 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Mon, 5 Feb 2024 08:45:20 -0600 Subject: [PATCH 35/39] Remove timeout overrides --- .tekton/scanner-pull-request.yaml | 6 +----- .tekton/scanner-push.yaml | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index cd18b7239..8c9a724d2 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -388,7 +388,6 @@ spec: value: $(tasks.build-container.results.IMAGE_URL) runAfter: - build-container - timeout: 3h taskRef: params: - name: name @@ -426,10 +425,7 @@ spec: operator: in values: - "false" - timeouts: - pipeline: "4h0m0s" - tasks: "3h0m0s" - finally: "1h0m0s" status: { } + taskRunTemplate: { } diff --git a/.tekton/scanner-push.yaml b/.tekton/scanner-push.yaml index 0217a54f8..3d714b14d 100644 --- a/.tekton/scanner-push.yaml +++ b/.tekton/scanner-push.yaml @@ -387,7 +387,6 @@ spec: value: $(tasks.build-container.results.IMAGE_URL) runAfter: - build-container - timeout: 3h taskRef: params: - name: name @@ -425,10 +424,7 @@ spec: operator: in values: - "false" - timeouts: - pipeline: "4h0m0s" - tasks: "3h0m0s" - finally: "1h0m0s" status: { } + taskRunTemplate: { } From eb3df6c7101aeb595e7043c384eb1b331a6cbaa5 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Mon, 5 Feb 2024 16:05:51 -0600 Subject: [PATCH 36/39] Trigger build From 3df9f5ac85e899fa1d3a3021c477d869b6aa1913 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Tue, 6 Feb 2024 08:56:54 -0600 Subject: [PATCH 37/39] Move dockerfile to be consistent with scanner-db --- .tekton/scanner-pull-request.yaml | 2 +- .tekton/scanner-push.yaml | 2 +- image/scanner/{konflux/Dockerfile => rhel/konflux.Dockerfile} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename image/scanner/{konflux/Dockerfile => rhel/konflux.Dockerfile} (100%) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 8c9a724d2..8b3d8ed32 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -20,7 +20,7 @@ spec: params: - name: dockerfile - value: image/scanner/konflux/Dockerfile + value: image/scanner/rhel/konflux.Dockerfile - name: git-url value: '{{repo_url}}' - name: image-expires-after diff --git a/.tekton/scanner-push.yaml b/.tekton/scanner-push.yaml index 3d714b14d..68377f5eb 100644 --- a/.tekton/scanner-push.yaml +++ b/.tekton/scanner-push.yaml @@ -18,7 +18,7 @@ spec: params: - name: dockerfile - value: image/scanner/konflux/Dockerfile + value: image/scanner/rhel/konflux.Dockerfile - name: git-url value: '{{repo_url}}' - name: image-expires-after diff --git a/image/scanner/konflux/Dockerfile b/image/scanner/rhel/konflux.Dockerfile similarity index 100% rename from image/scanner/konflux/Dockerfile rename to image/scanner/rhel/konflux.Dockerfile From e36ec93c9fc59e91ed1e89fe1a52a6e041456688 Mon Sep 17 00:00:00 2001 From: Kyle Lape Date: Tue, 6 Feb 2024 09:08:37 -0600 Subject: [PATCH 38/39] Update validation script with new dockerfile name --- scripts/konflux/fail-build-if-git-is-dirty.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/konflux/fail-build-if-git-is-dirty.sh b/scripts/konflux/fail-build-if-git-is-dirty.sh index 423907b60..4ea829b66 100755 --- a/scripts/konflux/fail-build-if-git-is-dirty.sh +++ b/scripts/konflux/fail-build-if-git-is-dirty.sh @@ -16,7 +16,7 @@ set -euo pipefail if [[ -f /cachi2/cachi2.env ]]; then # We can safely restore dockerfiles because the modified version of dockerfile interpreted by docker/buildah stays # outside, and these are local copies inside of the build context. - git restore "**/konflux/Dockerfile" + git restore "**/konflux.Dockerfile" fi # Next, make sure no other things that make it `-dirty` slipped through. If they did, fail the build. From d28fe061855b2c8f15a463d6d3ad0a35a0197af0 Mon Sep 17 00:00:00 2001 From: "red-hat-konflux[bot]" <126015336+red-hat-konflux[bot]@users.noreply.github.com> Date: Tue, 6 Feb 2024 09:41:12 -0600 Subject: [PATCH 39/39] Update RHTAP references (klape/rhtap-scanner-onboarding) (#1391) Co-authored-by: red-hat-konflux <123456+red-hat-konflux[bot]@users.noreply.github.com> --- .tekton/scanner-pull-request.yaml | 6 +++--- .tekton/scanner-push.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.tekton/scanner-pull-request.yaml b/.tekton/scanner-pull-request.yaml index 8b3d8ed32..3db74f7fb 100644 --- a/.tekton/scanner-pull-request.yaml +++ b/.tekton/scanner-pull-request.yaml @@ -85,7 +85,7 @@ spec: - name: name value: summary - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-summary:0.1@sha256:f65a69aaf71cbab382eff685eee522ad35068a4d91d233e76cef7d42ff15a686 + value: quay.io/redhat-appstudio-tekton-catalog/task-summary:0.1@sha256:29a64be421fdc203cb26c61b746c650e239ae924a73a825ad93bffb9e7ae7214 - name: kind value: task resolver: bundles @@ -272,7 +272,7 @@ spec: - name: name value: buildah - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-buildah:0.1@sha256:e45cf454d90b81d0e23107856c009083c279cbe07f9bc48538b1977d69713f35 + value: quay.io/redhat-appstudio-tekton-catalog/task-buildah:0.1@sha256:1ee1d570b3cf7feaf85a22c828076e798a57266ffcfea41c1aa0e9f85b0a885a - name: kind value: task resolver: bundles @@ -298,7 +298,7 @@ spec: - name: name value: source-build - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-source-build:0.1@sha256:f8c5dec871fb5347eb2fc61d44754bcc101897aecf953b374ab3e8315e1a9804 + value: quay.io/redhat-appstudio-tekton-catalog/task-source-build:0.1@sha256:3ad20adff4aa5cd153695b115133cb7c71c87f095af02fae5932396b1c72eb00 - name: kind value: task resolver: bundles diff --git a/.tekton/scanner-push.yaml b/.tekton/scanner-push.yaml index 68377f5eb..dd05d8543 100644 --- a/.tekton/scanner-push.yaml +++ b/.tekton/scanner-push.yaml @@ -84,7 +84,7 @@ spec: - name: name value: summary - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-summary:0.1@sha256:f65a69aaf71cbab382eff685eee522ad35068a4d91d233e76cef7d42ff15a686 + value: quay.io/redhat-appstudio-tekton-catalog/task-summary:0.1@sha256:29a64be421fdc203cb26c61b746c650e239ae924a73a825ad93bffb9e7ae7214 - name: kind value: task resolver: bundles @@ -271,7 +271,7 @@ spec: - name: name value: buildah - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-buildah:0.1@sha256:e45cf454d90b81d0e23107856c009083c279cbe07f9bc48538b1977d69713f35 + value: quay.io/redhat-appstudio-tekton-catalog/task-buildah:0.1@sha256:1ee1d570b3cf7feaf85a22c828076e798a57266ffcfea41c1aa0e9f85b0a885a - name: kind value: task resolver: bundles @@ -297,7 +297,7 @@ spec: - name: name value: source-build - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-source-build:0.1@sha256:f8c5dec871fb5347eb2fc61d44754bcc101897aecf953b374ab3e8315e1a9804 + value: quay.io/redhat-appstudio-tekton-catalog/task-source-build:0.1@sha256:3ad20adff4aa5cd153695b115133cb7c71c87f095af02fae5932396b1c72eb00 - name: kind value: task resolver: bundles