From 6773854ce8058bca1b7d07f942bddea160476736 Mon Sep 17 00:00:00 2001 From: Cristovao Cordeiro Date: Mon, 11 Sep 2023 16:56:13 +0200 Subject: [PATCH] ci: notify on failed vuln scan --- .github/workflows/Continuous-Testing.yaml | 64 +++----- .github/workflows/Image.yaml | 1 + .github/workflows/Tests.yaml | 110 ++----------- .github/workflows/Vulnerability-Scan.yaml | 178 ++++++++++++++++++++++ 4 files changed, 217 insertions(+), 136 deletions(-) create mode 100644 .github/workflows/Vulnerability-Scan.yaml diff --git a/.github/workflows/Continuous-Testing.yaml b/.github/workflows/Continuous-Testing.yaml index 9c24c1506..119bc708c 100644 --- a/.github/workflows/Continuous-Testing.yaml +++ b/.github/workflows/Continuous-Testing.yaml @@ -2,14 +2,15 @@ name: Continuous image testing on: schedule: - - cron: "0 1 * * *" + - cron: "*/4 * * * *" jobs: - list-released-images: + prepare-test-matrix: runs-on: ubuntu-latest - name: List the revisions of released images + name: Prepare released image revisions to be tested outputs: released-revisions-matrix: ${{ steps.prepare-test-matrix.outputs.released-revisions-matrix }} + last-scan: ${{ steps.last-scan.outputs.date }} steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -21,43 +22,24 @@ jobs: id: prepare-test-matrix run: ./src/tests/get_released_revisions.py --oci-images-path $PWD/oci - dispatch-tests: - runs-on: ubuntu-latest - name: Dispatch tests for released images - needs: [list-released-images] - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.list-released-images.outputs.released-revisions-matrix) }} - steps: - - name: Run tests for ${{ matrix.source-image }} - # Using this actions cause others can have this problem: - # https://github.com/convictional/trigger-workflow-and-wait/issues/61 - uses: mathze/workflow-dispatch-action@v1.2.0 - id: run-tests - env: - IS_A_ROCK: ${{ matrix.dockerfile-build == '' && true || false }} - with: - token: ${{ secrets.GITHUB_TOKEN }} - ref: ${{ github.ref_name }} - fail-on-error: true - workflow-name: Tests.yaml - # For continuous auditing, let's assume all images are NOT ROCKs and - # thus only run the most generic tests - payload: '{ "oci-image-name": "${{ matrix.source-image }}", "oci-image-path": "oci/${{ matrix.name }}", "is-a-rock": false, "test-from": "registry"}' - trigger-timeout: "5m" - wait-timeout: "45m" - run-id: dummy - use-marker-step: true - - - name: Write step summary + - name: Infer date of last scan + id: last-scan run: | - url='${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.run-tests.outputs.run-id }}' - echo " - Triggered tests for '${{ matrix.source-image }}' at [${url}](${url})" >> "$GITHUB_STEP_SUMMARY" + # This is scheduled to run every day, so let's look at the previous + # 26 hours, roughly + set -ex + last_scan="$(date --date='26 hours ago' +'%Y-%m-%dT%H:%M:00Z')" + echo "date=$last_scan" >> "$GITHUB_OUTPUT" - - name: Enforce test conclusion - if: ${{ steps.run-tests.outputs.run-conclusion != 'success' }} - # The previous step doesn't always raise an error - run: | - url='${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.run-tests.outputs.run-id }}' - echo "Testing of image '${{ matrix.source-image }}' failed at [${url}](${url})." - exit 1 + run-tests: + name: Run tests for released images + needs: [prepare-test-matrix] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.prepare-test-matrix.outputs.released-revisions-matrix) }} + uses: canonical/oci-factory/.github/workflows/Vulnerability-Scan.yaml@main + with: + oci-image-name: "${{ matrix.source-image }}" + oci-image-path: "oci/${{ matrix.name }}" + date-last-scan: ${{ needs.prepare-test-matrix.outputs.last-scan }} + secrets: inherit diff --git a/.github/workflows/Image.yaml b/.github/workflows/Image.yaml index 3ef7f7393..7a97775d3 100644 --- a/.github/workflows/Image.yaml +++ b/.github/workflows/Image.yaml @@ -218,6 +218,7 @@ jobs: is-a-rock: ${{ matrix.dockerfile-build == '' && true || false }} test-from: "cache" cache-key: "${{ github.run_id }}-${{ matrix.name }}_${{ matrix.commit }}_${{ matrix.revision }}" + secrets: inherit upload: runs-on: ubuntu-22.04 diff --git a/.github/workflows/Tests.yaml b/.github/workflows/Tests.yaml index 26cc588b3..9c9cbcdd3 100644 --- a/.github/workflows/Tests.yaml +++ b/.github/workflows/Tests.yaml @@ -31,11 +31,6 @@ on: required: false type: string default: '.vulnerability-report.json' - external_ref_id: #(1) - description: 'Optional ID for unique run detection' - required: false - type: string - default: "default-id" workflow_dispatch: inputs: oci-image-name: @@ -84,6 +79,8 @@ jobs: fetch-oci-image: runs-on: ubuntu-22.04 name: Fetch OCI image for testing + outputs: + test-cache-key: ${{ steps.cache.outputs.key }} steps: - name: ${{ inputs.external_ref_id }} #(2) run: echo 'Started by ${{ inputs.external_ref_id }}' >> "$GITHUB_STEP_SUMMARY" @@ -118,6 +115,10 @@ jobs: path: ${{ env.TEST_IMAGE_NAME}} key: ${{ github.run_id }}-${{ inputs.oci-image-name }}-${{ env.TEST_IMAGE_NAME }} + - name: Save cache key + id: cache + run: echo "key=${{ github.run_id }}-${{ inputs.oci-image-name }}-${{ env.TEST_IMAGE_NAME }}" >> "$GITHUB_OUTPUT" + test-oci-compliance: runs-on: ubuntu-22.04 @@ -127,7 +128,7 @@ jobs: - uses: actions/cache/restore@v3 with: path: ${{ env.TEST_IMAGE_NAME}} - key: ${{ github.run_id }}-${{ inputs.oci-image-name }}-${{ env.TEST_IMAGE_NAME }} + key: ${{ needs.fetch-oci-image.outputs.test-cache-key }} - name: Install Umoci run: | @@ -155,7 +156,7 @@ jobs: - uses: actions/cache/restore@v3 with: path: ${{ env.TEST_IMAGE_NAME}} - key: ${{ github.run_id }}-${{ inputs.oci-image-name }}-${{ env.TEST_IMAGE_NAME }} + key: ${{ needs.fetch-oci-image.outputs.test-cache-key }} - name: Copy image to Docker daemon run: | @@ -189,7 +190,7 @@ jobs: - uses: actions/cache/restore@v3 with: path: ${{ env.TEST_IMAGE_NAME}} - key: ${{ github.run_id }}-${{ inputs.oci-image-name }}-${{ env.TEST_IMAGE_NAME }} + key: ${{ needs.fetch-oci-image.outputs.test-cache-key }} - name: Copy image to Docker daemon run: | @@ -212,74 +213,16 @@ jobs: test-vulnerabilities: - runs-on: ubuntu-22.04 name: Vulnerability scan needs: [fetch-oci-image] - outputs: - vulnerability-report: ${{ steps.vulnerability-report.outputs.name }} - steps: - - uses: actions/checkout@v3 - - - id: vulnerability-report - run: | - full_name="${{ inputs.oci-image-name }}${{ inputs.vulnerability-report-suffix }}" - final_name="$(echo ${full_name} | sed 's/ghcr.io\/canonical\/oci-factory\///g' | tr ':' '_')" - echo "name=$final_name" >> "$GITHUB_OUTPUT" - - - uses: actions/cache/restore@v3 - with: - path: ${{ env.TEST_IMAGE_NAME}} - key: ${{ github.run_id }}-${{ inputs.oci-image-name }}-${{ env.TEST_IMAGE_NAME }} - - - name: Copy image to Docker daemon - run: | - docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \ - -v $PWD:/workdir -w /workdir \ - ${{ env.SKOPEO_IMAGE }} \ - copy oci:${{ env.TEST_IMAGE_NAME}}:${{ env.TEST_IMAGE_TAG }} \ - docker-daemon:${{ env.TEST_IMAGE_NAME}}:${{ env.TEST_IMAGE_TAG }} - - - name: Check for .trivyignore - id: trivyignore - run: | - if [ -f ${{ inputs.oci-image-path }}/.trivyignore ] - then - file=${{ inputs.oci-image-path }}/.trivyignore - else - # dummy .trivyignore file - file=.trivyignore - touch $file - fi - echo "file=$file" >> "$GITHUB_OUTPUT" - - - name: Scan for vulnerabilities - uses: aquasecurity/trivy-action@0.9.2 - with: - # NOTE: we're allowing images with vulnerabilities to be published - ignore-unfixed: true - trivyignores: ${{ steps.trivyignore.outputs.file }} - format: 'cosign-vuln' - severity: 'HIGH,CRITICAL' - exit-code: '1' - # NOTE: pebble is flagged with a HIGH vuln because of golang.org/x/crypto - # CVE-2021-43565, CVE-2022-27191 - skip-files: /bin/pebble - # missing ${{ runner.arch }} - output: '${{ steps.vulnerability-report.outputs.name }}' - image-ref: '${{ env.TEST_IMAGE_NAME}}:${{ env.TEST_IMAGE_TAG }}' - - - if: ${{ always() }} - run: | - cat ${{ steps.vulnerability-report.outputs.name }} - echo "report=$report" >> "$GITHUB_OUTPUT" + uses: canonical/oci-factory/.github/workflows/Vulnerability-Scan.yaml@main + with: + oci-image-name: "${{ inputs.oci-image-name }}" + oci-image-path: "${{ inputs.oci-image-path }}" + cache-key: "${{ needs.fetch-oci-image.outputs.test-cache-key }}" + vulnerability-report-suffix: "${{ inputs.vulnerability-report-suffix}}" - - uses: actions/cache/save@v3 - if: ${{ always() }} - with: - path: ${{ steps.vulnerability-report.outputs.name }} - key: ${{ github.run_id }}-${{ steps.vulnerability-report.outputs.name }} - test-malware: runs-on: ubuntu-22.04 name: Malware scan @@ -314,26 +257,3 @@ jobs: - name: Scan for malware run: | ./src/tests/malware_scan.py --filesystem ./raw/rootfs - - - upload-test-artefacts: - name: Upload test artefacts - runs-on: ubuntu-22.04 - if: ${{ always() }} - needs: - - test-vulnerabilities - - test-black-box - - test-oci-compliance - - test-malware - - test-efficiency - steps: - - name: Restore vulnerability report for upload - uses: actions/cache/restore@v3 - with: - path: ${{ needs.test-vulnerabilities.outputs.vulnerability-report }} - key: ${{ github.run_id }}-${{ needs.test-vulnerabilities.outputs.vulnerability-report }} - - - uses: actions/upload-artifact@v3 - with: - name: ${{ needs.test-vulnerabilities.outputs.vulnerability-report }} - path: ${{ needs.test-vulnerabilities.outputs.vulnerability-report }} diff --git a/.github/workflows/Vulnerability-Scan.yaml b/.github/workflows/Vulnerability-Scan.yaml new file mode 100644 index 000000000..2b99bb2e4 --- /dev/null +++ b/.github/workflows/Vulnerability-Scan.yaml @@ -0,0 +1,178 @@ +name: Vulnerability Scan +run-name: 'Tests - ${{ inputs.oci-image-name }} - ${{ github.ref }}' + +on: + workflow_call: + inputs: + oci-image-name: + description: 'Name of the image to be fetched and tested' + required: true + type: string + oci-image-path: + description: 'Path to the image in this repo (eg. "oci/foo")' + required: true + type: string + cache-key: + description: 'Key ID for restoring image (in OCI format) from cache' + required: false + type: string + default: '' + vulnerability-report-suffix: + description: 'Suffix for the vulnerability report artefact' + required: false + type: string + default: '.vulnerability-report.json' + date-last-scan: + description: 'If there are new CVEs after this date, we notify' + required: false + type: string + default: '9999-12-31T23:59:59' + +env: + TEST_IMAGE_NAME: 'test-img' + TEST_IMAGE_TAG: 'test' + SKOPEO_IMAGE: 'quay.io/skopeo/stable:v1.13' + +jobs: + test-vulnerabilities: + runs-on: ubuntu-22.04 + name: Vulnerability scan + outputs: + vulnerability-report: ${{ steps.vulnerability-report.outputs.name }} + notify: ${{ steps.check-report.outputs.notify }} + steps: + - uses: actions/checkout@v3 + + - id: vulnerability-report + run: | + full_name="${{ inputs.oci-image-name }}${{ inputs.vulnerability-report-suffix }}" + final_name="$(echo ${full_name} | sed 's/ghcr.io\/canonical\/oci-factory\///g' | tr ':' '_')" + echo "name=$final_name" >> "$GITHUB_OUTPUT" + + - uses: actions/cache/restore@v3 + if: ${{ inputs.cache-key != '' }} + with: + path: ${{ env.TEST_IMAGE_NAME}} + key: ${{ inputs.cache-key }} + fail-on-cache-miss: true + + - name: Copy image to Docker daemon + id: to-docker-daemon + run: | + d_img_name="${{ env.TEST_IMAGE_NAME}}:${{ env.TEST_IMAGE_TAG }}" + echo "name=$d_img_name" >> "$GITHUB_OUTPUT" + if [[ "${{ inputs.cache-key}}" != "" ]] + then + source=oci:${{ env.TEST_IMAGE_NAME}} + else + source=docker://${{ inputs.oci-image-name }} + fi + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \ + -v $PWD:/workdir -w /workdir \ + ${{ env.SKOPEO_IMAGE }} \ + copy $source \ + docker-daemon:$d_img_name + + - name: Check for .trivyignore + id: trivyignore + run: | + if [ -f ${{ inputs.oci-image-path }}/.trivyignore ] + then + file=${{ inputs.oci-image-path }}/.trivyignore + else + # dummy .trivyignore file + file=.trivyignore + touch $file + fi + echo "file=$file" >> "$GITHUB_OUTPUT" + + - name: Scan for vulnerabilities + uses: aquasecurity/trivy-action@0.9.2 + with: + # NOTE: we're allowing images with vulnerabilities to be published + ignore-unfixed: true + trivyignores: ${{ steps.trivyignore.outputs.file }} + format: 'cosign-vuln' + severity: 'HIGH,CRITICAL' + exit-code: '1' + # NOTE: pebble is flagged with a HIGH vuln because of golang.org/x/crypto + # CVE-2021-43565, CVE-2022-27191 + skip-files: /bin/pebble + # missing ${{ runner.arch }} + output: '${{ steps.vulnerability-report.outputs.name }}' + image-ref: '${{ steps.to-docker-daemon.outputs.name }}' + + - name: Process report + if: ${{ always() }} + id: check-report + run: | + report="${{ steps.vulnerability-report.outputs.name }}" + cat $report + echo "notify=false" >> "$GITHUB_OUTPUT" + set -x + last_modified_dates="$(jq -r 'try(.scanner.result.Results[].Vulnerabilities) + | select(. != null) + | .[].LastModifiedDate + | select(. != null)' < $report)" + + # We want to notify only if the CVEs have been updated since the last + # time this scan ran + for cve_updated in $last_modified_dates + do + if [[ "$cve_updated" > "${{ inputs.date-last-scan }}" ]] + then + echo "notify=true" >> "$GITHUB_OUTPUT" + break + fi + done + + - uses: actions/cache/save@v3 + if: ${{ always() }} + with: + path: ${{ steps.vulnerability-report.outputs.name }} + key: ${{ github.run_id }}-${{ steps.vulnerability-report.outputs.name }} + + - uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: ${{ steps.vulnerability-report.outputs.name }} + path: ${{ steps.vulnerability-report.outputs.name }} + + + # Many workflows are now using a similar notification job. It would be better + # if this was a common workflows reachable via a workflow_call + notify: + runs-on: ubuntu-22.04 + name: Notify on failure + needs: + - test-vulnerabilities + if: ${{ always() && needs.test-vulnerabilities.outputs.notify == 'false' }} + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Get contacts for ${{ inputs.oci-image-name }} + id: get-contacts + working-directory: ${{ inputs.oci-image-path }} + run: | + mm_channels=$(yq -r '.notify | ."mattermost-channels" | join(",")' < contacts.y*ml) + echo "mattermost-channels=${mm_channels}" >> "$GITHUB_OUTPUT" + + - name: Notify via Mattermost + env: + MM_BOT_TOKEN: ${{ secrets.MM_BOT_TOKEN }} + FINAL_STATUS: failure + MM_SERVER: ${{ secrets.MM_SERVER }} + URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SUMMARY: '' + FOOTER: "Triggered by ${{ github.triggering_actor }}. Ref: ${{ github.ref }}. Attempts: ${{ github.run_attempt }}" + TITLE: 'CVEs found for ${{ inputs.oci-image-name }}' + run: | + for channel in $(echo ${{ steps.get-contacts.outputs.mattermost-channels }} | tr ',' ' ') + do + #MM_CHANNEL_ID="${channel}" ./src/notifications/send_to_mattermost.sh + MM_CHANNEL_ID="fbdezwkcxpfofpysjore1wpfoc" ./src/notifications/send_to_mattermost.sh + done