From 8e8448f2650834daf567f4e18c0f3cf9b507c452 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Tue, 5 Apr 2022 17:22:33 -0400 Subject: [PATCH] Migrate Python tests to JavaScript (#160) --- .github/workflows/demo.yml | 26 ++--- .github/workflows/sarifdemo.yml | 60 ++++++------ .github/workflows/test.yml | 22 ----- Makefile | 68 ------------- dist/index.js | 16 +-- index.js | 16 +-- scripts/local.sh | 23 ----- tests/README.md | 24 ++--- tests/action.test.js | 61 ++++++++++-- tests/functional/output/.keep | 0 tests/functional/test_images.py | 21 ---- tests/functional/test_invalid_input.py | 61 ------------ tests/grype_command.test.js | 14 +-- tests/python/setup.py | 48 --------- tests/sarif_output.test.js | 2 + workflows/image.yml | 28 ------ workflows/tests.yml | 129 ------------------------- 17 files changed, 128 insertions(+), 491 deletions(-) delete mode 100644 Makefile delete mode 100755 scripts/local.sh delete mode 100644 tests/functional/output/.keep delete mode 100644 tests/functional/test_images.py delete mode 100644 tests/functional/test_invalid_input.py delete mode 100644 tests/python/setup.py delete mode 100644 workflows/image.yml delete mode 100644 workflows/tests.yml diff --git a/.github/workflows/demo.yml b/.github/workflows/demo.yml index 5414e14e..8c1db063 100644 --- a/.github/workflows/demo.yml +++ b/.github/workflows/demo.yml @@ -6,20 +6,20 @@ jobs: test-image: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: ./ - with: - image: "alpine:latest" - debug: true - fail-build: false + - uses: actions/checkout@v2 + - uses: ./ + with: + image: "alpine:latest" + debug: true + fail-build: false test-directory: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: ./ - with: - path: "tests/python" - debug: true - severity-cutoff: "negligible" - fail-build: false + - uses: actions/checkout@v2 + - uses: ./ + with: + path: "tests/fixtures/npm-project" + debug: true + severity-cutoff: "negligible" + fail-build: false diff --git a/.github/workflows/sarifdemo.yml b/.github/workflows/sarifdemo.yml index 9947e178..3b1acfcc 100644 --- a/.github/workflows/sarifdemo.yml +++ b/.github/workflows/sarifdemo.yml @@ -6,21 +6,21 @@ jobs: sarif-image: runs-on: ubuntu-latest steps: - - name: Checkout the code - uses: actions/checkout@v2 - - - name: Run the local Scan Action with SARIF generation enabled - id: scan - uses: ./ - with: - image: "debian:8" - debug: true - acs-report-enable: true - fail-build: false - #severity-cutoff: "Medium" - - - name: Inspect Generated SARIF - run: cat ${{ steps.scan.outputs.sarif }} + - name: Checkout the code + uses: actions/checkout@v2 + + - name: Run the local Scan Action with SARIF generation enabled + id: scan + uses: ./ + with: + image: "debian:8" + debug: true + acs-report-enable: true + fail-build: false + #severity-cutoff: "Medium" + + - name: Inspect Generated SARIF + run: cat ${{ steps.scan.outputs.sarif }} # Commented out to prevent incorrect SARIF uploads for this action # TODO: add functional tests that validate this @@ -32,21 +32,21 @@ jobs: sarif-directory: runs-on: ubuntu-latest steps: - - name: Checkout the code - uses: actions/checkout@v2 - - - name: Run the local Scan Action with SARIF generation enabled - id: scan - uses: ./ - with: - path: "tests/python" - debug: true - acs-report-enable: true - fail-build: false - #severity-cutoff: "Medium" - - - name: Inspect Generated SARIF - run: cat ${{ steps.scan.outputs.sarif }} + - name: Checkout the code + uses: actions/checkout@v2 + + - name: Run the local Scan Action with SARIF generation enabled + id: scan + uses: ./ + with: + path: "tests/fixtures/npm-project" + debug: true + acs-report-enable: true + fail-build: false + #severity-cutoff: "Medium" + + - name: Inspect Generated SARIF + run: cat ${{ steps.scan.outputs.sarif }} # Commented out to prevent incorrect SARIF uploads for this action # TODO: add functional tests that validate this # - name: Upload SARIF diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index de5d3a47..35568fff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,18 +12,6 @@ jobs: - 5000:5000 steps: - uses: actions/checkout@v2 - - run: echo $(uname -a) - - name: Check for npm (so make test works) - run: | - if ! [ -x "$(command -v npm)" ]; then - sudo apt update - sudo apt -y upgrade - sudo apt update - sudo apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates - curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash - - sudo apt -y install nodejs - sudo apt -y install gcc g++ make - fi - name: Build images run: | for distro in alpine centos debian; do @@ -39,13 +27,3 @@ jobs: - run: npm ci - run: npm audit --production - run: npm test - - functional: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - run: make check - - uses: actions/upload-artifact@v2 - with: - name: functional-test-output - path: tests/functional/output/* diff --git a/Makefile b/Makefile deleted file mode 100644 index 210e58db..00000000 --- a/Makefile +++ /dev/null @@ -1,68 +0,0 @@ -SHELL := /usr/bin/env bash -IMAGEDIRS = $(shell ls -d containers/* | cut -d '/' -f 2) -BOLD := $(shell tput -T linux bold) -PURPLE := $(shell tput -T linux setaf 5) -GREEN := $(shell tput -T linux setaf 2) -CYAN := $(shell tput -T linux setaf 6) -RED := $(shell tput -T linux setaf 1) -RESET := $(shell tput -T linux sgr0) -TITLE := $(BOLD)$(PURPLE) -SUCCESS := $(BOLD)$(GREEN) - -define title - @printf '$(TITLE)$(1)$(RESET)\n' -endef - - -.PHONY: all -all: build - @printf '$(SUCCESS)All checks pass!$(RESET)\n' - -help: - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(BOLD)$(CYAN)%-35s$(RESET)%s\n", $$1, $$2}' - @for job in $(shell egrep "\w+:$$" workflows/*.yml | grep -v 'with:\|jobs:\|steps:' | cut -d ' ' -f 3 | cut -d ':' -f 1); do \ - printf "$(BOLD)$(CYAN)%-35s$(RESET)Run %s action with act\n" $$job $$job; \ - done - -.PHONY: run -run: ## Run all Github Action steps as defined in the workflows directory - @for job in $(shell egrep "\w+:$$" workflows/*.yml | grep -v 'with:\|jobs:\|steps:' | cut -d ' ' -f 3 | cut -d ':' -f 1); do \ - printf "$(BOLD)$(CYAN)Running Step: %-35s$(RESET)\n" $$job; \ - ./act -v -W workflows -j $$job > tests/functional/output/$$job.output 2>&1; \ - echo $$? >> tests/functional/output/$$job.output; \ - done - -.PHONY: check -check: bootstrap run ## Run all Github Action steps and then verify them with tests - python3 -m venv venv - venv/bin/pip install pytest - venv/bin/pytest tests/functional - -.PHONY: boostrap -bootstrap: ## Download and install all go dependencies (+ prep tooling in the ./tmp dir) - $(call title,Boostrapping dependencies) - @pwd - # Install `act` to run the actions - curl https://raw.githubusercontent.com/nektos/act/master/install.sh | sh -s -- -b . v0.2.17 - # prep temp dirs - mkdir -p tests/functional/output - -.PHONY: run-docker-registry -run-docker-registry: bootstrap - # start a local registry - docker run -d -p 5000:5000 --name registry registry:2 | echo - -.PHONY: test -test: run-docker-registry bootstrap - npm run build - ./act -v -P ubuntu-latest=ghcr.io/catthehacker/ubuntu:js-latest -j test - -.PHONY: wipe-docker -wipe-docker: - docker kill $(shell docker ps -a -q) | echo - docker image prune -af - docker container prune -f - docker volume prune -f - -%: ## do a local build - scripts/local.sh "$*" diff --git a/dist/index.js b/dist/index.js index b6ab4361..f093b6b5 100644 --- a/dist/index.js +++ b/dist/index.js @@ -81,10 +81,10 @@ async function run() { // Grype accepts several input options, initially this action is supporting both `image` and `path`, so // a check must happen to ensure one is selected at least, and then return it const source = sourceInput(); - const debug = core.getInput("debug"); - const failBuild = core.getInput("fail-build"); - const acsReportEnable = core.getInput("acs-report-enable"); - const severityCutoff = core.getInput("severity-cutoff"); + const debug = core.getInput("debug") || "false"; + const failBuild = core.getInput("fail-build") || "true"; + const acsReportEnable = core.getInput("acs-report-enable") || "true"; + const severityCutoff = core.getInput("severity-cutoff") || "medium"; const out = await runScan({ source, debug, @@ -102,10 +102,10 @@ async function run() { async function runScan({ source, - debug = "false", - failBuild = "true", - acsReportEnable = "true", - severityCutoff = "medium", + debug, + failBuild, + acsReportEnable, + severityCutoff, }) { const out = {}; diff --git a/index.js b/index.js index d2d27c32..6b5c1757 100644 --- a/index.js +++ b/index.js @@ -66,10 +66,10 @@ async function run() { // Grype accepts several input options, initially this action is supporting both `image` and `path`, so // a check must happen to ensure one is selected at least, and then return it const source = sourceInput(); - const debug = core.getInput("debug"); - const failBuild = core.getInput("fail-build"); - const acsReportEnable = core.getInput("acs-report-enable"); - const severityCutoff = core.getInput("severity-cutoff"); + const debug = core.getInput("debug") || "false"; + const failBuild = core.getInput("fail-build") || "true"; + const acsReportEnable = core.getInput("acs-report-enable") || "true"; + const severityCutoff = core.getInput("severity-cutoff") || "medium"; const out = await runScan({ source, debug, @@ -87,10 +87,10 @@ async function run() { async function runScan({ source, - debug = "false", - failBuild = "true", - acsReportEnable = "true", - severityCutoff = "medium", + debug, + failBuild, + acsReportEnable, + severityCutoff, }) { const out = {}; diff --git a/scripts/local.sh b/scripts/local.sh deleted file mode 100755 index 1acf52a2..00000000 --- a/scripts/local.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# This script is only meant to run from Makefile in the parent directory of this repository - -SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" - -if [ $# -eq 0 ] - then - echo "No arguments supplied, this command requires a Github Action step tag to run, like:" - for job in $(egrep "\w+:$$" workflows/*.yml | grep -v 'with:\|jobs:\|steps:' | cut -d ' ' -f 3 | cut -d ':' -f 1); do \ - echo "- $job" - done - exit 1 -fi - -JOB=$1 - -# ensure this is at the root -cd "$SCRIPTPATH/.." - -# hardcoded path (workflows) -act -v -W workflows -j $JOB > tests/functional/output/$JOB.output 2>&1 -echo $? >> tests/functional/output/$JOB.output diff --git a/tests/README.md b/tests/README.md index edb971b3..00150375 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,27 +1,19 @@ # Developing tests -Tests are being implemented in javascript (and soon to be Typescript). -Some tests require a docker registry running locally on port 5000. This is handled -automatically in the Github action tests, -but if you want to run the tests yourself you will need to have docker installed -and run something like: +Some tests require a docker registry running locally on port 5000 as well as +some images built. ``` docker run -d -p 5000:5000 --name registry registry:2 -``` - -... or if you run `make test`, this is automatically handled for you. After -which time, you can just run `npm` directly: -``` -npm test +for distro in alpine centos debian; do + docker build -t localhost:5000/match-coverage/$distro ./tests/fixtures/image-$distro-match-coverage + docker push localhost:5000/match-coverage/$distro:latest +done ``` -Some of the existing tests are written in Python 3 and will -download [act](https://github.com/nektos/act) and create a Python virtual -environment to run them in. To run these locally, from the root directory execute: +Then, just run: ``` -npm run build -make check +npm test ``` diff --git a/tests/action.test.js b/tests/action.test.js index a0d4cd2d..3a90ca01 100644 --- a/tests/action.test.js +++ b/tests/action.test.js @@ -3,21 +3,42 @@ const os = require("os"); const path = require("path"); const process = require("process"); -const actionPath = path.join(__dirname, "../index.js"); +const actionPath = path.join(__dirname, "../dist/index.js"); // Execute the action, and return any outputs function runAction(inputs) { + // Set up the environment variables + const env = { + PATH: process.env.PATH, + RUNNER_TEMP: process.env.RUNNER_TEMP, + RUNNER_TOOL_CACHE: process.env.RUNNER_TOOL_CACHE, + }; // reverse core.js: const val = process.env[`INPUT_${name.replace(/ /g, '_').toUpperCase()}`] || ''; for (const k in inputs) { - process.env[`INPUT_${k}`.toUpperCase()] = inputs[k]; + // NOTE: there is a bug with node exec where environment variables with dashes + // are not always preserved - we will just have to rely on defaults in the code + env[`INPUT_${k}`.toUpperCase()] = inputs[k]; } - // capture stdout - const stdout = child_process - .execSync(`node ${actionPath}`, { - env: process.env, - }) - .toString("utf8"); - const outputs = {}; + + // capture stdout and exit code, and execute the command + let exitCode = 0; + let stdout; + try { + stdout = child_process + .execSync(`node ${actionPath}`, { + env, + }) + .toString("utf8"); + } catch (error) { + exitCode = error.status; + stdout = error.stdout.toString("utf8"); + } + + const outputs = { + exitCode, + stdout, + }; + // reverse setOutput command calls like: // ::set-output name=cmd::/tmp/actions/cache/grype/0.34.4/x64/grype for (const line of stdout.split(os.EOL)) { @@ -26,14 +47,34 @@ function runAction(inputs) { outputs[groups[1]] = groups[2]; } } + return outputs; } -describe("sbom-action", () => { +describe("scan-action", () => { it("runs download-grype", () => { const outputs = runAction({ run: "download-grype", }); expect(outputs.cmd).toBeDefined(); }); + + it("errors with invalid input", () => { + const outputs = runAction({ + image: "some-image", + path: "some-path", + }); + expect(outputs.exitCode).toBe(1); + expect(outputs.stdout).toContain( + "Cannot use both 'image' and 'path' as sources" + ); + expect(outputs.stdout).not.toContain("grype"); + }); + + it("fails due to vulnerabilities found", () => { + const outputs = runAction({ + image: "localhost:5000/match-coverage/debian:latest", + }); + expect(outputs.stdout).toContain("Failed minimum severity level."); + }); }); diff --git a/tests/functional/output/.keep b/tests/functional/output/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/functional/test_images.py b/tests/functional/test_images.py deleted file mode 100644 index 64af53e6..00000000 --- a/tests/functional/test_images.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -import pytest - -@pytest.fixture(scope="module") -def image_output(): - dirname = os.path.dirname(os.path.abspath(__file__)) - output_file = os.path.join(dirname, 'output/image.output') - with open(output_file) as _f: - return _f.read() - - -class TestSmoke: - - # basic validation - def test_zero_exit_status(self, image_output): - lines = image_output.split() - fail_context = '\n'.join(image_output.split('\n')[-20:]) - assert lines[-1] == '0', fail_context - - def test_found_vulnerabilities(self, image_output): - assert "Failed minimum severity level. Found vulnerabilities with level medium or higher" in image_output diff --git a/tests/functional/test_invalid_input.py b/tests/functional/test_invalid_input.py deleted file mode 100644 index 6c512394..00000000 --- a/tests/functional/test_invalid_input.py +++ /dev/null @@ -1,61 +0,0 @@ -import os -import pytest - -@pytest.fixture(scope="module") -def invalid_output(): - dirname = os.path.dirname(os.path.abspath(__file__)) - output_file = os.path.join(dirname, 'output/invalid-input.output') - with open(output_file) as _f: - return _f.read() - - -class TestInvalidInput: - - # unfortunately, non-zero is not enough, we want to make sure that there - # is something preventing invalid input altogether - def test_nonzero_exit_status(self, invalid_output): - lines = invalid_output.split() - assert lines[-1] == '1' - - def test_vulns_arent_reported(self, invalid_output): - # nothing should really get reported from grype because the input is not good - lines = invalid_output.split('\n') - for line in lines: - assert "discovered vulnerabilities at or above the severity threshold" not in line - - def test_error_is_reported(self, invalid_output): - assert "Cannot use both 'image' and 'path' as sources" in invalid_output - - def test_grype_never_runs(self, invalid_output): - lines = invalid_output.split('\n') - for line in lines: - assert "Running cmd: grype -vv -o json" not in line - - -@pytest.fixture(scope="module") -def sources_output(): - dirname = os.path.dirname(os.path.abspath(__file__)) - output_file = os.path.join(dirname, 'output/no-sources.output') - with open(output_file) as _f: - return _f.read() - - -class TestNoSources: - - def test_nonzero_exit_status(self, sources_output): - lines = sources_output.split() - assert lines[-1] == '1' - - def test_vulns_arent_reported(self, sources_output): - # nothing should really get reported from grype because there are no sources to use - lines = sources_output.split('\n') - for line in lines: - assert "discovered vulnerabilities at or above the severity threshold" not in line - - def test_error_is_reported(self, sources_output): - assert "At least one source for scanning needs to be provided. Available options are: image, and path" in sources_output - - def test_grype_never_runs(self, sources_output): - lines = sources_output.split('\n') - for line in lines: - assert "Running cmd: grype -vv -o json" not in line diff --git a/tests/grype_command.test.js b/tests/grype_command.test.js index 26f3e519..a44caffa 100644 --- a/tests/grype_command.test.js +++ b/tests/grype_command.test.js @@ -23,13 +23,15 @@ const mockExec = async (args) => { }; describe("Grype command", () => { - it("is invoked with defaults", async () => { - let cmd = await mockExec({ source: "python:3.8" }); - expect(cmd).toBe("grype -o sarif --fail-on medium python:3.8"); - }); - it("is invoked with dir", async () => { - let cmd = await mockExec({ source: "dir:.", severityCutoff: "high" }); + let cmd = await mockExec({ + source: "dir:.", + debug: "false", + failBuild: "false", + acsReportEnable: "true", + severityCutoff: "high", + version: "0.6.0", + }); expect(cmd).toBe("grype -o sarif --fail-on high dir:."); }); diff --git a/tests/python/setup.py b/tests/python/setup.py deleted file mode 100644 index 6e78745f..00000000 --- a/tests/python/setup.py +++ /dev/null @@ -1,48 +0,0 @@ -# Sample setup.py from a project that should spit out vulnerabilities -# -import sys -if sys.version_info < (3,6): - sys.exit('Sorry, Python < 3.6 is not supported') -import os - -from setuptools import setup - -from devml import __version__ - -if os.path.exists('README.md'): - LONG = open('README.md').read() -else: - LONG = '' - -setup( - name='devml', - version=__version__, - url='https://github.com/noahgift/devml', - license='MIT', - author='Noah Gift', - author_email='consulting@noahgift.com', - description="""Machine Learning, Statistics and Utilities around Developer Productivity, - Company Productivity and Project Productivity""", - long_description=LONG, - packages=['devml'], - include_package_data=True, - zip_safe=False, - platforms='any', - install_requires=[ - 'pandas', 'click==1.1.1', 'PyGithub', - 'click==1.1.1', - 'gitpython', - 'sensible', - 'scipy', - 'numpy', - ], - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3.6', - 'Topic :: Software Development :: Libraries :: Python Modules' - ], - scripts=["dml"], -) diff --git a/tests/sarif_output.test.js b/tests/sarif_output.test.js index 80d65869..8358b7fd 100644 --- a/tests/sarif_output.test.js +++ b/tests/sarif_output.test.js @@ -15,8 +15,10 @@ const testSource = async (source, vulnerabilities) => { const out = await runScan({ source, + debug: "false", failBuild: "false", acsReportEnable: "true", + severityCutoff: "medium", }); // expect to get sarif output diff --git a/workflows/image.yml b/workflows/image.yml deleted file mode 100644 index 6e9d25a9..00000000 --- a/workflows/image.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: "Test scan-action with images" - -on: [push] - -jobs: - localbuild: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - path: "scan-action" - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: build flask app - uses: docker/build-push-action@v2 - with: - context: ./scan-action/tests/fixtures/localbuild/ - tags: localbuild/testimage:latest - push: false - load: true - - - name: Use scan-action to check localbuild - uses: ./ - with: - image: "localbuild/testimage:latest" - fail-build: false diff --git a/workflows/tests.yml b/workflows/tests.yml deleted file mode 100644 index 83249d07..00000000 --- a/workflows/tests.yml +++ /dev/null @@ -1,129 +0,0 @@ -name: "[Demo] Run Scan Action" - -on: [push] - -jobs: - image: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - path: "scan-action" - - - uses: ./ - with: - image: "python:3.8" - debug: true - fail-build: false - - no-sources: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - path: "scan-action" - - - uses: ./ - with: - debug: true - fail-build: false - - invalid-input: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - path: "scan-action" - - - uses: ./ - with: - image: "python:3.8" - path: "/some/path" - debug: true - fail-build: false - -# XXX Port these to get verified with tests -# image-fail-build: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v2 -# with: -# path: "scan-action" -# -# - uses: ./ -# with: -# image: "python:3.8" -# debug: true -# fail-build: true -# -# image-severity: -# runs-on: ubuntu-latest -# steps: -# - uses: ./ -# with: -# path: "scan-action" -# -# # Do not fail because `fail-build` is unset (defaults to false) -# - uses: anchore/scan-action@main -# with: -# image: "python:3.8" -# debug: true -# severity-cutoff: Medium -# -# image-severity-fail-build: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v2 -# -# # Always testing the main branch -# - uses: anchore/scan-action@main -# with: -# image: "python:3.8" -# debug: true -# fail-build: true -# severity-cutoff: Medium -# -# test-directory: -# runs-on: ubuntu-latest -# steps: -# - uses: ./ -# with: -# path: "tests/python" -# debug: true -# severity-cutoff: "Negligible" -# -# directory-fail-build: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v2 -# -# - uses: ./ -# with: -# path: "tests/python" -# debug: true -# fail-build: true -# -# directory-severity: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v2 -# -# # Do not fail because `fail-build` is unset (defaults to false) -# - uses: anchore/scan-action@main -# with: -# path: "tests/python" -# debug: true -# severity-cutoff: Medium -# -# directory-severity-fail-build: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v2 -# -# # Always testing the main branch -# - uses: anchore/scan-action@main -# with: -# path: "tests/python" -# debug: true -# fail-build: true -# severity-cutoff: Medium