Skip to content

Commit

Permalink
reuse workflows in repos tests
Browse files Browse the repository at this point in the history 
and add statistics output to test workflow
  • Loading branch information
@bernt-matthias
bernt-matthias committed Feb 9, 2023
1 parent 0f88e2b commit c4ac8b4
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 167 deletions.
221 changes: 55 additions & 166 deletions .github/workflows/tools.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,200 +85,89 @@ jobs:

setup-ci-tools:
name: Setup as in CI for tools
if: ${{ github.repository_owner == 'galaxyproject' }}
uses: ./.github/workflows/wf_setup.yaml@v1
with:
# default-galaxy-fork: galaxyproject
# default-galaxy-branch: release_22.05
max-chunks: 40
secrets:
PAT: ${{ secrets.PAT }}

check-setup-ci-tools:
name: Check setup
needs: setup-ci-tools
runs-on: ubuntu-latest
outputs:
galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
fork: ${{ steps.get-fork-branch.outputs.fork }}
branch: ${{ steps.get-fork-branch.outputs.branch }}
repository-list: ${{ steps.discover.outputs.repository-list }}
chunk-count: ${{ steps.discover.outputs.chunk-count }}
chunk-list: ${{ steps.discover.outputs.chunk-list }}
strategy:
matrix:
python-version: ['3.7']
steps:
- name: Determine latest commit in the Galaxy repo
id: get-galaxy-sha
run: echo "galaxy-head-sha=$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" >> $GITHUB_OUTPUT
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache .cache/pip
uses: actions/cache@v3
id: cache-pip
with:
path: ~/.cache/pip
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
# Install the `wheel` package so that when installing other packages which
# are not available as wheels, pip will build a wheel for them, which can be cached.
- name: Install wheel
run: pip install wheel
- uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Artificially exclude workflows (because ci_find_repos discovers tool and workflow repos)
run: echo 'test/workflows/' > .tt_skip
- name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks
uses: ./
id: discover
env:
GITHUB_EVENT_NAME: schedule
with:
create-cache: true
galaxy-fork: ${{ env.GALAXY_FORK }}
galaxy-branch: ${{ env.GALAXY_BRANCH }}
max-chunks: ${{ env.MAX_CHUNKS }}
python-version: ${{ matrix.python-version }}
github-event-name-override: 'schedule'
- name: Check for empty commit range
run: if [ -n "${{ steps.discover.outputs.commit-range }}" ]; then exit 1; fi
run: if [ -n "${{ needs.setup-ci-tools.outputs.commit-range }}" ]; then exit 1; fi
# note: ci_find_repos will find all tools and workflows
- name: Check that all tools an workflows are in the repository list
run: |
if ! grep -q "tool1" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "tool1 must be in the repo list"; exit 1; fi
if ! grep -q "tool2" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "tool2 must be in the repo list"; exit 1; fi
if grep -q "example3" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "example3 must not be in the repo list"; exit 1; fi
if grep -q "example4" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "example4 must not be in the repo list"; exit 1; fi
if ! grep -q "tool1" <<<$(echo "${{ needs.setup-ci-tools.outputs.repository-list }}"); then echo "tool1 must be in the repo list"; exit 1; fi
if ! grep -q "tool2" <<<$(echo "${{ needs.setup-ci-tools.outputs.repository-list }}"); then echo "tool2 must be in the repo list"; exit 1; fi
if grep -q "example3" <<<$(echo "${{ needs.setup-ci-tools.outputs.repository-list }}"); then echo "example3 must not be in the repo list"; exit 1; fi
if grep -q "example4" <<<$(echo "${{ needs.setup-ci-tools.outputs.repository-list }}"); then echo "example4 must not be in the repo list"; exit 1; fi
# note: ci_find_tools will find onlytools
- name: Check that all tools are in the tool list
run: |
if ! grep -q "tool1.xml" <<<$(echo "${{ steps.discover.outputs.tool-list }}"); then echo "tool1.xml must be in the repo list"; exit 1; fi
if ! grep -q "tool2.xml" <<<$(echo "${{ steps.discover.outputs.tool-list }}"); then echo "tool2.xml must be in the repo list"; exit 1; fi
if ! grep -q "tool1.xml" <<<$(echo "${{ needs.setup-ci-tools.outputs.tool-list }}"); then echo "tool1.xml must be in the repo list"; exit 1; fi
if ! grep -q "tool2.xml" <<<$(echo "${{ needs.setup-ci-tools.outputs.tool-list }}"); then echo "tool2.xml must be in the repo list"; exit 1; fi
# chunk-count will only consider tools/workflows depending on the workflow input to the action
- name: Check that the number of chunks is three, i.e. the number of tools
run: if [ "${{ steps.discover.outputs.chunk-count }}" != "3" ]; then exit 1; fi
run: if [ "${{ needs.setup-ci-tools.outputs.chunk-count }}" != "3" ]; then exit 1; fi

lint:
name: Test linting of tools
needs: [setup-ci-tools]
name: Lint
needs: setup-ci-tools
uses: ./.github/workflows/wf_lint.yaml
with:
repository-list: ${{ needs.setup-ci-tools.outputs.repository-list }}
tool-list: ${{ needs.setup-ci-tools.outputs.tool-list }}
galaxy-head-sha: ${{ needs.setup-ci-tools.outputs.galaxy-head-sha }}
commit-range: ${{ needs.setup-ci-tools.outputs.commit-range }}
planemo-version: ${{ needs.setup-ci-tools.outputs.planemo-version }}
planemo-ci-action: ./

check-lint:
name: Check linting results
needs: lint
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['3.7']
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 1
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache .cache/pip
uses: actions/cache@v3
id: cache-pip
with:
path: ~/.cache/pip
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup-ci-tools.outputs.galaxy-head-sha }}
- name: Install wheel
run: pip install wheel
- name: Planemo lint tools
uses: ./
steps:
- uses: actions/download-artifact@v3
with:
mode: lint
report-level: all
fail-level: error
repository-list: ${{ needs.setup-ci-tools.outputs.repository-list }}
tool-list: ${{ needs.setup-ci-tools.outputs.tool-list }}
additional-planemo-options: --report_level all
continue-on-error: true


path: lint_report.txt
- name: check if all test tools were linted
run: |
grep tool1 lint_report.txt
grep tool2 lint_report.txt
grep "ERROR: Error 'HTTPConnectionPool" lint_report.txt
grep "Applying linter tests... CHECK" lint_report.txt
test-tools:
name: Test testing of tools
needs: [setup-ci-tools]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
chunk: ${{ fromJson(needs.setup-ci-tools.outputs.chunk-list) }}
python-version: ['3.7']
services:
postgres:
image: postgres:11
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
ports:
- 5432:5432
steps:
# checkout the repository
# and use it as the current working directory
- uses: actions/checkout@v3
with:
fetch-depth: 1
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache .cache/pip
uses: actions/cache@v3
id: cache-pip
with:
path: ~/.cache/pip
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup-ci-tools.outputs.galaxy-head-sha }}
- name: Get number of CPU cores
uses: SimenB/github-actions-cpu-cores@v1
id: cpu-cores
- name: Planemo test tools
uses: ./
id: test-tools
with:
mode: test
repository-list: ${{ needs.setup-ci-tools.outputs.repository-list }}
galaxy-fork: ${{ env.GALAXY_FORK }}
galaxy-branch: ${{ env.GALAXY_BRANCH }}
chunk: ${{ matrix.chunk }}
chunk-count: ${{ needs.setup-ci-tools.outputs.chunk-count }}
additional-planemo-options: --simultaneous_uploads --check_uploads_ok
galaxy-slots: ${{ steps.cpu-cores.outputs.count }}
- uses: actions/upload-artifact@v3
with:
name: 'Tool test output ${{ matrix.chunk }}'
path: upload
name: Test tools
needs: setup-ci-tools
uses: ./.github/workflows/wf_test.yaml
with:
galaxy-fork: ${{ needs.setup.outputs.galaxy-fork }}
galaxy-branch: ${{ needs.setup.outputs.galaxy-branch }}
planemo-version: ${{ needs.setup.outputs.planemo-version }}
repository-list: ${{ needs.setup.outputs.repository-list }}
chunk-count: ${{ needs.setup.outputs.chunk-count }}
chunk-list: ${{ needs.setup.outputs.chunk-list }}
galaxy-head-sha: ${{ needs.setup.outputs.galaxy-head-sha }}
# set these values if you want to use different values than the IUC defaults
# test-timeout: 900
planemo-ci-action: ./

combine_outputs:
name: Test combining 'chunked' test results
check-test:
name: Check testing results
needs: [setup-ci-tools, test-tools]
strategy:
matrix:
python-version: ['3.7']
# This job runs on Linux
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 1
- uses: actions/download-artifact@v3
with:
path: artifacts
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache .cache/pip
uses: actions/cache@v3
id: cache-pip
with:
path: ~/.cache/pip
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup-ci-tools.outputs.galaxy-head-sha }}
- name: Combine outputs
uses: ./
id: combine
with:
mode: combine
html-report: true
- name: Check statistics
run: if ! grep -q "3\s\+success" <<<$(echo ${{ steps.combine.outputs.statistics }}); then echo "wrong statistics"; exit 1; fi
run: if ! grep -q "3\s\+success" <<<$(echo ${{ needs.test-tools.outputs.statistics }}); then echo "wrong statistics"; exit 1; fi
- uses: actions/upload-artifact@v3
with:
name: 'All tool test results'
path: upload
- name: Check outputs
uses: ./
id: check
with:
mode: check
6 changes: 5 additions & 1 deletion .github/workflows/wf_test.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,9 @@ on:
result:
description: "Result of the testing"
value: ${{ jobs.test-success.result }}

statistics:
description: "Statistics of the test results"
value: "${{ jobs.combine_outputs.outputs.statistics }}"
jobs:
# Planemo test the changed repositories, each chunk creates an artifact
# containing HTML and JSON reports for the executed tests
Expand Down Expand Up @@ -133,6 +135,8 @@ jobs:
name: Combine chunked test results
needs: test
runs-on: ubuntu-latest
outputs:
galaxy-fork: ${{ steps.combine.outputs.statistics }}
strategy:
matrix:
python-version: ${{ fromJson(inputs.python-version-list) }}
Expand Down

0 comments on commit c4ac8b4

Please sign in to comment.