From 1425f67b55923173547c821d6332c3eb1d6bfef3 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sat, 28 Mar 2020 20:36:26 +0100 Subject: [PATCH 001/142] github actions for push and PR --- .github/workflows/pr.yaml | 549 ++++++++++++-------------------------- 1 file changed, 165 insertions(+), 384 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 85cee20c..3a6fe573 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -1,284 +1,119 @@ name: Galaxy Tool Linting and Tests for push and PR -on: - pull_request: - paths-ignore: - - '.github/**' - - 'deprecated/**' - - 'docs/**' - - '*' - push: - branches: - - main - - master - paths-ignore: - - '.github/**' - - 'deprecated/**' - - 'docs/**' - - '*' +on: [push, pull_request] env: - GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_23.1 - MAX_CHUNKS: 4 - MAX_FILE_SIZE: 1M -concurrency: - # Group runs by PR, but keep runs on the default branch separate - # because we do not want to cancel ToolShed uploads - group: pr-${{ (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main') && github.run_number || github.ref }} - cancel-in-progress: true + GALAXY_REPO: https://github.com/galaxyproject/galaxy + GALAXY_RELEASE: release_20.01 jobs: - # the setup job does two things: - # 1. cache the pip cache and .planemo - # 2. determine the list of changed repositories - # it produces one artifact which contains - # - a file with the latest SHA from the chosen branch of the Galaxy repo - # - a file containing the list of changed repositories - # which are needed in subsequent steps. setup: - name: Setup cache and determine changed repositories + name: Cache planemo setup runs-on: ubuntu-latest - outputs: - galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} - repository-list: ${{ steps.discover.outputs.repository-list }} - tool-list: ${{ steps.discover.outputs.tool-list }} - chunk-count: ${{ steps.discover.outputs.chunk-count }} - chunk-list: ${{ steps.discover.outputs.chunk-list }} - commit-range: ${{ steps.discover.outputs.commit-range }} strategy: matrix: - python-version: ['3.7'] + python-version: [3.7] steps: - - name: Print github context properties - run: | - echo 'event: ${{ github.event_name }}' - echo 'sha: ${{ github.sha }}' - echo 'ref: ${{ github.ref }}' - echo 'head_ref: ${{ github.head_ref }}' - echo 'base_ref: ${{ github.base_ref }}' - echo 'event.before: ${{ github.event.before }}' - echo 'event.after: ${{ github.event.after }}' - - name: Determine latest commit in the Galaxy repo - id: get-galaxy-sha - run: echo "galaxy-head-sha=$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" >> $GITHUB_OUTPUT - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + architecture: 'x64' # (x64 or x86) + - name: setup planemo + run: 'pip install planemo' + - name: fake a planemo run to update cache + run: 'touch tool.xml && PIP_QUIET=1 planemo test --galaxy_python_version ${{ matrix.python-version }} --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE' - name: Cache .cache/pip - uses: actions/cache@v3 + uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} - - name: Cache .planemo - uses: actions/cache@v3 - id: cache-planemo - with: - path: ~/.planemo - key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} - # Install the `wheel` package so that when installing other packages which - # are not available as wheels, pip will build a wheel for them, which can be cached. - - name: Install wheel - run: pip install wheel - - name: Install flake8 - run: pip install flake8 flake8-import-order - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks - uses: galaxyproject/planemo-ci-action@v1 - id: discover - with: - create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }} - galaxy-fork: ${{ env.GALAXY_FORK }} - galaxy-branch: ${{ env.GALAXY_BRANCH }} - max-chunks: ${{ env.MAX_CHUNKS }} - python-version: ${{ matrix.python-version }} - - name: Show commit range - run: echo '${{ steps.discover.outputs.commit-range }}' - - name: Show repository list - run: echo '${{ steps.discover.outputs.repository-list }}' - - name: Show tool list - run: echo '${{ steps.discover.outputs.tool-list }}' - - name: Show chunks - run: | - echo 'Using ${{ steps.discover.outputs.chunk-count }} chunks (${{ steps.discover.outputs.chunk-list }})' + key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - # Planemo lint the changed repositories lint: - name: Lint tool-list + name: lint tools needs: setup - if: ${{ needs.setup.outputs.repository-list != '' || needs.setup.outputs.tool-list != '' }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ['3.7'] + python-version: [3.7] steps: - - uses: actions/checkout@v3 + # checkout the repository to master + # and use it as the current working directory + - uses: actions/checkout@v1 with: fetch-depth: 1 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + architecture: 'x64' # (x64 or x86) - name: Cache .cache/pip - uses: actions/cache@v3 + uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - - name: Set fail level for pull request - if: ${{ github.event_name == 'pull_request' }} - run: - echo "FAIL_LEVEL=warn" >> "$GITHUB_ENV" - - name: Set fail level for merge - if: ${{ github.event_name != 'pull_request' }} - run: - echo "FAIL_LEVEL=error" >> "$GITHUB_ENV" - - name: Planemo lint - uses: galaxyproject/planemo-ci-action@v1 - id: lint - with: - mode: lint - fail-level: ${{ env.FAIL_LEVEL }} - repository-list: ${{ needs.setup.outputs.repository-list }} - tool-list: ${{ needs.setup.outputs.tool-list }} - - uses: actions/upload-artifact@v3 - if: ${{ failure() }} - with: - name: 'Tool linting output' - path: lint_report.txt + key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + - name: setup planemo + run: 'pip install planemo' + - name: planemo ci_find_repos + run: | + mkdir changed_repositories + planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list + - name: planemo lint + run: | + set -e + while read -r DIR + do + planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; + done < changed_repositories.list - # flake8 of Python scripts in the changed repositories - flake8: - name: Lint Python scripts + pylint: + name: lint python scripts needs: setup - if: ${{ github.event_name == 'pull_request' && needs.setup.outputs.repository-list != '' }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ['3.7'] + python-version: [3.7] steps: - - uses: actions/checkout@v3 + # checkout the repository to master + # and use it as the current working directory + - uses: actions/checkout@v1 with: fetch-depth: 1 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + architecture: 'x64' # (x64 or x86) - name: Cache .cache/pip - uses: actions/cache@v3 + uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Install flake8 - run: pip install flake8 flake8-import-order - - name: Flake8 - run: echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' flake8 --output-file pylint_report.txt --tee - - uses: actions/upload-artifact@v3 - if: ${{ failure() }} - with: - name: 'Python linting output' - path: pylint_report.txt - - lintr: - name: Lint R scripts - needs: setup - if: ${{ github.event_name == 'pull_request' && needs.setup.outputs.repository-list != '' }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-20.04] - r-version: ['release'] - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - uses: r-lib/actions/setup-r@v2 - with: - r-version: ${{ matrix.r-version }} - - name: Cache R packages - uses: actions/cache@v3 - with: - path: ${{ env.R_LIBS_USER }} - key: r_cache_${{ matrix.os }}_${{ matrix.r-version }} - - name: Install non-R lintr dependencies - run: sudo apt-get install libcurl4-openssl-dev - - name: Install lintr - run: | - install.packages('remotes') - remotes::install_cran("lintr") - shell: Rscript {0} - - name: Save repositories to file - run: echo '${{ needs.setup.outputs.repository-list }}' > repository_list.txt - - name: lintr + key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + - name: setup planemo + run: 'pip install planemo' + - name: planemo ci_find_repos run: | - library(lintr) - linters <- linters_with_defaults(line_length_linter = NULL, cyclocomp_linter = NULL, object_usage_linter = NULL, object_name_linter = NULL) - con <- file("repository_list.txt", "r") - status <- 0 - while (TRUE) { - repo <- readLines(con, n = 1) - if (length(repo) == 0) { - break - } - lnt <- lint_dir(repo, relative_path=T, linters=linters) - if (length(lnt) > 0) { - status <- 1 - for (l in lnt) { - rel_path <- paste(repo, l$filename, sep="/") - write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message, paste("(", l$line, ")")), stderr()) - write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message, paste("(", l$line, ")")), "rlint_report.txt", append=TRUE) - } - } - } - quit(status = status) - shell: Rscript {0} - - uses: actions/upload-artifact@v3 - if: ${{ failure() }} - with: - name: 'R linting output' - path: rlint_report.txt - - file_sizes: - name: Check file sizes - needs: setup - if: ${{ github.event_name == 'pull_request' && needs.setup.outputs.repository-list != '' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Check file sizes + mkdir changed_repositories + planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list + - name: setup pycodestyle + run: 'pip install pycodestyle' + - name: pycodestyle run: | - touch file_size_report.txt - git diff --diff-filter=d --name-only ${{ needs.setup.outputs.commit-range }} > git.diff - while read line; do - find "$line" -type f -size +${{ env.MAX_FILE_SIZE }} >> file_size_report.txt - done < git.diff - if [[ -s file_size_report.txt ]]; then - echo "Files larger than ${{ env.MAX_FILE_SIZE }} found" - cat file_size_report.txt - exit 1 - fi - - uses: actions/upload-artifact@v3 - if: ${{ failure() }} - with: - name: 'File size report' - path: file_size_report.txt + set -e + while read -r DIR + do + pycodestyle --ignore E501 "$DIR"; + done < changed_repositories.list - # Planemo test the changed repositories, each chunk creates an artifact - # containing HTML and JSON reports for the executed tests test: - name: Test tools - needs: setup - if: ${{ needs.setup.outputs.repository-list != '' }} + name: test tools + # This job runs on Linux runs-on: ubuntu-latest + needs: [lint,pylint] strategy: fail-fast: false matrix: - chunk: ${{ fromJson(needs.setup.outputs.chunk-list) }} - python-version: ['3.7'] + chunk: [0,1,2,3] + python-version: [3.7] services: postgres: image: postgres:11 @@ -289,173 +124,119 @@ jobs: ports: - 5432:5432 steps: - - uses: actions/checkout@v3 + # checkout the repository to master + # and use it as the current working directory + - uses: actions/checkout@v1 with: fetch-depth: 1 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + architecture: 'x64' # (x64 or x86) - name: Cache .cache/pip - uses: actions/cache@v3 + uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Cache .planemo - uses: actions/cache@v3 - id: cache-planemo - with: - path: ~/.planemo - key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v1 - id: cpu-cores - - name: Clean dotnet folder for space - run: rm -Rf /usr/share/dotnet - - name: Planemo test - uses: galaxyproject/planemo-ci-action@v1 - id: test - with: - mode: test - repository-list: ${{ needs.setup.outputs.repository-list }} - galaxy-fork: ${{ env.GALAXY_FORK }} - galaxy-branch: ${{ env.GALAXY_BRANCH }} - chunk: ${{ matrix.chunk }} - chunk-count: ${{ needs.setup.outputs.chunk-count }} - galaxy-slots: ${{ steps.cpu-cores.outputs.count }} - # Limit each test to 15 minutes - test_timeout: 1800 - - uses: actions/upload-artifact@v3 - with: - name: 'Tool test output ${{ matrix.chunk }}' + key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + - name: setup planemo + run: 'pip install planemo' + - name: planemo ci_find_repos + run: | + mkdir changed_repositories + planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list + - name: planemo ci_find_tools + run: | + touch changed_repositories_chunk.list changed_tools_chunk.list + if [ -s changed_repositories.list ]; then + if [ $(wc -l < changed_repositories.list) -eq 1 ]; then + planemo ci_find_tools --chunk_count 4 --chunk ${{ matrix.chunk }} \ + --output changed_tools_chunk.list \ + $(cat changed_repositories.list) + else + planemo ci_find_repos --chunk_count 4 --chunk ${{ matrix.chunk }} \ + --output changed_repositories_chunk.list \ + $(cat changed_repositories.list) + fi + fi + - name: set tool list + id: tools + run: echo "::set-output name=tool_list::$(tr '\n' ' ' < changed_repositories.list)" + - name: show repo list + run: 'cat changed_repositories.list' + - name: show changed tools/repositories chunk list + run: 'cat changed_tools_chunk.list changed_repositories_chunk.list' + - name: planemo test tools + run: | + if [ -s changed_tools_chunk.list ]; then + PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --biocontainers --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json tool_test_output.json $(cat changed_tools_chunk.list) || true + docker system prune --all --force --volumes || true + elif [ -s changed_repositories_chunk.list ]; then + while read -r DIR; do + if [[ "$DIR" =~ ^data_managers.* ]]; then + TESTPATH=$(planemo ci_find_tools "$DIR") + else + TESTPATH="$DIR" + fi + PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --biocontainers --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$DIR"/tool_test_output.json "$TESTPATH" || true + docker system prune --all --force --volumes || true + done < changed_repositories_chunk.list + else + echo '{"tests":[]}' > tool_test_output.json + fi + - name: Merge tool_test_output.json files + run: | + find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + + - name: Create tool_test_output.html + run: 'planemo test_reports tool_test_output.json --test_output tool_test_output.html' + - name: Copy artifacts into place + run: mkdir upload && mv tool_test_output.json tool_test_output.html upload + - uses: actions/upload-artifact@v1 + with: + name: 'Tool test output ${{ matrix.chunk }}' path: upload - # - combine the results of the test chunks (which will never fail due - # to `|| true`) and create a global test report as json and html which - # is provided as artifact - # - check if any tool test actually failed (by lookup in the combined json) - # and fail this step if this is the case combine_outputs: name: Combine chunked test results - needs: [setup, test] - if: ${{ always() && needs.setup.outputs.repository-list != '' }} - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.7'] - steps: - - uses: actions/download-artifact@v3 - with: - path: artifacts - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Cache .cache/pip - uses: actions/cache@v3 - id: cache-pip - with: - path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Combine outputs - uses: galaxyproject/planemo-ci-action@v1 - id: combine - with: - mode: combine - html-report: true - - uses: actions/upload-artifact@v3 - with: - name: 'All tool test results' - path: upload - - name: Check outputs - uses: galaxyproject/planemo-ci-action@v1 - id: check - with: - mode: check - - name: Check if all test chunks succeeded - run: | - NFILES=$(ls artifacts/ | grep "Tool test output" | wc -l) - if [[ "${{ needs.setup.outputs.chunk-count }}" != "$NFILES" ]]; then - exit 1 - fi - - # deploy the tools to the toolsheds (first TTS for testing) - deploy: - name: Deploy - needs: [setup, lint, combine_outputs] - if: ${{ (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) && github.repository_owner == 'galaxyproject' }} - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.7'] - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Cache .cache/pip - uses: actions/cache@v3 - id: cache-pip - with: - path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Deploy on testtoolshed - uses: galaxyproject/planemo-ci-action@v1 - with: - mode: deploy - repository-list: ${{ needs.setup.outputs.repository-list }} - shed-target: testtoolshed - shed-key: ${{ secrets.TTS_API_KEY }} - continue-on-error: true - - name: Deploy on toolshed - uses: galaxyproject/planemo-ci-action@v1 - with: - mode: deploy - repository-list: ${{ needs.setup.outputs.repository-list }} - shed-target: toolshed - shed-key: ${{ secrets.TS_API_KEY }} - - deploy-report: - name: Report deploy status - needs: [deploy] - if: ${{ always() && needs.deploy.result != 'success' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) && github.repository_owner == 'galaxyproject' }} - runs-on: ubuntu-latest - steps: - # report to the PR if deployment failed - - name: Get PR object - uses: 8BitJonny/gh-get-current-pr@2.2.0 - id: getpr - with: - sha: ${{ github.event.after }} - - name: Create comment - uses: peter-evans/create-or-update-comment@v2 - with: - token: ${{ secrets.PAT }} - issue-number: ${{ steps.getpr.outputs.number }} - body: | - Attention: deployment ${{ needs.deploy.result }}! - - https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - - determine-success: - name: Check workflow success - needs: [setup, lint, flake8, lintr, file_sizes, combine_outputs] - if: ${{ always() && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/main' }} + needs: test + # This job runs on Linux runs-on: ubuntu-latest steps: - - name: Check tool lint status - if: ${{ needs.lint.result != 'success' && needs.flake8.result != 'skipped' }} - run: exit 1 - - name: Indicate Python script lint status - if: ${{ needs.flake8.result != 'success' && needs.flake8.result != 'skipped' }} - run: exit 1 - - name: Indicate R script lint status - if: ${{ needs.lintr.result != 'success' && needs.lintr.result != 'skipped' }} - run: exit 1 - - name: Indicate file size check status - if: ${{ needs.file_sizes.result != 'success' && needs.file_sizes.result != 'skipped' }} - run: exit 1 - - name: Check tool test status - if: ${{ needs.combine_outputs.result != 'success' && needs.combine_outputs.result != 'skipped' }} - run: exit 1 + - uses: actions/download-artifact@v1 + with: + name: Tool test output 0 + - uses: actions/download-artifact@v1 + with: + name: Tool test output 1 + - uses: actions/download-artifact@v1 + with: + name: Tool test output 2 + - uses: actions/download-artifact@v1 + with: + name: Tool test output 3 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + architecture: 'x64' + - name: setup planemo + run: 'pip install planemo' + - name: setup jq + run: 'sudo apt-get install jq' + - name: combine outputs + run: | + find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + + - name: Create tool_test_output.html + run: 'planemo test_reports tool_test_output.json --test_output tool_test_output.html' + - name: Copy artifacts into place + run: mkdir upload && mv tool_test_output.json tool_test_output.html upload + - uses: actions/upload-artifact@v1 + with: + name: 'All tool test results' + path: upload + - name: check status of combined status + run: | + stat=$(jq '.["tests"][]["data"]["status"]' upload/tool_test_output.json | grep -v "success" || true) + if [[ "$(echo -n $stat | wc -l)" -gt "0" ]]; then + echo "unsuccessful tests found $(sort <<<$stat | uniq -c)" + exit 1 + fi From 2807b76a87f21a22f87876e10046517553979bc5 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 30 Mar 2020 11:08:48 +0200 Subject: [PATCH 002/142] add deployment job Co-Authored-By: Marius van den Beek --- .github/workflows/pr.yaml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3a6fe573..5c677378 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -240,3 +240,42 @@ jobs: echo "unsuccessful tests found $(sort <<<$stat | uniq -c)" exit 1 fi + + deploy: + name: Deploy + needs: combine_outputs + # This job runs on Linux + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/master' + steps: + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + architecture: 'x64' # (x64 or x86) + - name: Cache .cache/pip + uses: actions/cache@v1 + id: cache-pip + with: + path: ~/.cache/pip + key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + - name: setup planemo + run: 'pip install planemo' + - name: planemo ci_find_repos + run: | + mkdir changed_repositories + planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list + - name: deploy on testtoolshed + env: + SHED_KEY: ${{ secrets.tts_api_key }} + run: | + while read -r DIR; + do + planemo shed_update --shed_target testtoolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; + done < changed_repositories.list + - name: deploy on toolshed + env: + SHED_KEY: ${{ secrets.ts_api_key }} + run: | + while read -r DIR; + do planemo shed_update --shed_target toolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; + done < changed_repositories.list From 150d06b0a6254160244c737495857d6ea3b9cdb4 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 30 Mar 2020 19:15:26 +0200 Subject: [PATCH 003/142] Apply suggestions from code review reformatting and reformulations Co-Authored-By: Nicola Soranzo --- .github/workflows/pr.yaml | 78 +++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5c677378..eee41c6d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -5,7 +5,7 @@ env: GALAXY_RELEASE: release_20.01 jobs: setup: - name: Cache planemo setup + name: Setup Planemo runs-on: ubuntu-latest strategy: matrix: @@ -14,11 +14,12 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - architecture: 'x64' # (x64 or x86) - - name: setup planemo - run: 'pip install planemo' + - name: Install Planemo + run: pip install planemo - name: fake a planemo run to update cache - run: 'touch tool.xml && PIP_QUIET=1 planemo test --galaxy_python_version ${{ matrix.python-version }} --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE' + run: | + touch tool.xml + PIP_QUIET=1 planemo test --galaxy_python_version ${{ matrix.python-version }} --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip @@ -81,7 +82,6 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - architecture: 'x64' # (x64 or x86) - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip @@ -89,11 +89,11 @@ jobs: path: ~/.cache/pip key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - name: setup planemo - run: 'pip install planemo' + run: pip install planemo - name: planemo ci_find_repos run: | - mkdir changed_repositories - planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list + mkdir changed_repositories + planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list - name: setup pycodestyle run: 'pip install pycodestyle' - name: pycodestyle @@ -132,40 +132,39 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - architecture: 'x64' # (x64 or x86) - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - - name: setup planemo - run: 'pip install planemo' + - name: Install Planemo + run: pip install planemo - name: planemo ci_find_repos run: | - mkdir changed_repositories - planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list + mkdir changed_repositories + planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list - name: planemo ci_find_tools run: | - touch changed_repositories_chunk.list changed_tools_chunk.list - if [ -s changed_repositories.list ]; then - if [ $(wc -l < changed_repositories.list) -eq 1 ]; then - planemo ci_find_tools --chunk_count 4 --chunk ${{ matrix.chunk }} \ - --output changed_tools_chunk.list \ - $(cat changed_repositories.list) - else - planemo ci_find_repos --chunk_count 4 --chunk ${{ matrix.chunk }} \ - --output changed_repositories_chunk.list \ - $(cat changed_repositories.list) - fi + touch changed_repositories_chunk.list changed_tools_chunk.list + if [ -s changed_repositories.list ]; then + if [ $(wc -l < changed_repositories.list) -eq 1 ]; then + planemo ci_find_tools --chunk_count 4 --chunk ${{ matrix.chunk }} \ + --output changed_tools_chunk.list \ + $(cat changed_repositories.list) + else + planemo ci_find_repos --chunk_count 4 --chunk ${{ matrix.chunk }} \ + --output changed_repositories_chunk.list \ + $(cat changed_repositories.list) fi + fi - name: set tool list id: tools run: echo "::set-output name=tool_list::$(tr '\n' ' ' < changed_repositories.list)" - name: show repo list - run: 'cat changed_repositories.list' + run: cat changed_repositories.list - name: show changed tools/repositories chunk list - run: 'cat changed_tools_chunk.list changed_repositories_chunk.list' + run: cat changed_tools_chunk.list changed_repositories_chunk.list - name: planemo test tools run: | if [ -s changed_tools_chunk.list ]; then @@ -251,7 +250,6 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - architecture: 'x64' # (x64 or x86) - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip @@ -259,23 +257,23 @@ jobs: path: ~/.cache/pip key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - name: setup planemo - run: 'pip install planemo' + run: pip install planemo - name: planemo ci_find_repos run: | - mkdir changed_repositories - planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list + mkdir changed_repositories + planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list - name: deploy on testtoolshed env: - SHED_KEY: ${{ secrets.tts_api_key }} + SHED_KEY: ${{ secrets.tts_api_key }} run: | - while read -r DIR; - do - planemo shed_update --shed_target testtoolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; - done < changed_repositories.list + while read -r DIR; + do + planemo shed_update --shed_target testtoolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; + done < changed_repositories.list - name: deploy on toolshed env: - SHED_KEY: ${{ secrets.ts_api_key }} + SHED_KEY: ${{ secrets.ts_api_key }} run: | - while read -r DIR; - do planemo shed_update --shed_target toolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; - done < changed_repositories.list + while read -r DIR; + do planemo shed_update --shed_target toolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; + done < changed_repositories.list From 008f839fa022080937849db44cdc8d2c0ccee2d7 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 30 Mar 2020 19:20:47 +0200 Subject: [PATCH 004/142] switch to flake8 --- .github/workflows/pr.yaml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index eee41c6d..d295b2d4 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -65,7 +65,7 @@ jobs: planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; done < changed_repositories.list - pylint: + flake8: name: lint python scripts needs: setup runs-on: ubuntu-latest @@ -94,21 +94,16 @@ jobs: run: | mkdir changed_repositories planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list - - name: setup pycodestyle - run: 'pip install pycodestyle' - - name: pycodestyle - run: | - set -e - while read -r DIR - do - pycodestyle --ignore E501 "$DIR"; - done < changed_repositories.list + - name: setup flake8 + run: 'pip install flake8' + - name: flake8 + run: flake8 $(cat changed_repositories.list) test: name: test tools # This job runs on Linux runs-on: ubuntu-latest - needs: [lint,pylint] + needs: [lint,flake8] strategy: fail-fast: false matrix: From 64be57b1cf158c758637d19e9544aefe09966b3c Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 31 Mar 2020 17:53:34 +0200 Subject: [PATCH 005/142] Apply suggestions from code review Co-Authored-By: Nicola Soranzo --- .github/workflows/pr.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d295b2d4..5b3f1403 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -44,7 +44,6 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - architecture: 'x64' # (x64 or x86) - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip @@ -52,7 +51,7 @@ jobs: path: ~/.cache/pip key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - name: setup planemo - run: 'pip install planemo' + run: pip install planemo - name: planemo ci_find_repos run: | mkdir changed_repositories @@ -95,7 +94,7 @@ jobs: mkdir changed_repositories planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list - name: setup flake8 - run: 'pip install flake8' + run: pip install flake8 - name: flake8 run: flake8 $(cat changed_repositories.list) @@ -107,7 +106,7 @@ jobs: strategy: fail-fast: false matrix: - chunk: [0,1,2,3] + chunk: [0, 1, 2, 3] python-version: [3.7] services: postgres: @@ -182,7 +181,7 @@ jobs: run: | find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + - name: Create tool_test_output.html - run: 'planemo test_reports tool_test_output.json --test_output tool_test_output.html' + run: planemo test_reports tool_test_output.json --test_output tool_test_output.html - name: Copy artifacts into place run: mkdir upload && mv tool_test_output.json tool_test_output.html upload - uses: actions/upload-artifact@v1 @@ -211,29 +210,30 @@ jobs: - uses: actions/setup-python@v1 with: python-version: '3.7' - architecture: 'x64' - name: setup planemo - run: 'pip install planemo' + run: pip install planemo - name: setup jq - run: 'sudo apt-get install jq' + run: sudo apt-get install jq - name: combine outputs run: | find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + - name: Create tool_test_output.html - run: 'planemo test_reports tool_test_output.json --test_output tool_test_output.html' + run: planemo test_reports tool_test_output.json --test_output tool_test_output.html - name: Copy artifacts into place - run: mkdir upload && mv tool_test_output.json tool_test_output.html upload + run: | + mkdir upload + mv tool_test_output.json tool_test_output.html upload/ - uses: actions/upload-artifact@v1 with: name: 'All tool test results' path: upload - name: check status of combined status run: | - stat=$(jq '.["tests"][]["data"]["status"]' upload/tool_test_output.json | grep -v "success" || true) - if [[ "$(echo -n $stat | wc -l)" -gt "0" ]]; then - echo "unsuccessful tests found $(sort <<<$stat | uniq -c)" - exit 1 - fi + stat=$(jq '.["tests"][]["data"]["status"]' upload/tool_test_output.json | grep -v "success" || true) + if [[ "$(echo -n $stat | wc -l)" -gt "0" ]]; then + echo "unsuccessful tests found $(sort <<<$stat | uniq -c)" + exit 1 + fi deploy: name: Deploy @@ -244,7 +244,7 @@ jobs: steps: - uses: actions/setup-python@v1 with: - python-version: ${{ matrix.python-version }} + python-version: '3.7' - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip From 59105bfff679597bf4a23b9ee73c5a7d08f60741 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 31 Mar 2020 18:08:09 +0200 Subject: [PATCH 006/142] move caching up and check for cache miss --- .github/workflows/pr.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5b3f1403..13c87036 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,18 +14,20 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: Install Planemo - run: pip install planemo - - name: fake a planemo run to update cache - run: | - touch tool.xml - PIP_QUIET=1 planemo test --galaxy_python_version ${{ matrix.python-version }} --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + - name: Install Planemo + if: steps.cache-pip.outputs.cache-hit != 'true' + run: pip install planemo + - name: fake a planemo run to update cache + if: steps.cache-pip.outputs.cache-hit != 'true' + run: | + touch tool.xml + PIP_QUIET=1 planemo test --galaxy_python_version ${{ matrix.python-version }} --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE lint: name: lint tools From 8fb29ba9dd994de4fd691b1d0bc86753008f5be8 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 31 Mar 2020 22:09:17 +0200 Subject: [PATCH 007/142] generate changed_repositories.list once in setup and pass as artifact to subsequent jobs also store flake8 and its dependencies in the cache --- .github/workflows/pr.yaml | 193 +++++++++++++++++++++----------------- 1 file changed, 107 insertions(+), 86 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 13c87036..f09d3472 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -5,7 +5,7 @@ env: GALAXY_RELEASE: release_20.01 jobs: setup: - name: Setup Planemo + name: Setup cache and determine changed repositories runs-on: ubuntu-latest strategy: matrix: @@ -19,18 +19,29 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - - name: Install Planemo - if: steps.cache-pip.outputs.cache-hit != 'true' - run: pip install planemo + key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + - name: debug + run: echo ${{ steps.cache-pip.outputs.cache-hit }} + - name: install Planemo and flake8 + run: pip install planemo flake8 - name: fake a planemo run to update cache if: steps.cache-pip.outputs.cache-hit != 'true' run: | touch tool.xml PIP_QUIET=1 planemo test --galaxy_python_version ${{ matrix.python-version }} --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE + - uses: actions/checkout@v1 + with: + fetch-depth: 1 + - name: planemo ci_find_repos + run: | + planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list + - uses: actions/upload-artifact@v1 + with: + name: changed repositories + path: changed_repositories.list lint: - name: lint tools + name: Lint tools needs: setup runs-on: ubuntu-latest strategy: @@ -46,28 +57,28 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: Cache .cache/pip + - name: cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - name: setup planemo run: pip install planemo - - name: planemo ci_find_repos - run: | - mkdir changed_repositories - planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list + - uses: actions/download-artifact@v1 + with: + name: changed repositories + path: changed_repositories.list - name: planemo lint run: | set -e while read -r DIR do planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; - done < changed_repositories.list + done < changed_repositories.list/changed_repositories.list flake8: - name: lint python scripts + name: Lint python scripts needs: setup runs-on: ubuntu-latest strategy: @@ -83,22 +94,24 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: Cache .cache/pip + - name: cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - - name: setup planemo - run: pip install planemo - - name: planemo ci_find_repos - run: | - mkdir changed_repositories - planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list + key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - name: setup flake8 run: pip install flake8 + - uses: actions/download-artifact@v1 + with: + name: changed repositories + path: changed_repositories.list + - name: ls + run: ls -la + - name: ls + run: ls -la changed_repositories.list/changed_repositories.list - name: flake8 - run: flake8 $(cat changed_repositories.list) + run: flake8 $(cat changed_repositories.list/changed_repositories.list) test: name: test tools @@ -128,37 +141,38 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: Cache .cache/pip + - name: cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - - name: Install Planemo + key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + - name: install Planemo run: pip install planemo - - name: planemo ci_find_repos - run: | - mkdir changed_repositories - planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list + + - uses: actions/download-artifact@v1 + with: + name: changed repositories + path: changed_repositories.list - name: planemo ci_find_tools run: | touch changed_repositories_chunk.list changed_tools_chunk.list - if [ -s changed_repositories.list ]; then - if [ $(wc -l < changed_repositories.list) -eq 1 ]; then + if [ -s changed_repositories.list/changed_repositories.list ]; then + if [ $(wc -l < changed_repositories.list/changed_repositories.list) -eq 1 ]; then planemo ci_find_tools --chunk_count 4 --chunk ${{ matrix.chunk }} \ --output changed_tools_chunk.list \ - $(cat changed_repositories.list) + $(cat changed_repositories.list/changed_repositories.list) else planemo ci_find_repos --chunk_count 4 --chunk ${{ matrix.chunk }} \ --output changed_repositories_chunk.list \ - $(cat changed_repositories.list) + $(cat changed_repositories.list/changed_repositories.list) fi fi - name: set tool list id: tools - run: echo "::set-output name=tool_list::$(tr '\n' ' ' < changed_repositories.list)" + run: echo "::set-output name=tool_list::$(tr '\n' ' ' < changed_repositories.list/changed_repositories.list)" - name: show repo list - run: cat changed_repositories.list + run: cat changed_repositories.list/changed_repositories.list - name: show changed tools/repositories chunk list run: cat changed_tools_chunk.list changed_repositories_chunk.list - name: planemo test tools @@ -179,12 +193,12 @@ jobs: else echo '{"tests":[]}' > tool_test_output.json fi - - name: Merge tool_test_output.json files + - name: merge tool_test_output.json files run: | find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + - - name: Create tool_test_output.html + - name: create tool_test_output.html run: planemo test_reports tool_test_output.json --test_output tool_test_output.html - - name: Copy artifacts into place + - name: copy artifacts into place run: mkdir upload && mv tool_test_output.json tool_test_output.html upload - uses: actions/upload-artifact@v1 with: @@ -197,45 +211,52 @@ jobs: # This job runs on Linux runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v1 - with: - name: Tool test output 0 - - uses: actions/download-artifact@v1 - with: - name: Tool test output 1 - - uses: actions/download-artifact@v1 - with: - name: Tool test output 2 - - uses: actions/download-artifact@v1 - with: - name: Tool test output 3 - - uses: actions/setup-python@v1 - with: - python-version: '3.7' - - name: setup planemo - run: pip install planemo - - name: setup jq - run: sudo apt-get install jq - - name: combine outputs - run: | - find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + - - name: Create tool_test_output.html - run: planemo test_reports tool_test_output.json --test_output tool_test_output.html - - name: Copy artifacts into place - run: | - mkdir upload - mv tool_test_output.json tool_test_output.html upload/ - - uses: actions/upload-artifact@v1 - with: - name: 'All tool test results' - path: upload - - name: check status of combined status - run: | - stat=$(jq '.["tests"][]["data"]["status"]' upload/tool_test_output.json | grep -v "success" || true) - if [[ "$(echo -n $stat | wc -l)" -gt "0" ]]; then - echo "unsuccessful tests found $(sort <<<$stat | uniq -c)" - exit 1 - fi + - uses: actions/download-artifact@v1 + with: + name: Tool test output 0 + - uses: actions/download-artifact@v1 + with: + name: Tool test output 1 + - uses: actions/download-artifact@v1 + with: + name: Tool test output 2 + - uses: actions/download-artifact@v1 + with: + name: Tool test output 3 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + - name: cache .cache/pip + uses: actions/cache@v1 + id: cache-pip + with: + path: ~/.cache/pip + key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + + - name: setup planemo + run: pip install planemo + - name: setup jq + run: sudo apt-get install jq + - name: combine outputs + run: | + find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + + - name: create tool_test_output.html + run: planemo test_reports tool_test_output.json --test_output tool_test_output.html + - name: copy artifacts into place + run: | + mkdir upload + mv tool_test_output.json tool_test_output.html upload/ + - uses: actions/upload-artifact@v1 + with: + name: 'All tool test results' + path: upload + - name: check status of combined status + run: | + stat=$(jq '.["tests"][]["data"]["status"]' upload/tool_test_output.json | grep -v "success" || true) + if [[ "$(echo -n $stat | wc -l)" -gt "0" ]]; then + echo "unsuccessful tests found $(sort <<<$stat | uniq -c)" + exit 1 + fi deploy: name: Deploy @@ -247,18 +268,18 @@ jobs: - uses: actions/setup-python@v1 with: python-version: '3.7' - - name: Cache .cache/pip + - name: cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - name: setup planemo run: pip install planemo - - name: planemo ci_find_repos - run: | - mkdir changed_repositories - planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list + - uses: actions/download-artifact@v1 + with: + name: changed repositories + path: changed_repositories.list - name: deploy on testtoolshed env: SHED_KEY: ${{ secrets.tts_api_key }} @@ -266,11 +287,11 @@ jobs: while read -r DIR; do planemo shed_update --shed_target testtoolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; - done < changed_repositories.list + done < changed_repositories.list/changed_repositories.list - name: deploy on toolshed env: SHED_KEY: ${{ secrets.ts_api_key }} run: | while read -r DIR; do planemo shed_update --shed_target toolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; - done < changed_repositories.list + done < changed_repositories.list/changed_repositories.list From 97e95fc71794043ce2c1f90fbbd96bab36779c3b Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 1 Apr 2020 17:08:27 +0200 Subject: [PATCH 008/142] cache .planemo Co-Authored-By: Marius van den Beek --- .github/workflows/pr.yaml | 64 ++++++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f09d3472..96877a13 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,13 +19,19 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - - name: debug - run: echo ${{ steps.cache-pip.outputs.cache-hit }} + key: pip_cache_py${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + - name: determine latest galaxy commit + run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) + - name: Cache .planemo + uses: actions/cache@v1 + id: cache-planemo + with: + path: ~/.planemo + key: planemo_cache_py${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - name: install Planemo and flake8 run: pip install planemo flake8 - name: fake a planemo run to update cache - if: steps.cache-pip.outputs.cache-hit != 'true' + if: steps.cache-pip.outputs.cache-hit != 'true' && steps.cache-planemo.outputs.cache-hit != 'true' run: | touch tool.xml PIP_QUIET=1 planemo test --galaxy_python_version ${{ matrix.python-version }} --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE @@ -37,7 +43,7 @@ jobs: planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list - uses: actions/upload-artifact@v1 with: - name: changed repositories + name: Changed repositories path: changed_repositories.list lint: @@ -67,7 +73,7 @@ jobs: run: pip install planemo - uses: actions/download-artifact@v1 with: - name: changed repositories + name: Changed repositories path: changed_repositories.list - name: planemo lint run: | @@ -104,12 +110,8 @@ jobs: run: pip install flake8 - uses: actions/download-artifact@v1 with: - name: changed repositories + name: Changed repositories path: changed_repositories.list - - name: ls - run: ls -la - - name: ls - run: ls -la changed_repositories.list/changed_repositories.list - name: flake8 run: flake8 $(cat changed_repositories.list/changed_repositories.list) @@ -147,12 +149,19 @@ jobs: with: path: ~/.cache/pip key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + - name: determine latest galaxy commit + run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) + - name: Cache .planemo + uses: actions/cache@v1 + id: cache-planemo + with: + path: ~/.planemo + key: planemo_cache_py${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - name: install Planemo run: pip install planemo - - uses: actions/download-artifact@v1 with: - name: changed repositories + name: Changed repositories path: changed_repositories.list - name: planemo ci_find_tools run: | @@ -208,6 +217,9 @@ jobs: combine_outputs: name: Combine chunked test results needs: test + strategy: + matrix: + python-version: [3.7] # This job runs on Linux runs-on: ubuntu-latest steps: @@ -225,14 +237,21 @@ jobs: name: Tool test output 3 - uses: actions/setup-python@v1 with: - python-version: '3.7' + python-version: ${{ matrix.python-version }} - name: cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - + - name: determine latest galaxy commit + run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) + - name: Cache .planemo + uses: actions/cache@v1 + id: cache-planemo + with: + path: ~/.planemo + key: planemo_cache_py${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - name: setup planemo run: pip install planemo - name: setup jq @@ -262,23 +281,34 @@ jobs: name: Deploy needs: combine_outputs # This job runs on Linux + strategy: + matrix: + python-version: [3.7] runs-on: ubuntu-latest if: github.ref == 'refs/heads/master' steps: - uses: actions/setup-python@v1 with: - python-version: '3.7' + python-version: ${{ matrix.python-version }} - name: cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + - name: determine latest galaxy commit + run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) + - name: Cache .planemo + uses: actions/cache@v1 + id: cache-planemo + with: + path: ~/.planemo + key: planemo_cache_py${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - name: setup planemo run: pip install planemo - uses: actions/download-artifact@v1 with: - name: changed repositories + name: Changed repositories path: changed_repositories.list - name: deploy on testtoolshed env: From a850206fbe855a02474aed72a3d81da57e6d1777 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 1 Apr 2020 18:21:59 +0200 Subject: [PATCH 009/142] remove unnecessary planemo cache uses only test needs it also remove one seemingly unnecessary step --- .github/workflows/pr.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 96877a13..3bff6773 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -177,9 +177,6 @@ jobs: $(cat changed_repositories.list/changed_repositories.list) fi fi - - name: set tool list - id: tools - run: echo "::set-output name=tool_list::$(tr '\n' ' ' < changed_repositories.list/changed_repositories.list)" - name: show repo list run: cat changed_repositories.list/changed_repositories.list - name: show changed tools/repositories chunk list @@ -244,14 +241,6 @@ jobs: with: path: ~/.cache/pip key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - - name: determine latest galaxy commit - run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) - - name: Cache .planemo - uses: actions/cache@v1 - id: cache-planemo - with: - path: ~/.planemo - key: planemo_cache_py${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - name: setup planemo run: pip install planemo - name: setup jq From 52597cc28b6281352f237dd71818c92c5327ee6a Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 1 Apr 2020 19:45:07 +0200 Subject: [PATCH 010/142] fixes for merging - move output of changed repos to setup job - switch to checkout@v2 - unlimited checkout in setup step (depth: 0) - check for cache hit needs to be disjunction - fetch all branches - add missing checkout step to deploy - added a few comments docs - triple dot instead of double dots - get diff range for merge from github context Co-Authored-By: Nicola Soranzo --- .github/workflows/pr.yaml | 168 ++++++++++++++++++++++---------------- 1 file changed, 98 insertions(+), 70 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3bff6773..bf6e34a5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -11,6 +11,14 @@ jobs: matrix: python-version: [3.7] steps: + - name: ECHO + run: echo event_name ${{ github.event_name }} sha ${{ github.sha }} ref ${{ github.ref }} + - name: ECHO + run: echo head_ref ${{ github.head_ref }} base_ref ${{ github.head_ref }} pull_request_head_sha ${{ github.event.pull_request.head.sha }} + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -20,7 +28,7 @@ jobs: with: path: ~/.cache/pip key: pip_cache_py${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - - name: determine latest galaxy commit + - name: Determine latest galaxy commit run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) - name: Cache .planemo uses: actions/cache@v1 @@ -28,19 +36,41 @@ jobs: with: path: ~/.planemo key: planemo_cache_py${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - - name: install Planemo and flake8 + - name: Install Planemo and flake8 run: pip install planemo flake8 - - name: fake a planemo run to update cache - if: steps.cache-pip.outputs.cache-hit != 'true' && steps.cache-planemo.outputs.cache-hit != 'true' + - name: Fake a planemo run to update cache + if: steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' run: | touch tool.xml PIP_QUIET=1 planemo test --galaxy_python_version ${{ matrix.python-version }} --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: - fetch-depth: 1 - - name: planemo ci_find_repos + fetch-depth: 0 + - name: get all branches + run: | + git fetch --no-tags --prune --depth=1 origin +refs/heads/*:refs/remotes/origin/* + - name: Determine commit range for feature branch + if: github.ref != 'refs/heads/master' + run: echo ::set-env name=COMMIT_RANGE::"origin/master..." + + # for merge commit it is either + # - a 3 way merge: then the range is determined by the two parents: HEAD^ + # (what was master before the merge) and HEAD^2 (the HEAD of the feature + # branch) + # - a fast forward merge then the commit is what happend wrt to the parent + # is the range + - name: Determine commit range for merge to master + if: github.ref == 'refs/heads/master' + run: echo ::set-env name=COMMIT_RANGE::${{ github.event.before }}...${{ github.event.after }} + # $(git rev-parse HEAD^2 &> /dev/null && echo "HEAD^...HEAD^2" || echo "HEAD^") + + - name: more debug + run: git diff --name-only ${{ env.COMMIT_RANGE }} -- + - name: Planemo ci_find_repos run: | - planemo ci_find_repos --changed_in_commit_range origin/master... --exclude packages --exclude deprecated --output changed_repositories.list + planemo ci_find_repos --changed_in_commit_range ${{ env.COMMIT_RANGE }} --exclude packages --exclude deprecated --output changed_repositories.list + - name: Show repo list + run: cat changed_repositories.list - uses: actions/upload-artifact@v1 with: name: Changed repositories @@ -57,31 +87,30 @@ jobs: steps: # checkout the repository to master # and use it as the current working directory - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: fetch-depth: 1 - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: cache .cache/pip + - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - - name: setup planemo + - name: Setup planemo run: pip install planemo - uses: actions/download-artifact@v1 with: name: Changed repositories path: changed_repositories.list - - name: planemo lint + - name: Planemo lint run: | - set -e - while read -r DIR - do - planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; - done < changed_repositories.list/changed_repositories.list + set -e + while read -r DIR; do + planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; + done < changed_repositories.list/changed_repositories.list flake8: name: Lint python scripts @@ -94,29 +123,29 @@ jobs: steps: # checkout the repository to master # and use it as the current working directory - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: fetch-depth: 1 - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: cache .cache/pip + - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - - name: setup flake8 + - name: Setup flake8 run: pip install flake8 - uses: actions/download-artifact@v1 with: name: Changed repositories path: changed_repositories.list - - name: flake8 + - name: Flake8 run: flake8 $(cat changed_repositories.list/changed_repositories.list) test: - name: test tools + name: Test tools # This job runs on Linux runs-on: ubuntu-latest needs: [lint,flake8] @@ -137,19 +166,19 @@ jobs: steps: # checkout the repository to master # and use it as the current working directory - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: fetch-depth: 1 - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: cache .cache/pip + - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - - name: determine latest galaxy commit + - name: Determine latest galaxy commit run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) - name: Cache .planemo uses: actions/cache@v1 @@ -157,13 +186,13 @@ jobs: with: path: ~/.planemo key: planemo_cache_py${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - - name: install Planemo + - name: Install Planemo run: pip install planemo - uses: actions/download-artifact@v1 with: name: Changed repositories path: changed_repositories.list - - name: planemo ci_find_tools + - name: Planemo ci_find_tools run: | touch changed_repositories_chunk.list changed_tools_chunk.list if [ -s changed_repositories.list/changed_repositories.list ]; then @@ -177,40 +206,43 @@ jobs: $(cat changed_repositories.list/changed_repositories.list) fi fi - - name: show repo list - run: cat changed_repositories.list/changed_repositories.list - - name: show changed tools/repositories chunk list + - name: Show changed tools/repositories chunk list run: cat changed_tools_chunk.list changed_repositories_chunk.list - - name: planemo test tools + - name: Planemo test tools run: | - if [ -s changed_tools_chunk.list ]; then - PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --biocontainers --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json tool_test_output.json $(cat changed_tools_chunk.list) || true - docker system prune --all --force --volumes || true - elif [ -s changed_repositories_chunk.list ]; then - while read -r DIR; do - if [[ "$DIR" =~ ^data_managers.* ]]; then - TESTPATH=$(planemo ci_find_tools "$DIR") - else - TESTPATH="$DIR" - fi - PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --biocontainers --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$DIR"/tool_test_output.json "$TESTPATH" || true - docker system prune --all --force --volumes || true - done < changed_repositories_chunk.list - else - echo '{"tests":[]}' > tool_test_output.json - fi - - name: merge tool_test_output.json files + if [ -s changed_tools_chunk.list ]; then + PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --biocontainers --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json tool_test_output.json $(cat changed_tools_chunk.list) || true + docker system prune --all --force --volumes || true + elif [ -s changed_repositories_chunk.list ]; then + while read -r DIR; do + if [[ "$DIR" =~ ^data_managers.* ]]; then + TESTPATH=$(planemo ci_find_tools "$DIR") + else + TESTPATH="$DIR" + fi + PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --biocontainers --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$DIR"/tool_test_output.json "$TESTPATH" || true + docker system prune --all --force --volumes || true + done < changed_repositories_chunk.list + else + echo '{"tests":[]}' > tool_test_output.json + fi + - name: Merge tool_test_output.json files run: | find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + - - name: create tool_test_output.html + - name: Create tool_test_output.html run: planemo test_reports tool_test_output.json --test_output tool_test_output.html - - name: copy artifacts into place + - name: Copy artifacts into place run: mkdir upload && mv tool_test_output.json tool_test_output.html upload - uses: actions/upload-artifact@v1 with: name: 'Tool test output ${{ matrix.chunk }}' path: upload + # - combine the results of the test chunks (which will never fail due + # to `|| true`) and create a global test report as json and html which + # is provided as artifact + # - check if any tool test actually failed (by lookup in the combined json) + # and fail this step if this is the case combine_outputs: name: Combine chunked test results needs: test @@ -235,22 +267,22 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: cache .cache/pip + - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - - name: setup planemo + - name: Setup planemo run: pip install planemo - - name: setup jq + - name: Setup jq run: sudo apt-get install jq - - name: combine outputs + - name: Combine outputs run: | find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + - - name: create tool_test_output.html + - name: Create tool_test_output.html run: planemo test_reports tool_test_output.json --test_output tool_test_output.html - - name: copy artifacts into place + - name: Copy artifacts into place run: | mkdir upload mv tool_test_output.json tool_test_output.html upload/ @@ -258,7 +290,7 @@ jobs: with: name: 'All tool test results' path: upload - - name: check status of combined status + - name: Check status of combined status run: | stat=$(jq '.["tests"][]["data"]["status"]' upload/tool_test_output.json | grep -v "success" || true) if [[ "$(echo -n $stat | wc -l)" -gt "0" ]]; then @@ -276,38 +308,34 @@ jobs: runs-on: ubuntu-latest if: github.ref == 'refs/heads/master' steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: cache .cache/pip + - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - - name: determine latest galaxy commit - run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) - - name: Cache .planemo - uses: actions/cache@v1 - id: cache-planemo - with: - path: ~/.planemo - key: planemo_cache_py${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - - name: setup planemo + - name: Setup planemo run: pip install planemo - uses: actions/download-artifact@v1 with: name: Changed repositories path: changed_repositories.list - - name: deploy on testtoolshed + # TODO remove --owner + - name: Deploy on testtoolshed env: SHED_KEY: ${{ secrets.tts_api_key }} run: | while read -r DIR; do - planemo shed_update --shed_target testtoolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; + planemo shed_update --shed_target testtoolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" --owner matthias || exit 1; done < changed_repositories.list/changed_repositories.list - - name: deploy on toolshed + - name: Deploy on toolshed env: SHED_KEY: ${{ secrets.ts_api_key }} run: | From b7824802dab2b8a6a80e88816aba83178e997384 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 3 Apr 2020 17:59:01 +0200 Subject: [PATCH 011/142] persistent galaxy.sha and pip cache depends on sha - make galaxy sha persistent as artifacts - place changed_repos.list and galaxy.sha in a single folder in order to have only one artifact for both - use the galaxy sha also for caching pip - store workflow artifacts in HOME - add comments remove debug --- .github/workflows/pr.yaml | 124 ++++++++++++++++++++++---------------- 1 file changed, 71 insertions(+), 53 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index bf6e34a5..6d12a267 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -4,6 +4,13 @@ env: GALAXY_REPO: https://github.com/galaxyproject/galaxy GALAXY_RELEASE: release_20.01 jobs: + # the setup job does two things: + # 1. cache the pip cache and and + # 2. determine the list of changed repositories + # it produces one artifact which contains + # - a file with the latest SHA from the chosen branch of the Galaxy repo + # - a file containing the changed repositories + # which are needed in subsequent steps. setup: name: Setup cache and determine changed repositories runs-on: ubuntu-latest @@ -11,10 +18,6 @@ jobs: matrix: python-version: [3.7] steps: - - name: ECHO - run: echo event_name ${{ github.event_name }} sha ${{ github.sha }} ref ${{ github.ref }} - - name: ECHO - run: echo head_ref ${{ github.head_ref }} base_ref ${{ github.head_ref }} pull_request_head_sha ${{ github.event.pull_request.head.sha }} - name: Dump GitHub context env: GITHUB_CONTEXT: ${{ toJson(github) }} @@ -22,14 +25,18 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - name: make dir for workflow artifacts + run: mkdir ../workflow_artifacts + - name: Determine latest galaxy commit + run: | + echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) + echo ${{ env.GALAXY_HEAD_SHA }} > ../workflow_artifacts/galaxy.sha - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - - name: Determine latest galaxy commit - run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) + key: pip_cache_py${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - name: Cache .planemo uses: actions/cache@v1 id: cache-planemo @@ -49,33 +56,28 @@ jobs: - name: get all branches run: | git fetch --no-tags --prune --depth=1 origin +refs/heads/*:refs/remotes/origin/* + # the range of commits to check for changes is simply master... for + # all events happening on the feature branch, for events on the master + # branch (i.e. for merge) we use the sha before and after the event + # (note that this does not work for feature branch events since we want all + # commits on the feature branch and not just the commits of the last event) - name: Determine commit range for feature branch if: github.ref != 'refs/heads/master' run: echo ::set-env name=COMMIT_RANGE::"origin/master..." - - # for merge commit it is either - # - a 3 way merge: then the range is determined by the two parents: HEAD^ - # (what was master before the merge) and HEAD^2 (the HEAD of the feature - # branch) - # - a fast forward merge then the commit is what happend wrt to the parent - # is the range - name: Determine commit range for merge to master if: github.ref == 'refs/heads/master' run: echo ::set-env name=COMMIT_RANGE::${{ github.event.before }}...${{ github.event.after }} - # $(git rev-parse HEAD^2 &> /dev/null && echo "HEAD^...HEAD^2" || echo "HEAD^") - - - name: more debug - run: git diff --name-only ${{ env.COMMIT_RANGE }} -- - name: Planemo ci_find_repos run: | - planemo ci_find_repos --changed_in_commit_range ${{ env.COMMIT_RANGE }} --exclude packages --exclude deprecated --output changed_repositories.list + planemo ci_find_repos --changed_in_commit_range ${{ env.COMMIT_RANGE }} --exclude packages --exclude deprecated --output ../workflow_artifacts/changed_repositories.list - name: Show repo list - run: cat changed_repositories.list + run: cat ../workflow_artifacts/changed_repositories.list - uses: actions/upload-artifact@v1 with: - name: Changed repositories - path: changed_repositories.list + name: Workflow artifacts + path: ../workflow_artifacts/ + # planemo lint the changed repositories lint: name: Lint tools needs: setup @@ -93,25 +95,28 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - uses: actions/download-artifact@v1 + with: + name: Workflow artifacts + path: ../workflow_artifacts/ + - name: Determine latest galaxy commit + run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - name: Setup planemo run: pip install planemo - - uses: actions/download-artifact@v1 - with: - name: Changed repositories - path: changed_repositories.list - name: Planemo lint run: | set -e while read -r DIR; do planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; - done < changed_repositories.list/changed_repositories.list + done < ../workflow_artifacts/changed_repositories.list + # flake8 of python scripts in the changed repositories flake8: name: Lint python scripts needs: setup @@ -129,21 +134,25 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - uses: actions/download-artifact@v1 + with: + name: Workflow artifacts + path: ../workflow_artifacts/ + - name: Determine latest galaxy commit + run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - name: Setup flake8 run: pip install flake8 - - uses: actions/download-artifact@v1 - with: - name: Changed repositories - path: changed_repositories.list - name: Flake8 - run: flake8 $(cat changed_repositories.list/changed_repositories.list) + run: flake8 $(cat ../workflow_artifacts/changed_repositories.list) + # planemo test the changed repositories, each chunk creates an artifact + # containing html and json reports for the executed tests test: name: Test tools # This job runs on Linux @@ -172,14 +181,18 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - uses: actions/download-artifact@v1 + with: + name: Workflow artifacts + path: ../workflow_artifacts/ + - name: Determine latest galaxy commit + run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - - name: Determine latest galaxy commit - run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) + key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - name: Cache .planemo uses: actions/cache@v1 id: cache-planemo @@ -188,22 +201,18 @@ jobs: key: planemo_cache_py${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - name: Install Planemo run: pip install planemo - - uses: actions/download-artifact@v1 - with: - name: Changed repositories - path: changed_repositories.list - name: Planemo ci_find_tools run: | touch changed_repositories_chunk.list changed_tools_chunk.list - if [ -s changed_repositories.list/changed_repositories.list ]; then - if [ $(wc -l < changed_repositories.list/changed_repositories.list) -eq 1 ]; then + if [ -s ../workflow_artifacts/changed_repositories.list ]; then + if [ $(wc -l < ../workflow_artifacts/changed_repositories.list) -eq 1 ]; then planemo ci_find_tools --chunk_count 4 --chunk ${{ matrix.chunk }} \ --output changed_tools_chunk.list \ - $(cat changed_repositories.list/changed_repositories.list) + $(cat ../workflow_artifacts/changed_repositories.list) else planemo ci_find_repos --chunk_count 4 --chunk ${{ matrix.chunk }} \ --output changed_repositories_chunk.list \ - $(cat changed_repositories.list/changed_repositories.list) + $(cat ../workflow_artifacts/changed_repositories.list) fi fi - name: Show changed tools/repositories chunk list @@ -267,12 +276,18 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - uses: actions/download-artifact@v1 + with: + name: Workflow artifacts + path: ../workflow_artifacts/ + - name: Determine latest galaxy commit + run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - name: Setup planemo run: pip install planemo - name: Setup jq @@ -298,6 +313,7 @@ jobs: exit 1 fi + # deploy the tools to the toolsheds (first TTS for testing) deploy: name: Deploy needs: combine_outputs @@ -314,18 +330,20 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - uses: actions/download-artifact@v1 + with: + name: Workflow artifacts + path: ../workflow_artifacts/ + - name: Determine latest galaxy commit + run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} + key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - name: Setup planemo run: pip install planemo - - uses: actions/download-artifact@v1 - with: - name: Changed repositories - path: changed_repositories.list # TODO remove --owner - name: Deploy on testtoolshed env: @@ -334,11 +352,11 @@ jobs: while read -r DIR; do planemo shed_update --shed_target testtoolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" --owner matthias || exit 1; - done < changed_repositories.list/changed_repositories.list + done < ../workflow_artifacts/changed_repositories.list - name: Deploy on toolshed env: SHED_KEY: ${{ secrets.ts_api_key }} run: | while read -r DIR; do planemo shed_update --shed_target toolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; - done < changed_repositories.list/changed_repositories.list + done < ../workflow_artifacts/changed_repositories.list From 113b9ccadc9a240ad63cd4f131ed9499b2403893 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 3 Apr 2020 11:15:55 +0100 Subject: [PATCH 012/142] Distinguish between pushes to master and PRs --- .github/workflows/pr.yaml | 62 ++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6d12a267..0356ba9c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -5,11 +5,11 @@ env: GALAXY_RELEASE: release_20.01 jobs: # the setup job does two things: - # 1. cache the pip cache and and + # 1. cache the pip cache and .planemo # 2. determine the list of changed repositories # it produces one artifact which contains # - a file with the latest SHA from the chosen branch of the Galaxy repo - # - a file containing the changed repositories + # - a file containing the list of changed repositories # which are needed in subsequent steps. setup: name: Setup cache and determine changed repositories @@ -18,10 +18,15 @@ jobs: matrix: python-version: [3.7] steps: - - name: Dump GitHub context - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - run: echo "$GITHUB_CONTEXT" + - name: Print github context properties + run: | + echo 'event: ${{ github.event_name }}' + echo 'sha: ${{ github.sha }}' + echo 'ref: ${{ github.ref }}' + echo 'head_ref: ${{ github.head_ref }}' + echo 'base_ref: ${{ github.base_ref }}' + echo 'event.before: ${{ github.event.before }}' + echo 'event.after: ${{ github.event.after }}' - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -53,23 +58,26 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: get all branches + # The range of commits to check for changes is: + # - `origin/master...` for all events happening on a feature branch + # - for events on the master branch we compare against the sha before the event + # (note that this does not work for feature branch events since we want all + # commits on the feature branch and not just the commits of the last event) + # - for pull requests we compare against the 1st ancestor, given the current + # HEAD is the merge between the PR branch and the base branch + - name: Set commit range for a feature branch + if: github.ref != 'refs/heads/master' && github.event_name == 'push' run: | - git fetch --no-tags --prune --depth=1 origin +refs/heads/*:refs/remotes/origin/* - # the range of commits to check for changes is simply master... for - # all events happening on the feature branch, for events on the master - # branch (i.e. for merge) we use the sha before and after the event - # (note that this does not work for feature branch events since we want all - # commits on the feature branch and not just the commits of the last event) - - name: Determine commit range for feature branch - if: github.ref != 'refs/heads/master' - run: echo ::set-env name=COMMIT_RANGE::"origin/master..." - - name: Determine commit range for merge to master + git fetch origin master + echo ::set-env name=COMMIT_RANGE::"origin/master..." + - name: Set commit range for the master branch if: github.ref == 'refs/heads/master' - run: echo ::set-env name=COMMIT_RANGE::${{ github.event.before }}...${{ github.event.after }} + run: echo ::set-env name=COMMIT_RANGE::${{ github.event.before }}.."" + - name: Set commit range for a pull request + if: github.event_name == 'pull_request' + run: echo ::set-env name=COMMIT_RANGE::"HEAD~.." - name: Planemo ci_find_repos - run: | - planemo ci_find_repos --changed_in_commit_range ${{ env.COMMIT_RANGE }} --exclude packages --exclude deprecated --output ../workflow_artifacts/changed_repositories.list + run: planemo ci_find_repos --changed_in_commit_range ${{ env.COMMIT_RANGE }} --exclude packages --exclude deprecated --output ../workflow_artifacts/changed_repositories.list - name: Show repo list run: cat ../workflow_artifacts/changed_repositories.list - uses: actions/upload-artifact@v1 @@ -146,7 +154,7 @@ jobs: with: path: ~/.cache/pip key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - - name: Setup flake8 + - name: Setup flake8 run: pip install flake8 - name: Flake8 run: flake8 $(cat ../workflow_artifacts/changed_repositories.list) @@ -236,12 +244,13 @@ jobs: echo '{"tests":[]}' > tool_test_output.json fi - name: Merge tool_test_output.json files - run: | - find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + + run: find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + - name: Create tool_test_output.html run: planemo test_reports tool_test_output.json --test_output tool_test_output.html - name: Copy artifacts into place - run: mkdir upload && mv tool_test_output.json tool_test_output.html upload + run: | + mkdir upload + mv tool_test_output.json tool_test_output.html upload/ - uses: actions/upload-artifact@v1 with: name: 'Tool test output ${{ matrix.chunk }}' @@ -251,7 +260,7 @@ jobs: # to `|| true`) and create a global test report as json and html which # is provided as artifact # - check if any tool test actually failed (by lookup in the combined json) - # and fail this step if this is the case + # and fail this step if this is the case combine_outputs: name: Combine chunked test results needs: test @@ -293,8 +302,7 @@ jobs: - name: Setup jq run: sudo apt-get install jq - name: Combine outputs - run: | - find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + + run: find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + - name: Create tool_test_output.html run: planemo test_reports tool_test_output.json --test_output tool_test_output.html - name: Copy artifacts into place From 5e320b94c575c08e3049357ef8f35f140accee21 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sat, 4 Apr 2020 12:59:31 +0200 Subject: [PATCH 013/142] make it explicit that events to the master are push and stylistic changes --- .github/workflows/pr.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0356ba9c..9c0108c3 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -65,15 +65,15 @@ jobs: # commits on the feature branch and not just the commits of the last event) # - for pull requests we compare against the 1st ancestor, given the current # HEAD is the merge between the PR branch and the base branch - - name: Set commit range for a feature branch + - name: Set commit range (push to the feature branch) if: github.ref != 'refs/heads/master' && github.event_name == 'push' run: | git fetch origin master echo ::set-env name=COMMIT_RANGE::"origin/master..." - - name: Set commit range for the master branch - if: github.ref == 'refs/heads/master' + - name: Set commit range (push to the master branch, i.e. merge) + if: github.ref == 'refs/heads/master' && github.event_name == 'push' run: echo ::set-env name=COMMIT_RANGE::${{ github.event.before }}.."" - - name: Set commit range for a pull request + - name: Set commit range (pull request) if: github.event_name == 'pull_request' run: echo ::set-env name=COMMIT_RANGE::"HEAD~.." - name: Planemo ci_find_repos From ed931e3700990c03142c04e2f2ef9e13a818d71a Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 6 Apr 2020 12:53:17 +0200 Subject: [PATCH 014/142] guard flake8 --- .github/workflows/pr.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9c0108c3..b9abf47f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -157,7 +157,10 @@ jobs: - name: Setup flake8 run: pip install flake8 - name: Flake8 - run: flake8 $(cat ../workflow_artifacts/changed_repositories.list) + run: | + if [ -s ../workflow_artifacts/changed_repositories.list ]; then + flake8 $(cat ../workflow_artifacts/changed_repositories.list) + fi # planemo test the changed repositories, each chunk creates an artifact # containing html and json reports for the executed tests From b0162b0d981bb91a404cf1fa450f368208acf158 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 6 Apr 2020 13:34:50 +0200 Subject: [PATCH 015/142] i.e. -> e.g. Co-Authored-By: Nicola Soranzo --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b9abf47f..92343faa 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -70,7 +70,7 @@ jobs: run: | git fetch origin master echo ::set-env name=COMMIT_RANGE::"origin/master..." - - name: Set commit range (push to the master branch, i.e. merge) + - name: Set commit range (push to the master branch, e.g. merge) if: github.ref == 'refs/heads/master' && github.event_name == 'push' run: echo ::set-env name=COMMIT_RANGE::${{ github.event.before }}.."" - name: Set commit range (pull request) From 07af27ad4981856e53e10674d3361c331618ebf0 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 6 Apr 2020 21:56:58 +0100 Subject: [PATCH 016/142] Restore correct owner --- .github/workflows/pr.yaml | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 92343faa..b7738462 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -95,7 +95,7 @@ jobs: matrix: python-version: [3.7] steps: - # checkout the repository to master + # checkout the repository # and use it as the current working directory - uses: actions/checkout@v2 with: @@ -115,7 +115,7 @@ jobs: with: path: ~/.cache/pip key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - - name: Setup planemo + - name: Install Planemo run: pip install planemo - name: Planemo lint run: | @@ -126,7 +126,7 @@ jobs: # flake8 of python scripts in the changed repositories flake8: - name: Lint python scripts + name: Lint Python scripts needs: setup runs-on: ubuntu-latest strategy: @@ -154,12 +154,12 @@ jobs: with: path: ~/.cache/pip key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - - name: Setup flake8 + - name: Install flake8 run: pip install flake8 - name: Flake8 run: | if [ -s ../workflow_artifacts/changed_repositories.list ]; then - flake8 $(cat ../workflow_artifacts/changed_repositories.list) + flake8 $(cat ../workflow_artifacts/changed_repositories.list) fi # planemo test the changed repositories, each chunk creates an artifact @@ -184,7 +184,7 @@ jobs: ports: - 5432:5432 steps: - # checkout the repository to master + # checkout the repository # and use it as the current working directory - uses: actions/checkout@v2 with: @@ -300,9 +300,9 @@ jobs: with: path: ~/.cache/pip key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - - name: Setup planemo + - name: Install Planemo run: pip install planemo - - name: Setup jq + - name: Install jq run: sudo apt-get install jq - name: Combine outputs run: find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + @@ -320,7 +320,7 @@ jobs: run: | stat=$(jq '.["tests"][]["data"]["status"]' upload/tool_test_output.json | grep -v "success" || true) if [[ "$(echo -n $stat | wc -l)" -gt "0" ]]; then - echo "unsuccessful tests found $(sort <<<$stat | uniq -c)" + echo "Unsuccessful tests found $(sort <<<$stat | uniq -c)" exit 1 fi @@ -328,7 +328,6 @@ jobs: deploy: name: Deploy needs: combine_outputs - # This job runs on Linux strategy: matrix: python-version: [3.7] @@ -353,21 +352,19 @@ jobs: with: path: ~/.cache/pip key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} - - name: Setup planemo + - name: Install Planemo run: pip install planemo - # TODO remove --owner - name: Deploy on testtoolshed env: SHED_KEY: ${{ secrets.tts_api_key }} run: | - while read -r DIR; - do - planemo shed_update --shed_target testtoolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" --owner matthias || exit 1; + while read -r DIR; do + planemo shed_update --shed_target testtoolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; done < ../workflow_artifacts/changed_repositories.list - name: Deploy on toolshed env: SHED_KEY: ${{ secrets.ts_api_key }} run: | - while read -r DIR; - do planemo shed_update --shed_target toolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; + while read -r DIR; do + planemo shed_update --shed_target toolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; done < ../workflow_artifacts/changed_repositories.list From 1e715436ff69d29fcb028a5e65734c759a1410ce Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 9 Apr 2020 01:11:04 +0100 Subject: [PATCH 017/142] Fix 'Check status of combined status' GH workflow step --- .github/workflows/pr.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b7738462..f63861f4 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -318,9 +318,8 @@ jobs: path: upload - name: Check status of combined status run: | - stat=$(jq '.["tests"][]["data"]["status"]' upload/tool_test_output.json | grep -v "success" || true) - if [[ "$(echo -n $stat | wc -l)" -gt "0" ]]; then - echo "Unsuccessful tests found $(sort <<<$stat | uniq -c)" + if jq '.["tests"][]["data"]["status"]' upload/tool_test_output.json | grep -v "success"; then + echo "Unsuccessful tests found, inspect the 'All tool test results' artifact for details." exit 1 fi From c69746fa8aba52182823d8949740bf85cf554903 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 10 Apr 2020 14:10:18 +0200 Subject: [PATCH 018/142] githup actions: fix chache names, storing of sha latest galaxy commit sha was not saved in artifacts file (workflow_artifacts/galaxy.sha) since the environment variable seems to be available only after the step where it is set. pip cache name was inconsistent: - pip_cache_py - pip_cache_python_ --- .github/workflows/pr.yaml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f63861f4..819f1d43 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -33,21 +33,23 @@ jobs: - name: make dir for workflow artifacts run: mkdir ../workflow_artifacts - name: Determine latest galaxy commit + run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) + - name: Save latest galaxy commit to artifact file and show it run: | - echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) echo ${{ env.GALAXY_HEAD_SHA }} > ../workflow_artifacts/galaxy.sha + cat ../workflow_artifacts/galaxy.sha - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} - name: Cache .planemo uses: actions/cache@v1 id: cache-planemo with: path: ~/.planemo - key: planemo_cache_py${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} + key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} - name: Install Planemo and flake8 run: pip install planemo flake8 - name: Fake a planemo run to update cache @@ -114,7 +116,7 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} - name: Install Planemo run: pip install planemo - name: Planemo lint @@ -153,7 +155,7 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} - name: Install flake8 run: pip install flake8 - name: Flake8 @@ -203,13 +205,13 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} - name: Cache .planemo uses: actions/cache@v1 id: cache-planemo with: path: ~/.planemo - key: planemo_cache_py${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} + key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} - name: Install Planemo run: pip install planemo - name: Planemo ci_find_tools @@ -299,7 +301,7 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} - name: Install Planemo run: pip install planemo - name: Install jq @@ -350,7 +352,7 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_python_${{ matrix.python-version }}_${{ env.GALAXY_HEAD_SHA }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} - name: Install Planemo run: pip install planemo - name: Deploy on testtoolshed From a99fdbf688c22c6af4cf52879dcc5cf283ba1f79 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sun, 12 Apr 2020 13:40:52 +0200 Subject: [PATCH 019/142] Don't output sha Co-Authored-By: Nicola Soranzo --- .github/workflows/pr.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 819f1d43..6219f561 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -35,9 +35,7 @@ jobs: - name: Determine latest galaxy commit run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) - name: Save latest galaxy commit to artifact file and show it - run: | - echo ${{ env.GALAXY_HEAD_SHA }} > ../workflow_artifacts/galaxy.sha - cat ../workflow_artifacts/galaxy.sha + run: echo ${{ env.GALAXY_HEAD_SHA }} > ../workflow_artifacts/galaxy.sha - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip From c370de710304e6204dd8536c23f8758cd683c751 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 22 Apr 2020 10:30:11 +0200 Subject: [PATCH 020/142] rewording --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6219f561..654f78c8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -34,7 +34,7 @@ jobs: run: mkdir ../workflow_artifacts - name: Determine latest galaxy commit run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) - - name: Save latest galaxy commit to artifact file and show it + - name: Save latest galaxy commit to artifact file run: echo ${{ env.GALAXY_HEAD_SHA }} > ../workflow_artifacts/galaxy.sha - name: Cache .cache/pip uses: actions/cache@v1 From 50e197a24b5f082633e8fcd502c1283af3656bfb Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 29 Apr 2020 10:28:06 +0200 Subject: [PATCH 021/142] use v2 of upload and download artifact --- .github/workflows/pr.yaml | 56 ++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 654f78c8..aa300bd2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -27,15 +27,16 @@ jobs: echo 'base_ref: ${{ github.base_ref }}' echo 'event.before: ${{ github.event.before }}' echo 'event.after: ${{ github.event.after }}' + - uses: actions/checkout@v2 + with: + fetch-depth: 0 - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: make dir for workflow artifacts - run: mkdir ../workflow_artifacts - name: Determine latest galaxy commit run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) - name: Save latest galaxy commit to artifact file - run: echo ${{ env.GALAXY_HEAD_SHA }} > ../workflow_artifacts/galaxy.sha + run: echo ${{ env.GALAXY_HEAD_SHA }} > galaxy.sha - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip @@ -55,9 +56,6 @@ jobs: run: | touch tool.xml PIP_QUIET=1 planemo test --galaxy_python_version ${{ matrix.python-version }} --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE - - uses: actions/checkout@v2 - with: - fetch-depth: 0 # The range of commits to check for changes is: # - `origin/master...` for all events happening on a feature branch # - for events on the master branch we compare against the sha before the event @@ -77,13 +75,18 @@ jobs: if: github.event_name == 'pull_request' run: echo ::set-env name=COMMIT_RANGE::"HEAD~.." - name: Planemo ci_find_repos - run: planemo ci_find_repos --changed_in_commit_range ${{ env.COMMIT_RANGE }} --exclude packages --exclude deprecated --output ../workflow_artifacts/changed_repositories.list + run: planemo ci_find_repos --changed_in_commit_range ${{ env.COMMIT_RANGE }} --exclude packages --exclude deprecated --output changed_repositories.list - name: Show repo list - run: cat ../workflow_artifacts/changed_repositories.list - - uses: actions/upload-artifact@v1 + run: cat changed_repositories.list + - uses: actions/upload-artifact@v2 with: name: Workflow artifacts - path: ../workflow_artifacts/ + path: changed_repositories.list + - uses: actions/upload-artifact@v2 + with: + name: Workflow artifacts + path: galaxy.sha + # planemo lint the changed repositories lint: @@ -103,7 +106,7 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - uses: actions/download-artifact@v1 + - uses: actions/download-artifact@v2 with: name: Workflow artifacts path: ../workflow_artifacts/ @@ -142,7 +145,7 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - uses: actions/download-artifact@v1 + - uses: actions/download-artifact@v2 with: name: Workflow artifacts path: ../workflow_artifacts/ @@ -192,7 +195,7 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - uses: actions/download-artifact@v1 + - uses: actions/download-artifact@v2 with: name: Workflow artifacts path: ../workflow_artifacts/ @@ -254,7 +257,7 @@ jobs: run: | mkdir upload mv tool_test_output.json tool_test_output.html upload/ - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v2 with: name: 'Tool test output ${{ matrix.chunk }}' path: upload @@ -273,27 +276,14 @@ jobs: # This job runs on Linux runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v1 - with: - name: Tool test output 0 - - uses: actions/download-artifact@v1 - with: - name: Tool test output 1 - - uses: actions/download-artifact@v1 + - uses: actions/download-artifact@v2 with: - name: Tool test output 2 - - uses: actions/download-artifact@v1 - with: - name: Tool test output 3 + path: artifacts - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - uses: actions/download-artifact@v1 - with: - name: Workflow artifacts - path: ../workflow_artifacts/ - name: Determine latest galaxy commit - run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) + run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat "artifacts/Workflow artifacts/galaxy.sha") - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip @@ -305,14 +295,14 @@ jobs: - name: Install jq run: sudo apt-get install jq - name: Combine outputs - run: find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + + run: find artifacts/ -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + - name: Create tool_test_output.html run: planemo test_reports tool_test_output.json --test_output tool_test_output.html - name: Copy artifacts into place run: | mkdir upload mv tool_test_output.json tool_test_output.html upload/ - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v2 with: name: 'All tool test results' path: upload @@ -339,7 +329,7 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - uses: actions/download-artifact@v1 + - uses: actions/download-artifact@v2 with: name: Workflow artifacts path: ../workflow_artifacts/ From 15286de159931a9d4a0c1eb9ec856e85d4b5cf53 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 5 May 2020 10:51:15 +0200 Subject: [PATCH 022/142] fix to early checkout fix bug introduced in https://github.com/galaxyproject/tools-iuc/pull/3002 where checkout was moved to top because upload@v2 can act on `../`. result: fake planemo run takes whole IUC tree -> time out solution: `cd ..` before upload --- .github/workflows/pr.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index aa300bd2..ef3f3d4b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -27,9 +27,6 @@ jobs: echo 'base_ref: ${{ github.base_ref }}' echo 'event.before: ${{ github.event.before }}' echo 'event.after: ${{ github.event.after }}' - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -37,6 +34,10 @@ jobs: run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) - name: Save latest galaxy commit to artifact file run: echo ${{ env.GALAXY_HEAD_SHA }} > galaxy.sha + - uses: actions/upload-artifact@v2 + with: + name: Workflow artifacts + path: galaxy.sha - name: Cache .cache/pip uses: actions/cache@v1 id: cache-pip @@ -56,6 +57,9 @@ jobs: run: | touch tool.xml PIP_QUIET=1 planemo test --galaxy_python_version ${{ matrix.python-version }} --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # The range of commits to check for changes is: # - `origin/master...` for all events happening on a feature branch # - for events on the master branch we compare against the sha before the event @@ -82,11 +86,6 @@ jobs: with: name: Workflow artifacts path: changed_repositories.list - - uses: actions/upload-artifact@v2 - with: - name: Workflow artifacts - path: galaxy.sha - # planemo lint the changed repositories lint: From e0207692fa73ddddeac0c00a8f222878a340873e Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 26 May 2020 11:24:18 +0200 Subject: [PATCH 023/142] Workflow updates ci: - check final success - use pip to install planemo - rename workflow to differentiate it more clearly from the PR workflow - small changes as done in https://github.com/galaxyproject/tools-iuc/pull/2939 - sync with pr workflow - switch to 20.05 ci+pr: - use cache@v1.2 - upload-artifacti v2 -> v2.0.1 Co-authored-by: Marius van den Beek --- .github/workflows/pr.yaml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ef3f3d4b..b9efd5c8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -39,13 +39,13 @@ jobs: name: Workflow artifacts path: galaxy.sha - name: Cache .cache/pip - uses: actions/cache@v1 + uses: actions/cache@v1.2.0 id: cache-pip with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} - name: Cache .planemo - uses: actions/cache@v1 + uses: actions/cache@v1.2.0 id: cache-planemo with: path: ~/.planemo @@ -82,10 +82,14 @@ jobs: run: planemo ci_find_repos --changed_in_commit_range ${{ env.COMMIT_RANGE }} --exclude packages --exclude deprecated --output changed_repositories.list - name: Show repo list run: cat changed_repositories.list - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v2.0.1 with: name: Workflow artifacts path: changed_repositories.list + - uses: actions/upload-artifact@v2.0.1 + with: + name: Workflow artifacts + path: galaxy.sha # planemo lint the changed repositories lint: @@ -112,7 +116,7 @@ jobs: - name: Determine latest galaxy commit run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) - name: Cache .cache/pip - uses: actions/cache@v1 + uses: actions/cache@v1.2.0 id: cache-pip with: path: ~/.cache/pip @@ -151,7 +155,7 @@ jobs: - name: Determine latest galaxy commit run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) - name: Cache .cache/pip - uses: actions/cache@v1 + uses: actions/cache@v1.2.0 id: cache-pip with: path: ~/.cache/pip @@ -201,13 +205,13 @@ jobs: - name: Determine latest galaxy commit run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) - name: Cache .cache/pip - uses: actions/cache@v1 + uses: actions/cache@v1.2.0 id: cache-pip with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} - name: Cache .planemo - uses: actions/cache@v1 + uses: actions/cache@v1.2.0 id: cache-planemo with: path: ~/.planemo @@ -256,7 +260,7 @@ jobs: run: | mkdir upload mv tool_test_output.json tool_test_output.html upload/ - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v2.0.1 with: name: 'Tool test output ${{ matrix.chunk }}' path: upload @@ -284,7 +288,7 @@ jobs: - name: Determine latest galaxy commit run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat "artifacts/Workflow artifacts/galaxy.sha") - name: Cache .cache/pip - uses: actions/cache@v1 + uses: actions/cache@v1.2.0 id: cache-pip with: path: ~/.cache/pip @@ -301,7 +305,7 @@ jobs: run: | mkdir upload mv tool_test_output.json tool_test_output.html upload/ - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v2.0.1 with: name: 'All tool test results' path: upload @@ -335,7 +339,7 @@ jobs: - name: Determine latest galaxy commit run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) - name: Cache .cache/pip - uses: actions/cache@v1 + uses: actions/cache@v1.2.0 id: cache-pip with: path: ~/.cache/pip From 4e2af59c20d3e8b8f193a460366a3c44fa21ec32 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 28 May 2020 03:21:36 +0100 Subject: [PATCH 024/142] CI: Install `wheel` package to cache more packages And switch PR testing to Galaxy 20.05 too. --- .github/workflows/pr.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b9efd5c8..501b169f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -2,7 +2,7 @@ name: Galaxy Tool Linting and Tests for push and PR on: [push, pull_request] env: GALAXY_REPO: https://github.com/galaxyproject/galaxy - GALAXY_RELEASE: release_20.01 + GALAXY_RELEASE: release_20.05 jobs: # the setup job does two things: # 1. cache the pip cache and .planemo @@ -50,6 +50,10 @@ jobs: with: path: ~/.planemo key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} + # Install the `wheel` package so that when installing other packages which + # are not available as wheels, pip will build a wheel for them, which can be cached. + - name: Install wheel + run: pip install wheel - name: Install Planemo and flake8 run: pip install planemo flake8 - name: Fake a planemo run to update cache From 520344a0ceeda4a6618bbefa54697fb62b1c57e5 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 2 Jun 2020 20:49:28 +0100 Subject: [PATCH 025/142] Sync CI configuration with tools-devteam repo (#3055) Restore flake8-import-order linting, lost when moving from Travis to GitHub workflows. --- .github/workflows/pr.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 501b169f..b17b5379 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -55,8 +55,8 @@ jobs: - name: Install wheel run: pip install wheel - name: Install Planemo and flake8 - run: pip install planemo flake8 - - name: Fake a planemo run to update cache + run: pip install planemo flake8 flake8-import-order + - name: Fake a Planemo run to update cache if: steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' run: | touch tool.xml @@ -95,7 +95,7 @@ jobs: name: Workflow artifacts path: galaxy.sha - # planemo lint the changed repositories + # Planemo lint the changed repositories lint: name: Lint tools needs: setup @@ -134,7 +134,7 @@ jobs: planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; done < ../workflow_artifacts/changed_repositories.list - # flake8 of python scripts in the changed repositories + # flake8 of Python scripts in the changed repositories flake8: name: Lint Python scripts needs: setup @@ -165,15 +165,15 @@ jobs: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} - name: Install flake8 - run: pip install flake8 + run: pip install flake8 flake8-import-order - name: Flake8 run: | if [ -s ../workflow_artifacts/changed_repositories.list ]; then flake8 $(cat ../workflow_artifacts/changed_repositories.list) fi - # planemo test the changed repositories, each chunk creates an artifact - # containing html and json reports for the executed tests + # Planemo test the changed repositories, each chunk creates an artifact + # containing HTML and JSON reports for the executed tests test: name: Test tools # This job runs on Linux From 559a16780b1e192e38802cccfc0f0f89e046c4f5 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 10 Jun 2020 13:02:36 +0200 Subject: [PATCH 026/142] add lintr to PR workflow --- .github/workflows/pr.yaml | 58 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b17b5379..4290db00 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -83,7 +83,7 @@ jobs: if: github.event_name == 'pull_request' run: echo ::set-env name=COMMIT_RANGE::"HEAD~.." - name: Planemo ci_find_repos - run: planemo ci_find_repos --changed_in_commit_range ${{ env.COMMIT_RANGE }} --exclude packages --exclude deprecated --output changed_repositories.list + run: planemo ci_find_repos --changed_in_commit_range ${{ env.COMMIT_RANGE }} --exclude packages --exclude deprecated --exclude_from .tt_skip --exclude_from .tt_biocontainer_skip --output changed_repositories.list - name: Show repo list run: cat changed_repositories.list - uses: actions/upload-artifact@v2.0.1 @@ -172,13 +172,63 @@ jobs: flake8 $(cat ../workflow_artifacts/changed_repositories.list) fi + lintr: + name: Lint R scripts + needs: setup + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + # checkout the repository to master + # and use it as the current working directory + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/download-artifact@v2 + with: + name: Workflow artifacts + path: ../workflow_artifacts/ + - uses: r-lib/actions/setup-r@master + - name: Cache R packages + uses: actions/cache@v1 + with: + path: ${{ env.R_LIBS_USER }} + key: lintr-rpackages + - name: Install non-R lintr dependencies + run: sudo apt-get install libcurl4-openssl-dev + - name: Install lintr + run: | + install.packages('remotes') + remotes::install_cran("lintr") + shell: Rscript {0} + - name: lintr + run: | + library(lintr) + linters <- with_defaults(line_length_linter = NULL) + con = file("../workflow_artifacts/changed_repositories.list", "r") + lnt = c() + while ( TRUE ) { + repo = readLines(con, n = 1) + if ( length(repo) == 0 ) { + break + } + lnt <- c(lnt, lint_dir(repo, relative_path=T, linters=linters)) + } + if(length(lnt) > 0){ + for(l in lnt){ + write(paste(paste(l$filename, l$line_number, l$column_number, sep=":"), l$message), stderr()) + } + quit(status = 1) + } + shell: Rscript {0} + # Planemo test the changed repositories, each chunk creates an artifact # containing HTML and JSON reports for the executed tests test: name: Test tools # This job runs on Linux runs-on: ubuntu-latest - needs: [lint,flake8] + needs: setup strategy: fail-fast: false matrix: @@ -241,7 +291,7 @@ jobs: - name: Planemo test tools run: | if [ -s changed_tools_chunk.list ]; then - PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --biocontainers --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json tool_test_output.json $(cat changed_tools_chunk.list) || true + PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --biocontainers --no_dependency_resolution --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json tool_test_output.json $(cat changed_tools_chunk.list) || true docker system prune --all --force --volumes || true elif [ -s changed_repositories_chunk.list ]; then while read -r DIR; do @@ -250,7 +300,7 @@ jobs: else TESTPATH="$DIR" fi - PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --biocontainers --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$DIR"/tool_test_output.json "$TESTPATH" || true + PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --biocontainers --no_dependency_resolution --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$DIR"/tool_test_output.json "$TESTPATH" || true docker system prune --all --force --volumes || true done < changed_repositories_chunk.list else From 88cd13aaaa44db46e5f904ccb1af917bb0ae4f85 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 10 Jun 2020 15:27:46 +0200 Subject: [PATCH 027/142] update cache action --- .github/workflows/pr.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4290db00..eb96e2f5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -39,13 +39,13 @@ jobs: name: Workflow artifacts path: galaxy.sha - name: Cache .cache/pip - uses: actions/cache@v1.2.0 + uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} - name: Cache .planemo - uses: actions/cache@v1.2.0 + uses: actions/cache@v2 id: cache-planemo with: path: ~/.planemo @@ -120,7 +120,7 @@ jobs: - name: Determine latest galaxy commit run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) - name: Cache .cache/pip - uses: actions/cache@v1.2.0 + uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip @@ -159,7 +159,7 @@ jobs: - name: Determine latest galaxy commit run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) - name: Cache .cache/pip - uses: actions/cache@v1.2.0 + uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip @@ -190,7 +190,7 @@ jobs: path: ../workflow_artifacts/ - uses: r-lib/actions/setup-r@master - name: Cache R packages - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: ${{ env.R_LIBS_USER }} key: lintr-rpackages @@ -259,13 +259,13 @@ jobs: - name: Determine latest galaxy commit run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) - name: Cache .cache/pip - uses: actions/cache@v1.2.0 + uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} - name: Cache .planemo - uses: actions/cache@v1.2.0 + uses: actions/cache@v2 id: cache-planemo with: path: ~/.planemo @@ -342,7 +342,7 @@ jobs: - name: Determine latest galaxy commit run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat "artifacts/Workflow artifacts/galaxy.sha") - name: Cache .cache/pip - uses: actions/cache@v1.2.0 + uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip @@ -393,7 +393,7 @@ jobs: - name: Determine latest galaxy commit run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) - name: Cache .cache/pip - uses: actions/cache@v1.2.0 + uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip From bff5893a7da6bba12fee2e54748f5b2d8d5c820b Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 10 Jun 2020 16:57:00 +0200 Subject: [PATCH 028/142] try failfast --- .github/workflows/pr.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index eb96e2f5..eb425b03 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -176,8 +176,6 @@ jobs: name: Lint R scripts needs: setup runs-on: ubuntu-latest - strategy: - fail-fast: false steps: # checkout the repository to master # and use it as the current working directory From c097df36454bf4c4c9bc55c9e3d487eb6371df43 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 10 Jun 2020 17:06:24 +0200 Subject: [PATCH 029/142] remove duplicated upload step --- .github/workflows/pr.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index eb425b03..22574a6f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -90,10 +90,6 @@ jobs: with: name: Workflow artifacts path: changed_repositories.list - - uses: actions/upload-artifact@v2.0.1 - with: - name: Workflow artifacts - path: galaxy.sha # Planemo lint the changed repositories lint: From e174a719581d7986bfff375050e67561d8749244 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 10 Jun 2020 19:27:23 +0200 Subject: [PATCH 030/142] Restructure R linting Co-authored-by: Nicola Soranzo --- .github/workflows/pr.yaml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 22574a6f..c81e857d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -199,21 +199,23 @@ jobs: run: | library(lintr) linters <- with_defaults(line_length_linter = NULL) - con = file("../workflow_artifacts/changed_repositories.list", "r") - lnt = c() - while ( TRUE ) { - repo = readLines(con, n = 1) - if ( length(repo) == 0 ) { + con <- file("../workflow_artifacts/changed_repositories.list", "r") + status <- 0 + while (TRUE) { + repo <- readLines(con, n = 1) + if (length(repo) == 0) { break } - lnt <- c(lnt, lint_dir(repo, relative_path=T, linters=linters)) - } - if(length(lnt) > 0){ - for(l in lnt){ - write(paste(paste(l$filename, l$line_number, l$column_number, sep=":"), l$message), stderr()) + lnt <- lint_dir(repo, relative_path=T, linters=linters) + if (length(lnt) > 0) { + status <- 1 + for (l in lnt) { + rel_path <- paste(repo, l$filename, sep="/") + write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message), stderr()) + } } - quit(status = 1) } + quit(status = status) shell: Rscript {0} # Planemo test the changed repositories, each chunk creates an artifact From 88f424d08a8d942d1a3ac23780c7d883bca18202 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Thu, 11 Jun 2020 12:04:34 +0200 Subject: [PATCH 031/142] Version R cache Co-authored-by: Nicola Soranzo --- .github/workflows/pr.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c81e857d..9db8003c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -172,6 +172,9 @@ jobs: name: Lint R scripts needs: setup runs-on: ubuntu-latest + strategy: + matrix: + r-version: [4.0.1] steps: # checkout the repository to master # and use it as the current working directory @@ -183,11 +186,13 @@ jobs: name: Workflow artifacts path: ../workflow_artifacts/ - uses: r-lib/actions/setup-r@master + with: + r-version: ${{ matrix.r-version }} - name: Cache R packages uses: actions/cache@v2 with: path: ${{ env.R_LIBS_USER }} - key: lintr-rpackages + key: r_cache_${{ matrix.r-version }} - name: Install non-R lintr dependencies run: sudo apt-get install libcurl4-openssl-dev - name: Install lintr From ebea9e64e56befcff8ab91bcbdfb0715e6b6d757 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 7 Jul 2020 14:41:59 +0200 Subject: [PATCH 032/142] make deployment depend on linters as well --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9db8003c..6782a9ca 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -374,7 +374,7 @@ jobs: # deploy the tools to the toolsheds (first TTS for testing) deploy: name: Deploy - needs: combine_outputs + needs: [lint,flake8,lintr,combine_outputs] strategy: matrix: python-version: [3.7] From f095719c4c478dc324fd68ce1ef6d18bda44081f Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 24 Jul 2020 19:23:59 +0100 Subject: [PATCH 033/142] Deploy on TS even if it failed on TTS --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6782a9ca..277093f8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -408,6 +408,7 @@ jobs: while read -r DIR; do planemo shed_update --shed_target testtoolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; done < ../workflow_artifacts/changed_repositories.list + continue-on-error: true - name: Deploy on toolshed env: SHED_KEY: ${{ secrets.ts_api_key }} From 580580cce1a9b56b834c3ba2d075416936f837d1 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 4 Aug 2020 23:35:36 +0200 Subject: [PATCH 034/142] dont skip but test without container when in .tt_biocontainer_skip --- .github/workflows/pr.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 277093f8..38d2b0fd 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -83,7 +83,7 @@ jobs: if: github.event_name == 'pull_request' run: echo ::set-env name=COMMIT_RANGE::"HEAD~.." - name: Planemo ci_find_repos - run: planemo ci_find_repos --changed_in_commit_range ${{ env.COMMIT_RANGE }} --exclude packages --exclude deprecated --exclude_from .tt_skip --exclude_from .tt_biocontainer_skip --output changed_repositories.list + run: planemo ci_find_repos --changed_in_commit_range ${{ env.COMMIT_RANGE }} --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_repositories.list - name: Show repo list run: cat changed_repositories.list - uses: actions/upload-artifact@v2.0.1 @@ -291,8 +291,13 @@ jobs: run: cat changed_tools_chunk.list changed_repositories_chunk.list - name: Planemo test tools run: | + if grep -lqf .tt_biocontainer_skip changed_tools_chunk.list changed_repositories_chunk.list; then + PLANEMO_OPTIONS="" + else + PLANEMO_OPTIONS="--biocontainers --no_dependency_resolution --no_conda_auto_init" + fi if [ -s changed_tools_chunk.list ]; then - PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --biocontainers --no_dependency_resolution --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json tool_test_output.json $(cat changed_tools_chunk.list) || true + PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json tool_test_output.json $(cat changed_tools_chunk.list) || true docker system prune --all --force --volumes || true elif [ -s changed_repositories_chunk.list ]; then while read -r DIR; do @@ -301,7 +306,7 @@ jobs: else TESTPATH="$DIR" fi - PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --biocontainers --no_dependency_resolution --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$DIR"/tool_test_output.json "$TESTPATH" || true + PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$DIR"/tool_test_output.json "$TESTPATH" || true docker system prune --all --force --volumes || true done < changed_repositories_chunk.list else From 8a94e7037d11d06d8145070da66d4f852a1c45fa Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 22 Sep 2020 19:19:13 +0200 Subject: [PATCH 035/142] Update galaxy release for PRs as well --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 38d2b0fd..7149e3e0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -2,7 +2,7 @@ name: Galaxy Tool Linting and Tests for push and PR on: [push, pull_request] env: GALAXY_REPO: https://github.com/galaxyproject/galaxy - GALAXY_RELEASE: release_20.05 + GALAXY_RELEASE: release_20.09 jobs: # the setup job does two things: # 1. cache the pip cache and .planemo From c427a910cc97a23d046b06b6e133a934dd1003ff Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 1 Oct 2020 23:58:45 +0100 Subject: [PATCH 036/142] Don't try to deploy on forks --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7149e3e0..6b63927f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -384,7 +384,7 @@ jobs: matrix: python-version: [3.7] runs-on: ubuntu-latest - if: github.ref == 'refs/heads/master' + if: github.ref == 'refs/heads/master' && github.repository_owner == 'galaxyproject' steps: - uses: actions/checkout@v2 with: From 386dafd442bf9a7dd491c309a7e068eadf2682cd Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 13 Oct 2020 13:03:10 +0200 Subject: [PATCH 037/142] adapt PR workflow to new way of setting env vars currently we see: ``` Warning: The `set-env` command is deprecated and will be disabled soon. Please upgrade to using Environment Files. For more information see: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/ ``` --- .github/workflows/pr.yaml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6b63927f..31eedec3 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -31,9 +31,9 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Determine latest galaxy commit - run: echo ::set-env name=GALAXY_HEAD_SHA::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1) + run: echo "GALAXY_HEAD_SHA=$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1)" >> $GITHUB_ENV - name: Save latest galaxy commit to artifact file - run: echo ${{ env.GALAXY_HEAD_SHA }} > galaxy.sha + run: echo $GALAXY_HEAD_SHA > galaxy.sha - uses: actions/upload-artifact@v2 with: name: Workflow artifacts @@ -43,13 +43,13 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA - name: Cache .planemo uses: actions/cache@v2 id: cache-planemo with: path: ~/.planemo - key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} + key: planemo_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA # Install the `wheel` package so that when installing other packages which # are not available as wheels, pip will build a wheel for them, which can be cached. - name: Install wheel @@ -75,15 +75,15 @@ jobs: if: github.ref != 'refs/heads/master' && github.event_name == 'push' run: | git fetch origin master - echo ::set-env name=COMMIT_RANGE::"origin/master..." + echo "COMMIT_RANGE=origin/master..." >> $GITHUB_ENV - name: Set commit range (push to the master branch, e.g. merge) if: github.ref == 'refs/heads/master' && github.event_name == 'push' - run: echo ::set-env name=COMMIT_RANGE::${{ github.event.before }}.."" + run: echo "COMMIT_RANGE=${{ github.event.before }}.." >> $GITHUB_ENV - name: Set commit range (pull request) if: github.event_name == 'pull_request' - run: echo ::set-env name=COMMIT_RANGE::"HEAD~.." + run: echo "COMMIT_RANGE=HEAD~.." >> $GITHUB_ENV - name: Planemo ci_find_repos - run: planemo ci_find_repos --changed_in_commit_range ${{ env.COMMIT_RANGE }} --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_repositories.list + run: planemo ci_find_repos --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_repositories.list - name: Show repo list run: cat changed_repositories.list - uses: actions/upload-artifact@v2.0.1 @@ -114,13 +114,13 @@ jobs: name: Workflow artifacts path: ../workflow_artifacts/ - name: Determine latest galaxy commit - run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) + run: echo "GALAXY_HEAD_SHA=$(cat ../workflow_artifacts/galaxy.sha)" >> $GITHUB_ENV - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA - name: Install Planemo run: pip install planemo - name: Planemo lint @@ -153,13 +153,13 @@ jobs: name: Workflow artifacts path: ../workflow_artifacts/ - name: Determine latest galaxy commit - run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) + run: echo "GALAXY_HEAD_SHA=$(cat ../workflow_artifacts/galaxy.sha)" >> $GITHUB_ENV - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} + key: pip_cache_py_${{ matrix.python-version }}_$gxy_$GALAXY_HEAD_SHA - name: Install flake8 run: pip install flake8 flake8-import-order - name: Flake8 @@ -258,19 +258,19 @@ jobs: name: Workflow artifacts path: ../workflow_artifacts/ - name: Determine latest galaxy commit - run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) + run: echo "GALAXY_HEAD_SHA=$(cat ../workflow_artifacts/galaxy.sha)" >> $GITHUB_ENV - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA - name: Cache .planemo uses: actions/cache@v2 id: cache-planemo with: path: ~/.planemo - key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} + key: planemo_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA - name: Install Planemo run: pip install planemo - name: Planemo ci_find_tools @@ -346,13 +346,13 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Determine latest galaxy commit - run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat "artifacts/Workflow artifacts/galaxy.sha") + run: echo "GALAXY_HEAD_SHA=$(cat ../workflow_artifacts/galaxy.sha)" >> $GITHUB_ENV - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA - name: Install Planemo run: pip install planemo - name: Install jq @@ -397,13 +397,13 @@ jobs: name: Workflow artifacts path: ../workflow_artifacts/ - name: Determine latest galaxy commit - run: echo ::set-env name=GALAXY_HEAD_SHA::$(cat ../workflow_artifacts/galaxy.sha) + run: echo "GALAXY_HEAD_SHA=$(cat ../workflow_artifacts/galaxy.sha)" >> $GITHUB_ENV - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ env.GALAXY_HEAD_SHA }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA - name: Install Planemo run: pip install planemo - name: Deploy on testtoolshed From 007f7b1704b09128d9fa68c434f61f5debb6bc28 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 20 Oct 2020 16:57:08 +0100 Subject: [PATCH 038/142] Fix typo in ``pr.yaml`` --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 31eedec3..bba9f6cf 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -159,7 +159,7 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_$gxy_$GALAXY_HEAD_SHA + key: pip_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA - name: Install flake8 run: pip install flake8 flake8-import-order - name: Flake8 From 0b57ce2827adc9e9c0a8b0c6fa9eac9dfa946f65 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 16 Oct 2020 12:52:06 +0200 Subject: [PATCH 039/142] group tools for planemo tesing - simplifies the logic a bit since tools and data managers don't need to be treated separately. - check if every path in find_tools is in the output of find_repos, i.e. check for missing .shed.yml - all linters run now on the result of ci_find_repos - tool testing uses ci_find_tools as suggested here https://github.com/galaxyproject/tools-iuc/pull/3248#issuecomment-708993697 --- .github/workflows/pr.yaml | 69 +++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index bba9f6cf..7ce9a0b7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -86,10 +86,16 @@ jobs: run: planemo ci_find_repos --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_repositories.list - name: Show repo list run: cat changed_repositories.list + - name: Planemo ci_find_tools + run: planemo ci_find_tools --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_tools.list + - name: Show tool list + run: cat changed_tools.list - uses: actions/upload-artifact@v2.0.1 with: name: Workflow artifacts - path: changed_repositories.list + path: | + changed_repositories.list + changed_tools.list # Planemo lint the changed repositories lint: @@ -129,6 +135,16 @@ jobs: while read -r DIR; do planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; done < ../workflow_artifacts/changed_repositories.list + - name: Check if each tool is in the list of repositories + run: | + set -e + while read -r DIR; do + dn=$(dirname $DIR); + if ! grep -lq $dn ../workflow_artifacts/changed_repos.list; then + >&2 echo "$DIR not in reository list: .shed.yml file missing" + exit 1 + fi + done < ../workflow_artifacts/changed_tools.list # flake8 of Python scripts in the changed repositories flake8: @@ -274,43 +290,24 @@ jobs: - name: Install Planemo run: pip install planemo - name: Planemo ci_find_tools - run: | - touch changed_repositories_chunk.list changed_tools_chunk.list - if [ -s ../workflow_artifacts/changed_repositories.list ]; then - if [ $(wc -l < ../workflow_artifacts/changed_repositories.list) -eq 1 ]; then - planemo ci_find_tools --chunk_count 4 --chunk ${{ matrix.chunk }} \ - --output changed_tools_chunk.list \ - $(cat ../workflow_artifacts/changed_repositories.list) - else - planemo ci_find_repos --chunk_count 4 --chunk ${{ matrix.chunk }} \ - --output changed_repositories_chunk.list \ - $(cat ../workflow_artifacts/changed_repositories.list) - fi - fi - - name: Show changed tools/repositories chunk list - run: cat changed_tools_chunk.list changed_repositories_chunk.list + run: planemo ci_find_tools --chunk_count 4 --chunk ${{ matrix.chunk }} --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_tools_chunk.list + - name: Show tool list + run: cat changed_tools_chunk.list - name: Planemo test tools run: | - if grep -lqf .tt_biocontainer_skip changed_tools_chunk.list changed_repositories_chunk.list; then - PLANEMO_OPTIONS="" - else - PLANEMO_OPTIONS="--biocontainers --no_dependency_resolution --no_conda_auto_init" - fi - if [ -s changed_tools_chunk.list ]; then - PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json tool_test_output.json $(cat changed_tools_chunk.list) || true - docker system prune --all --force --volumes || true - elif [ -s changed_repositories_chunk.list ]; then - while read -r DIR; do - if [[ "$DIR" =~ ^data_managers.* ]]; then - TESTPATH=$(planemo ci_find_tools "$DIR") - else - TESTPATH="$DIR" - fi - PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$DIR"/tool_test_output.json "$TESTPATH" || true - docker system prune --all --force --volumes || true - done < changed_repositories_chunk.list - else - echo '{"tests":[]}' > tool_test_output.json + mkdir json_output/ + while read -r TOOLS; do + if grep -lqf .tt_biocontainer_skip <(echo $TOOLS); then + PLANEMO_OPTIONS="" + else + PLANEMO_OPTIONS="--biocontainers --no_dependency_resolution --no_conda_auto_init" + fi + json=$(mktemp -u -p json_output --suff .json) + PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --no_conda_auto_init --galaxy_branch $GALAXY_RELEASE --biocontainers --no_dependency_resolution --galaxy_python_version ${{ matrix.python-version }} --test_output_json $json $TOOLS || true + docker system prune --all --force --volumes || true + done < changed_tools_chunk.list + if [ ! -s changed_tools_chunk.list ]; then + echo '{"tests":[]}' > $(mktemp -u -p json_output --suff .json) fi - name: Merge tool_test_output.json files run: find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + From f3572f1fd65ad6a51fd5f91815a1272c0b73a133 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 2 Nov 2020 15:03:32 +0100 Subject: [PATCH 040/142] use latest v2 upload action --- .github/workflows/pr.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7ce9a0b7..8c029e91 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -90,7 +90,7 @@ jobs: run: planemo ci_find_tools --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_tools.list - name: Show tool list run: cat changed_tools.list - - uses: actions/upload-artifact@v2.0.1 + - uses: actions/upload-artifact@v2 with: name: Workflow artifacts path: | @@ -317,7 +317,7 @@ jobs: run: | mkdir upload mv tool_test_output.json tool_test_output.html upload/ - - uses: actions/upload-artifact@v2.0.1 + - uses: actions/upload-artifact@v2 with: name: 'Tool test output ${{ matrix.chunk }}' path: upload @@ -362,7 +362,7 @@ jobs: run: | mkdir upload mv tool_test_output.json tool_test_output.html upload/ - - uses: actions/upload-artifact@v2.0.1 + - uses: actions/upload-artifact@v2 with: name: 'All tool test results' path: upload From bede310359e76eca7091e87800e89437fa5226b5 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 12 Nov 2020 15:49:05 +0100 Subject: [PATCH 041/142] fix ci_find_tools call in test --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8c029e91..50efec74 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -290,7 +290,7 @@ jobs: - name: Install Planemo run: pip install planemo - name: Planemo ci_find_tools - run: planemo ci_find_tools --chunk_count 4 --chunk ${{ matrix.chunk }} --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_tools_chunk.list + run: planemo ci_find_tools --chunk_count 4 --chunk ${{ matrix.chunk }} --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_tools_chunk.list $(cat ../workflow_artifacts/changed_repositories.list) - name: Show tool list run: cat changed_tools_chunk.list - name: Planemo test tools From c8c5a51533ee008e9f2c814a5eb7fe124d5d023c Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 13 Nov 2020 16:47:47 +0100 Subject: [PATCH 042/142] make chunks dynamic in PR workflow --- .github/workflows/pr.yaml | 91 +++++++++++++++++++++++---------------- 1 file changed, 54 insertions(+), 37 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 50efec74..f959aa50 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,6 +14,8 @@ jobs: setup: name: Setup cache and determine changed repositories runs-on: ubuntu-latest + outputs: + chunks: ${{ steps.get-chunks.outputs.chunks }} strategy: matrix: python-version: [3.7] @@ -86,16 +88,22 @@ jobs: run: planemo ci_find_repos --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_repositories.list - name: Show repo list run: cat changed_repositories.list - - name: Planemo ci_find_tools - run: planemo ci_find_tools --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_tools.list - - name: Show tool list - run: cat changed_tools.list - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v2.0.1 with: name: Workflow artifacts - path: | - changed_repositories.list - changed_tools.list + path: changed_repositories.list + - name: Get number of changed tools and compute number of chunks + id: get-chunks + run: | + planemo ci_find_tools --output changed_tools.list $(cat changed_repositories.list) + ntools=$(wc -l changed_tools.list) + log=$(printf "%.0f" $(echo "l($ntools+1)/l(2)" | bc -l)) + nchunks=$(echo "4 * $log" | bc) + if [[ $nchunks -gt 39 ]]; then + nchunks="39" + fi + chunk_list=$(echo "["$(seq -s ", " 0 $nchunks)"]") + echo "::set-output name=chunks::$chunk_list" # Planemo lint the changed repositories lint: @@ -135,16 +143,6 @@ jobs: while read -r DIR; do planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; done < ../workflow_artifacts/changed_repositories.list - - name: Check if each tool is in the list of repositories - run: | - set -e - while read -r DIR; do - dn=$(dirname $DIR); - if ! grep -lq $dn ../workflow_artifacts/changed_repos.list; then - >&2 echo "$DIR not in reository list: .shed.yml file missing" - exit 1 - fi - done < ../workflow_artifacts/changed_tools.list # flake8 of Python scripts in the changed repositories flake8: @@ -249,7 +247,7 @@ jobs: strategy: fail-fast: false matrix: - chunk: [0, 1, 2, 3] + chunk: ${{ fromJson(needs.setup.outputs.chunks) }} python-version: [3.7] services: postgres: @@ -290,24 +288,43 @@ jobs: - name: Install Planemo run: pip install planemo - name: Planemo ci_find_tools - run: planemo ci_find_tools --chunk_count 4 --chunk ${{ matrix.chunk }} --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_tools_chunk.list $(cat ../workflow_artifacts/changed_repositories.list) - - name: Show tool list - run: cat changed_tools_chunk.list + run: | + touch changed_repositories_chunk.list changed_tools_chunk.list + if [ -s ../workflow_artifacts/changed_repositories.list ]; then + if [ $(wc -l < ../workflow_artifacts/changed_repositories.list) -eq 1 ]; then + planemo ci_find_tools --chunk_count 4 --chunk ${{ matrix.chunk }} \ + --output changed_tools_chunk.list \ + $(cat ../workflow_artifacts/changed_repositories.list) + else + planemo ci_find_repos --chunk_count 4 --chunk ${{ matrix.chunk }} \ + --output changed_repositories_chunk.list \ + $(cat ../workflow_artifacts/changed_repositories.list) + fi + fi + - name: Show changed tools/repositories chunk list + run: cat changed_tools_chunk.list changed_repositories_chunk.list - name: Planemo test tools run: | - mkdir json_output/ - while read -r TOOLS; do - if grep -lqf .tt_biocontainer_skip <(echo $TOOLS); then - PLANEMO_OPTIONS="" - else - PLANEMO_OPTIONS="--biocontainers --no_dependency_resolution --no_conda_auto_init" - fi - json=$(mktemp -u -p json_output --suff .json) - PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --no_conda_auto_init --galaxy_branch $GALAXY_RELEASE --biocontainers --no_dependency_resolution --galaxy_python_version ${{ matrix.python-version }} --test_output_json $json $TOOLS || true - docker system prune --all --force --volumes || true - done < changed_tools_chunk.list - if [ ! -s changed_tools_chunk.list ]; then - echo '{"tests":[]}' > $(mktemp -u -p json_output --suff .json) + if grep -lqf .tt_biocontainer_skip changed_tools_chunk.list changed_repositories_chunk.list; then + PLANEMO_OPTIONS="" + else + PLANEMO_OPTIONS="--biocontainers --no_dependency_resolution --no_conda_auto_init" + fi + if [ -s changed_tools_chunk.list ]; then + PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json tool_test_output.json $(cat changed_tools_chunk.list) || true + docker system prune --all --force --volumes || true + elif [ -s changed_repositories_chunk.list ]; then + while read -r DIR; do + if [[ "$DIR" =~ ^data_managers.* ]]; then + TESTPATH=$(planemo ci_find_tools "$DIR") + else + TESTPATH="$DIR" + fi + PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$DIR"/tool_test_output.json "$TESTPATH" || true + docker system prune --all --force --volumes || true + done < changed_repositories_chunk.list + else + echo '{"tests":[]}' > tool_test_output.json fi - name: Merge tool_test_output.json files run: find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + @@ -317,7 +334,7 @@ jobs: run: | mkdir upload mv tool_test_output.json tool_test_output.html upload/ - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v2.0.1 with: name: 'Tool test output ${{ matrix.chunk }}' path: upload @@ -362,7 +379,7 @@ jobs: run: | mkdir upload mv tool_test_output.json tool_test_output.html upload/ - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v2.0.1 with: name: 'All tool test results' path: upload From 7e1cea4385cd3de31152174cd55e98f7b851b7ff Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 15 Nov 2020 14:33:18 +0100 Subject: [PATCH 043/142] switch to linear foo to compute number of chunks --- .github/workflows/pr.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f959aa50..03990864 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -3,6 +3,7 @@ on: [push, pull_request] env: GALAXY_REPO: https://github.com/galaxyproject/galaxy GALAXY_RELEASE: release_20.09 + MAX_CHUNKS: 4 jobs: # the setup job does two things: # 1. cache the pip cache and .planemo @@ -96,13 +97,11 @@ jobs: id: get-chunks run: | planemo ci_find_tools --output changed_tools.list $(cat changed_repositories.list) - ntools=$(wc -l changed_tools.list) - log=$(printf "%.0f" $(echo "l($ntools+1)/l(2)" | bc -l)) - nchunks=$(echo "4 * $log" | bc) - if [[ $nchunks -gt 39 ]]; then - nchunks="39" + nchunks=$(wc -l changed_tools.list) + if [[ $nchunks -gt $MAX_CHUNKS ]]; then + nchunks=$MAX_CHUNKS fi - chunk_list=$(echo "["$(seq -s ", " 0 $nchunks)"]") + chunk_list=$(echo "["$(seq -s ", " 1 $nchunks)"]") echo "::set-output name=chunks::$chunk_list" # Planemo lint the changed repositories From c2da83b4ef6ea1dfc9b7c7b99d511d4bc1fe4961 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 16 Nov 2020 13:38:38 +0100 Subject: [PATCH 044/142] improved bash commands Co-authored-by: Nicola Soranzo --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 03990864..2d06f71f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -98,10 +98,10 @@ jobs: run: | planemo ci_find_tools --output changed_tools.list $(cat changed_repositories.list) nchunks=$(wc -l changed_tools.list) - if [[ $nchunks -gt $MAX_CHUNKS ]]; then + if [ "$nchunks" -gt "$MAX_CHUNKS" ]; then nchunks=$MAX_CHUNKS fi - chunk_list=$(echo "["$(seq -s ", " 1 $nchunks)"]") + chunk_list='['$(seq -s ", " 0 $(($nchunks - 1)))']' echo "::set-output name=chunks::$chunk_list" # Planemo lint the changed repositories From 2df8952cae41f7b00815d5bc1189b74a60d40197 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 16 Nov 2020 14:23:23 +0100 Subject: [PATCH 045/142] fix wc call Co-authored-by: Nicola Soranzo --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2d06f71f..e635a848 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -97,7 +97,7 @@ jobs: id: get-chunks run: | planemo ci_find_tools --output changed_tools.list $(cat changed_repositories.list) - nchunks=$(wc -l changed_tools.list) + nchunks=$(wc -l < changed_tools.list) if [ "$nchunks" -gt "$MAX_CHUNKS" ]; then nchunks=$MAX_CHUNKS fi From 69e40f633c33bb2c26a918d1f732c46c0f5212aa Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 16 Nov 2020 17:40:39 +0000 Subject: [PATCH 046/142] Pass the calculated number of chunks to planemo (#3306) --- .github/workflows/pr.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e635a848..3fd1371f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,7 +16,8 @@ jobs: name: Setup cache and determine changed repositories runs-on: ubuntu-latest outputs: - chunks: ${{ steps.get-chunks.outputs.chunks }} + nchunks: ${{ steps.get-chunks.outputs.nchunks }} + chunk_list: ${{ steps.get-chunks.outputs.chunk_list }} strategy: matrix: python-version: [3.7] @@ -101,8 +102,8 @@ jobs: if [ "$nchunks" -gt "$MAX_CHUNKS" ]; then nchunks=$MAX_CHUNKS fi - chunk_list='['$(seq -s ", " 0 $(($nchunks - 1)))']' - echo "::set-output name=chunks::$chunk_list" + echo "::set-output name=nchunks::$nchunks" + echo "::set-output name=chunk_list::'['$(seq -s ", " 0 $(($nchunks - 1)))']'" # Planemo lint the changed repositories lint: @@ -246,7 +247,7 @@ jobs: strategy: fail-fast: false matrix: - chunk: ${{ fromJson(needs.setup.outputs.chunks) }} + chunk: ${{ fromJson(needs.setup.outputs.chunk_list) }} python-version: [3.7] services: postgres: @@ -291,11 +292,11 @@ jobs: touch changed_repositories_chunk.list changed_tools_chunk.list if [ -s ../workflow_artifacts/changed_repositories.list ]; then if [ $(wc -l < ../workflow_artifacts/changed_repositories.list) -eq 1 ]; then - planemo ci_find_tools --chunk_count 4 --chunk ${{ matrix.chunk }} \ + planemo ci_find_tools --chunk_count ${{ needs.setup.outputs.nchunks }} --chunk ${{ matrix.chunk }} \ --output changed_tools_chunk.list \ $(cat ../workflow_artifacts/changed_repositories.list) else - planemo ci_find_repos --chunk_count 4 --chunk ${{ matrix.chunk }} \ + planemo ci_find_repos --chunk_count ${{ needs.setup.outputs.nchunks }} --chunk ${{ matrix.chunk }} \ --output changed_repositories_chunk.list \ $(cat ../workflow_artifacts/changed_repositories.list) fi From 606e8946a93c0ed78a3813af06febb867fbce0fd Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 18 Nov 2020 10:28:46 +0100 Subject: [PATCH 047/142] fix RP workflow by removing quotes --- .github/workflows/pr.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3fd1371f..19cfe489 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -103,7 +103,10 @@ jobs: nchunks=$MAX_CHUNKS fi echo "::set-output name=nchunks::$nchunks" - echo "::set-output name=chunk_list::'['$(seq -s ", " 0 $(($nchunks - 1)))']'" + echo "::set-output name=chunk_list::[$(seq -s ", " 0 $(($nchunks - 1)))]" + - name: Show chunks + run: | + echo 'Using ${{ steps.get-chunks.outputs.nchunks }} chunks (${{ steps.get-chunks.outputs.chunk_list }})' # Planemo lint the changed repositories lint: From 6e34dd020af596cebc95173b241409cd981b4036 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 20 Nov 2020 16:49:29 +0000 Subject: [PATCH 048/142] Rename secrets to uppercase When a secret is added (Settings -> Secrets -> New repository secret) on GitHub, the secret name is changed to uppercase in the secrets list, even if it is actually lower/mixed case. I find this confusing. N.B.: THIS CHANGE REQUIRES REMOVING AND RE-ADDING THE 2 SECRETS USING THE NEW UPPERCASE NAME! --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 19cfe489..dcbbb3e2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -425,7 +425,7 @@ jobs: run: pip install planemo - name: Deploy on testtoolshed env: - SHED_KEY: ${{ secrets.tts_api_key }} + SHED_KEY: ${{ secrets.TTS_API_KEY }} run: | while read -r DIR; do planemo shed_update --shed_target testtoolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; @@ -433,7 +433,7 @@ jobs: continue-on-error: true - name: Deploy on toolshed env: - SHED_KEY: ${{ secrets.ts_api_key }} + SHED_KEY: ${{ secrets.TS_API_KEY }} run: | while read -r DIR; do planemo shed_update --shed_target toolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; From c502fa8224b34406b9ebdf9738dc9f379c5521cc Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 23 Nov 2020 14:17:15 +0100 Subject: [PATCH 049/142] Bash fixes Co-authored-by: Nicola Soranzo --- .github/workflows/pr.yaml | 85 +++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index dcbbb3e2..8d193983 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -90,23 +90,25 @@ jobs: run: planemo ci_find_repos --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_repositories.list - name: Show repo list run: cat changed_repositories.list - - uses: actions/upload-artifact@v2.0.1 + - name: Planemo ci_find_tools + run: planemo ci_find_tools --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_tools.list + - name: Show tool list + run: cat changed_tools.list + - uses: actions/upload-artifact@v2 with: name: Workflow artifacts - path: changed_repositories.list + path: | + changed_repositories.list + changed_tools.list - name: Get number of changed tools and compute number of chunks id: get-chunks run: | - planemo ci_find_tools --output changed_tools.list $(cat changed_repositories.list) nchunks=$(wc -l < changed_tools.list) if [ "$nchunks" -gt "$MAX_CHUNKS" ]; then nchunks=$MAX_CHUNKS fi echo "::set-output name=nchunks::$nchunks" - echo "::set-output name=chunk_list::[$(seq -s ", " 0 $(($nchunks - 1)))]" - - name: Show chunks - run: | - echo 'Using ${{ steps.get-chunks.outputs.nchunks }} chunks (${{ steps.get-chunks.outputs.chunk_list }})' + echo "::set-output name=chunk_list::'['$(seq -s ", " 0 $(($nchunks - 1)))']'" # Planemo lint the changed repositories lint: @@ -146,6 +148,16 @@ jobs: while read -r DIR; do planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; done < ../workflow_artifacts/changed_repositories.list + - name: Check if each changed tool is in the list of changed repositories + run: | + set -e + while read -r TOOL; do + dn=$(dirname $TOOL); + if ! grep -q "$dn" ../workflow_artifacts/changed_repos.list; then + echo "Tool $TOOL not in changed repositories list: .shed.yml file missing" >&2 + exit 1 + fi + done < ../workflow_artifacts/changed_tools.list # flake8 of Python scripts in the changed repositories flake8: @@ -292,52 +304,37 @@ jobs: run: pip install planemo - name: Planemo ci_find_tools run: | - touch changed_repositories_chunk.list changed_tools_chunk.list + touch changed_tools_chunk.list if [ -s ../workflow_artifacts/changed_repositories.list ]; then - if [ $(wc -l < ../workflow_artifacts/changed_repositories.list) -eq 1 ]; then - planemo ci_find_tools --chunk_count ${{ needs.setup.outputs.nchunks }} --chunk ${{ matrix.chunk }} \ - --output changed_tools_chunk.list \ - $(cat ../workflow_artifacts/changed_repositories.list) - else - planemo ci_find_repos --chunk_count ${{ needs.setup.outputs.nchunks }} --chunk ${{ matrix.chunk }} \ - --output changed_repositories_chunk.list \ - $(cat ../workflow_artifacts/changed_repositories.list) - fi + planemo ci_find_tools --chunk_count ${{ needs.setup.outputs.nchunks }} --chunk ${{ matrix.chunk }} --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_tools_chunk.list $(cat ../workflow_artifacts/changed_repositories.list) fi - - name: Show changed tools/repositories chunk list - run: cat changed_tools_chunk.list changed_repositories_chunk.list + - name: Show changed tools chunk list + run: cat changed_tools_chunk.list - name: Planemo test tools run: | - if grep -lqf .tt_biocontainer_skip changed_tools_chunk.list changed_repositories_chunk.list; then - PLANEMO_OPTIONS="" - else - PLANEMO_OPTIONS="--biocontainers --no_dependency_resolution --no_conda_auto_init" - fi - if [ -s changed_tools_chunk.list ]; then - PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json tool_test_output.json $(cat changed_tools_chunk.list) || true - docker system prune --all --force --volumes || true - elif [ -s changed_repositories_chunk.list ]; then - while read -r DIR; do - if [[ "$DIR" =~ ^data_managers.* ]]; then - TESTPATH=$(planemo ci_find_tools "$DIR") - else - TESTPATH="$DIR" - fi - PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$DIR"/tool_test_output.json "$TESTPATH" || true - docker system prune --all --force --volumes || true - done < changed_repositories_chunk.list - else - echo '{"tests":[]}' > tool_test_output.json + mkdir json_output + while read -r TOOL; do + if grep -q "$TOOL" .tt_biocontainer_skip; then + PLANEMO_OPTIONS="" + else + PLANEMO_OPTIONS="--biocontainers --no_dependency_resolution --no_conda_auto_init" + fi + json=$(mktemp -u -p json_output --suff .json) + PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$json" "$TOOL" || true + docker system prune --all --force --volumes || true + done < changed_tools_chunk.list + if [ ! -s changed_tools_chunk.list ]; then + echo '{"tests":[]}' > "$(mktemp -u -p json_output --suff .json)" fi - name: Merge tool_test_output.json files - run: find . -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + + run: planemo merge_test_reports json_output/*.json tool_test_output.json - name: Create tool_test_output.html run: planemo test_reports tool_test_output.json --test_output tool_test_output.html - name: Copy artifacts into place run: | mkdir upload mv tool_test_output.json tool_test_output.html upload/ - - uses: actions/upload-artifact@v2.0.1 + - uses: actions/upload-artifact@v2 with: name: 'Tool test output ${{ matrix.chunk }}' path: upload @@ -382,7 +379,7 @@ jobs: run: | mkdir upload mv tool_test_output.json tool_test_output.html upload/ - - uses: actions/upload-artifact@v2.0.1 + - uses: actions/upload-artifact@v2 with: name: 'All tool test results' path: upload @@ -425,7 +422,7 @@ jobs: run: pip install planemo - name: Deploy on testtoolshed env: - SHED_KEY: ${{ secrets.TTS_API_KEY }} + SHED_KEY: ${{ secrets.tts_api_key }} run: | while read -r DIR; do planemo shed_update --shed_target testtoolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; @@ -433,7 +430,7 @@ jobs: continue-on-error: true - name: Deploy on toolshed env: - SHED_KEY: ${{ secrets.TS_API_KEY }} + SHED_KEY: ${{ secrets.ts_api_key }} run: | while read -r DIR; do planemo shed_update --shed_target toolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; From 0342b333e35394bdf969d0c94e9c131d4bc72ac2 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 23 Nov 2020 16:06:02 +0100 Subject: [PATCH 050/142] fix type in PR workflow --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8d193983..76ad993d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -153,7 +153,7 @@ jobs: set -e while read -r TOOL; do dn=$(dirname $TOOL); - if ! grep -q "$dn" ../workflow_artifacts/changed_repos.list; then + if ! grep -q "$dn" ../workflow_artifacts/changed_repositories.list; then echo "Tool $TOOL not in changed repositories list: .shed.yml file missing" >&2 exit 1 fi From e84253af7524149bea30d714b4a0d9826fd63d8a Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 23 Nov 2020 19:09:16 +0100 Subject: [PATCH 051/142] require min of 1 chunk --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 76ad993d..412408d3 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -106,6 +106,8 @@ jobs: nchunks=$(wc -l < changed_tools.list) if [ "$nchunks" -gt "$MAX_CHUNKS" ]; then nchunks=$MAX_CHUNKS + elif [ "$nchunks" -eq "0" ] + nchunks=1 fi echo "::set-output name=nchunks::$nchunks" echo "::set-output name=chunk_list::'['$(seq -s ", " 0 $(($nchunks - 1)))']'" From d85d83180b43332f81631c25a786498b57f49884 Mon Sep 17 00:00:00 2001 From: Dave B Date: Mon, 23 Nov 2020 15:34:02 -0500 Subject: [PATCH 052/142] Update kraken2 wrapper and data manager to version 2.1.1 (#3338) * Update kraken2 wrapper and data manager to version 2.1.1 - PR workflow: Check if any changed repo is a substring of each changed tool - Added true and false values to booleans, simplifying cheetah - Made tool version a token in data manager - Added support for the --load-factor parameter - Use the argument only where possible without breaking tool rerun Co-authored-by: Nicola Soranzo --- .github/workflows/pr.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 412408d3..c383cedd 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -154,8 +154,16 @@ jobs: run: | set -e while read -r TOOL; do - dn=$(dirname $TOOL); - if ! grep -q "$dn" ../workflow_artifacts/changed_repositories.list; then + match_found=false; + while read -r REPO; do + case "$TOOL" in + "$REPO"*) + match_found=true + break + ;; + esac + done < ../workflow_artifacts/changed_repositories.list + if [ "$match_found" != 'true' ]; then echo "Tool $TOOL not in changed repositories list: .shed.yml file missing" >&2 exit 1 fi From e738ddde653991b5a8050c3aa30bc414f5b41f72 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 23 Nov 2020 21:44:57 +0100 Subject: [PATCH 053/142] unquote for comparision Co-authored-by: Nicola Soranzo --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c383cedd..cbc84038 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -106,7 +106,7 @@ jobs: nchunks=$(wc -l < changed_tools.list) if [ "$nchunks" -gt "$MAX_CHUNKS" ]; then nchunks=$MAX_CHUNKS - elif [ "$nchunks" -eq "0" ] + elif [ "$nchunks" -eq 0 ]; then nchunks=1 fi echo "::set-output name=nchunks::$nchunks" From f49ead1016f69f0a75cff7ef1cbe4f5d26ecdaa9 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 24 Nov 2020 01:39:45 +0000 Subject: [PATCH 054/142] Lint R files --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index cbc84038..bfa5145b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -242,7 +242,7 @@ jobs: - name: lintr run: | library(lintr) - linters <- with_defaults(line_length_linter = NULL) + linters <- with_defaults(line_length_linter = NULL, object_usage_linter = NULL) con <- file("../workflow_artifacts/changed_repositories.list", "r") status <- 0 while (TRUE) { From 99ca283d2b8c1244cd07ffb72bbaf667bcd0beb6 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 24 Nov 2020 02:57:13 +0000 Subject: [PATCH 055/142] Save Galaxy HEAD sha as job output instead of artifact --- .github/workflows/pr.yaml | 42 +++++++++++++-------------------------- 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index bfa5145b..874d4f8e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,6 +16,7 @@ jobs: name: Setup cache and determine changed repositories runs-on: ubuntu-latest outputs: + galaxy_head_sha: ${{ steps.get-galaxy-sha.outputs.galaxy_head_sha }} nchunks: ${{ steps.get-chunks.outputs.nchunks }} chunk_list: ${{ steps.get-chunks.outputs.chunk_list }} strategy: @@ -34,26 +35,21 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: Determine latest galaxy commit - run: echo "GALAXY_HEAD_SHA=$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1)" >> $GITHUB_ENV - - name: Save latest galaxy commit to artifact file - run: echo $GALAXY_HEAD_SHA > galaxy.sha - - uses: actions/upload-artifact@v2 - with: - name: Workflow artifacts - path: galaxy.sha + - name: Determine latest commit in the Galaxy repo + id: get-galaxy-sha + run: echo "::set-output name=galaxy_head_sha::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1)" - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy_head_sha }} - name: Cache .planemo uses: actions/cache@v2 id: cache-planemo with: path: ~/.planemo - key: planemo_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA + key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy_head_sha }} # Install the `wheel` package so that when installing other packages which # are not available as wheels, pip will build a wheel for them, which can be cached. - name: Install wheel @@ -134,14 +130,12 @@ jobs: with: name: Workflow artifacts path: ../workflow_artifacts/ - - name: Determine latest galaxy commit - run: echo "GALAXY_HEAD_SHA=$(cat ../workflow_artifacts/galaxy.sha)" >> $GITHUB_ENV - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} - name: Install Planemo run: pip install planemo - name: Planemo lint @@ -191,14 +185,12 @@ jobs: with: name: Workflow artifacts path: ../workflow_artifacts/ - - name: Determine latest galaxy commit - run: echo "GALAXY_HEAD_SHA=$(cat ../workflow_artifacts/galaxy.sha)" >> $GITHUB_ENV - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} - name: Install flake8 run: pip install flake8 flake8-import-order - name: Flake8 @@ -296,20 +288,18 @@ jobs: with: name: Workflow artifacts path: ../workflow_artifacts/ - - name: Determine latest galaxy commit - run: echo "GALAXY_HEAD_SHA=$(cat ../workflow_artifacts/galaxy.sha)" >> $GITHUB_ENV - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} - name: Cache .planemo uses: actions/cache@v2 id: cache-planemo with: path: ~/.planemo - key: planemo_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA + key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} - name: Install Planemo run: pip install planemo - name: Planemo ci_find_tools @@ -356,7 +346,7 @@ jobs: # and fail this step if this is the case combine_outputs: name: Combine chunked test results - needs: test + needs: [setup, test] strategy: matrix: python-version: [3.7] @@ -369,14 +359,12 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: Determine latest galaxy commit - run: echo "GALAXY_HEAD_SHA=$(cat ../workflow_artifacts/galaxy.sha)" >> $GITHUB_ENV - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} - name: Install Planemo run: pip install planemo - name: Install jq @@ -403,7 +391,7 @@ jobs: # deploy the tools to the toolsheds (first TTS for testing) deploy: name: Deploy - needs: [lint,flake8,lintr,combine_outputs] + needs: [setup, lint, flake8, lintr, combine_outputs] strategy: matrix: python-version: [3.7] @@ -420,14 +408,12 @@ jobs: with: name: Workflow artifacts path: ../workflow_artifacts/ - - name: Determine latest galaxy commit - run: echo "GALAXY_HEAD_SHA=$(cat ../workflow_artifacts/galaxy.sha)" >> $GITHUB_ENV - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} - name: Install Planemo run: pip install planemo - name: Deploy on testtoolshed From 3994b45662b9fc7744c98066c1d39b22707e4be0 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 24 Nov 2020 03:32:53 +0000 Subject: [PATCH 056/142] Run `planemo ci_find_tools` on changed repos to calculate chunks Otherwise the number of chunks won't take into account tools for which the XML was not modified but other files in the repo were. --- .github/workflows/pr.yaml | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 874d4f8e..d754d53e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,6 +17,7 @@ jobs: runs-on: ubuntu-latest outputs: galaxy_head_sha: ${{ steps.get-galaxy-sha.outputs.galaxy_head_sha }} + commit_range: ${{ steps.get-commit-range.outputs.commit_range }} nchunks: ${{ steps.get-chunks.outputs.nchunks }} chunk_list: ${{ steps.get-chunks.outputs.chunk_list }} strategy: @@ -82,21 +83,25 @@ jobs: - name: Set commit range (pull request) if: github.event_name == 'pull_request' run: echo "COMMIT_RANGE=HEAD~.." >> $GITHUB_ENV + - id: get-commit-range + run: echo "::set-output name=commit_range::$COMMIT_RANGE" - name: Planemo ci_find_repos run: planemo ci_find_repos --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_repositories.list - name: Show repo list run: cat changed_repositories.list - - name: Planemo ci_find_tools - run: planemo ci_find_tools --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_tools.list - - name: Show tool list - run: cat changed_tools.list - uses: actions/upload-artifact@v2 with: name: Workflow artifacts - path: | - changed_repositories.list - changed_tools.list - - name: Get number of changed tools and compute number of chunks + path: changed_repositories.list + - name: Planemo ci_find_tools for the changed repos + run: | + touch changed_tools.list + if [ -s changed_repositories.list ]; then + planemo ci_find_tools --output changed_tools.list $(cat changed_repositories.list) + fi + - name: Show tool list + run: cat changed_tools.list + - name: Compute chunks id: get-chunks run: | nchunks=$(wc -l < changed_tools.list) @@ -122,7 +127,7 @@ jobs: # and use it as the current working directory - uses: actions/checkout@v2 with: - fetch-depth: 1 + fetch-depth: 0 - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -144,6 +149,8 @@ jobs: while read -r DIR; do planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; done < ../workflow_artifacts/changed_repositories.list + - name: Planemo ci_find_tools for the commit range + run: planemo ci_find_tools --changed_in_commit_range ${{ needs.setup.outputs.commit_range }} --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_tools.list - name: Check if each changed tool is in the list of changed repositories run: | set -e @@ -161,7 +168,7 @@ jobs: echo "Tool $TOOL not in changed repositories list: .shed.yml file missing" >&2 exit 1 fi - done < ../workflow_artifacts/changed_tools.list + done < changed_tools.list # flake8 of Python scripts in the changed repositories flake8: @@ -302,11 +309,11 @@ jobs: key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} - name: Install Planemo run: pip install planemo - - name: Planemo ci_find_tools + - name: Planemo ci_find_tools for the change repos, chunked run: | touch changed_tools_chunk.list if [ -s ../workflow_artifacts/changed_repositories.list ]; then - planemo ci_find_tools --chunk_count ${{ needs.setup.outputs.nchunks }} --chunk ${{ matrix.chunk }} --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_tools_chunk.list $(cat ../workflow_artifacts/changed_repositories.list) + planemo ci_find_tools --chunk_count ${{ needs.setup.outputs.nchunks }} --chunk ${{ matrix.chunk }} --output changed_tools_chunk.list $(cat ../workflow_artifacts/changed_repositories.list) fi - name: Show changed tools chunk list run: cat changed_tools_chunk.list From f66452758ee22c2a80ce3cf8274eaca56b5869f3 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 24 Nov 2020 05:12:25 +0000 Subject: [PATCH 057/142] Sync ci and pr workflows --- .github/workflows/pr.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d754d53e..0369a903 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -321,14 +321,14 @@ jobs: run: | mkdir json_output while read -r TOOL; do - if grep -q "$TOOL" .tt_biocontainer_skip; then - PLANEMO_OPTIONS="" - else - PLANEMO_OPTIONS="--biocontainers --no_dependency_resolution --no_conda_auto_init" - fi - json=$(mktemp -u -p json_output --suff .json) - PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$json" "$TOOL" || true - docker system prune --all --force --volumes || true + if grep -q "$TOOL" .tt_biocontainer_skip; then + PLANEMO_OPTIONS="" + else + PLANEMO_OPTIONS="--biocontainers --no_dependency_resolution --no_conda_auto_init" + fi + json=$(mktemp -u -p json_output --suff .json) + PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$json" "$TOOL" || true + docker system prune --all --force --volumes || true done < changed_tools_chunk.list if [ ! -s changed_tools_chunk.list ]; then echo '{"tests":[]}' > "$(mktemp -u -p json_output --suff .json)" @@ -388,7 +388,7 @@ jobs: with: name: 'All tool test results' path: upload - - name: Check status of combined status + - name: Check status of combined outputs run: | if jq '.["tests"][]["data"]["status"]' upload/tool_test_output.json | grep -v "success"; then echo "Unsuccessful tests found, inspect the 'All tool test results' artifact for details." From e176a13861c24c562040d1f81856c8f78050fc0c Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 25 Nov 2020 11:59:52 +0000 Subject: [PATCH 058/142] Don't skip tools in `.tt_biocontainer_skip` during weekly CI testing (#3348) * Don't skip tools in `.tt_biocontainer_skip` during weekly CI testing * Fix regression in check for tool that can't be tested in biocontainers Partially revert commits be43a25be5eead2d1a1ae8b5ee8c49a35b6579f8 and 2c9ea0cdc81b499ec07aa2f0b54b575586b01d42 . Add explanatory comments. * Refactor check --- .github/workflows/pr.yaml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0369a903..ef624ce4 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -155,16 +155,8 @@ jobs: run: | set -e while read -r TOOL; do - match_found=false; - while read -r REPO; do - case "$TOOL" in - "$REPO"*) - match_found=true - break - ;; - esac - done < ../workflow_artifacts/changed_repositories.list - if [ "$match_found" != 'true' ]; then + # Check if any changed repo dir is a substring of $TOOL + if ! echo $TOOL | grep -qf ../workflow_artifacts/changed_repositories.list; then echo "Tool $TOOL not in changed repositories list: .shed.yml file missing" >&2 exit 1 fi @@ -321,7 +313,8 @@ jobs: run: | mkdir json_output while read -r TOOL; do - if grep -q "$TOOL" .tt_biocontainer_skip; then + # Check if any of the lines in .tt_biocontainer_skip is a substring of $TOOL + if echo $TOOL | grep -qf .tt_biocontainer_skip; then PLANEMO_OPTIONS="" else PLANEMO_OPTIONS="--biocontainers --no_dependency_resolution --no_conda_auto_init" From 4fb6bee1e9659abb167b1a9d51514fbc6cf04624 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 26 Nov 2020 09:49:08 +0100 Subject: [PATCH 059/142] unquote TOOL - unquote `$TOOL` (which is renamed to TOOL_GROUP) because its a space separated list of tools - restore `--group_tools` argument for ci_find_tools --- .github/workflows/pr.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ef624ce4..1e9948f1 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -305,22 +305,22 @@ jobs: run: | touch changed_tools_chunk.list if [ -s ../workflow_artifacts/changed_repositories.list ]; then - planemo ci_find_tools --chunk_count ${{ needs.setup.outputs.nchunks }} --chunk ${{ matrix.chunk }} --output changed_tools_chunk.list $(cat ../workflow_artifacts/changed_repositories.list) + planemo ci_find_tools --chunk_count ${{ needs.setup.outputs.nchunks }} --chunk ${{ matrix.chunk }} --group_tools --output changed_tools_chunk.list $(cat ../workflow_artifacts/changed_repositories.list) fi - name: Show changed tools chunk list run: cat changed_tools_chunk.list - name: Planemo test tools run: | mkdir json_output - while read -r TOOL; do - # Check if any of the lines in .tt_biocontainer_skip is a substring of $TOOL - if echo $TOOL | grep -qf .tt_biocontainer_skip; then + while read -r TOOL_GROUP; do + # Check if any of the lines in .tt_biocontainer_skip is a substring of $TOOL_GROUP + if echo $TOOL_GROUP | grep -qf .tt_biocontainer_skip; then PLANEMO_OPTIONS="" else PLANEMO_OPTIONS="--biocontainers --no_dependency_resolution --no_conda_auto_init" fi json=$(mktemp -u -p json_output --suff .json) - PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$json" "$TOOL" || true + PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$json" $TOOL_GROUP || true docker system prune --all --force --volumes || true done < changed_tools_chunk.list if [ ! -s changed_tools_chunk.list ]; then From 20351ea5c6701da8cbcd814bb3f33046b5b9db46 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 27 Dec 2020 12:11:19 +0100 Subject: [PATCH 060/142] do not lint R for cyclomatic complexity as already done in devteam repo --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1e9948f1..64e27a4b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -233,7 +233,7 @@ jobs: - name: lintr run: | library(lintr) - linters <- with_defaults(line_length_linter = NULL, object_usage_linter = NULL) + linters <- with_defaults(line_length_linter = NULL, cyclocomp_linter = NULL, object_usage_linter = NULL) con <- file("../workflow_artifacts/changed_repositories.list", "r") status <- 0 while (TRUE) { From df3144515fb7d747ef7381697fe627f055fa9592 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Thu, 31 Dec 2020 14:51:01 +0100 Subject: [PATCH 061/142] fix ci --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 64e27a4b..c6c6863c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -150,7 +150,7 @@ jobs: planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; done < ../workflow_artifacts/changed_repositories.list - name: Planemo ci_find_tools for the commit range - run: planemo ci_find_tools --changed_in_commit_range ${{ needs.setup.outputs.commit_range }} --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_tools.list + run: planemo ci_find_tools --changed_in_commit_range ${{ needs.setup.outputs.commit_range }} --exclude packages --exclude deprecated --exclude .tt_skip --exclude .tt_biocontainer_skip --exclude_from .tt_skip --output changed_tools.list - name: Check if each changed tool is in the list of changed repositories run: | set -e @@ -224,7 +224,7 @@ jobs: path: ${{ env.R_LIBS_USER }} key: r_cache_${{ matrix.r-version }} - name: Install non-R lintr dependencies - run: sudo apt-get install libcurl4-openssl-dev + run: sudo apt-get install libcurl4-openssl-dev - name: Install lintr run: | install.packages('remotes') From ccfbe4ef988010fbdfb617b84fa1f1a283a4adfc Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Thu, 31 Dec 2020 14:59:12 +0100 Subject: [PATCH 062/142] more exclude --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c6c6863c..62544ad8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -150,7 +150,7 @@ jobs: planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; done < ../workflow_artifacts/changed_repositories.list - name: Planemo ci_find_tools for the commit range - run: planemo ci_find_tools --changed_in_commit_range ${{ needs.setup.outputs.commit_range }} --exclude packages --exclude deprecated --exclude .tt_skip --exclude .tt_biocontainer_skip --exclude_from .tt_skip --output changed_tools.list + run: planemo ci_find_tools --changed_in_commit_range ${{ needs.setup.outputs.commit_range }} --exclude .github --exclude packages --exclude deprecated --exclude .tt_skip --exclude .tt_biocontainer_skip --exclude_from .tt_skip --output changed_tools.list - name: Check if each changed tool is in the list of changed repositories run: | set -e From 45e1f5f8e892a8589289cd00919d029aec3846de Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Sun, 3 Jan 2021 17:23:19 +0000 Subject: [PATCH 063/142] Revert changes in pr.yaml added in https://github.com/galaxyproject/tools-iuc/pull/3403 . The Planemo issue should be fixed in 0.74.1 --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 62544ad8..4a638502 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -150,7 +150,7 @@ jobs: planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; done < ../workflow_artifacts/changed_repositories.list - name: Planemo ci_find_tools for the commit range - run: planemo ci_find_tools --changed_in_commit_range ${{ needs.setup.outputs.commit_range }} --exclude .github --exclude packages --exclude deprecated --exclude .tt_skip --exclude .tt_biocontainer_skip --exclude_from .tt_skip --output changed_tools.list + run: planemo ci_find_tools --changed_in_commit_range ${{ needs.setup.outputs.commit_range }} --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_tools.list - name: Check if each changed tool is in the list of changed repositories run: | set -e From 05f2e02a81b74912c406e47662cfcf98c3431332 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 12 Jan 2021 16:40:50 +0000 Subject: [PATCH 064/142] Sync ci and pr GitHub workflows (#3424) * Sync ci and pr GitHub workflows * Update bowtie2 to 2.4.2 --- .github/workflows/pr.yaml | 64 +++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4a638502..6a11304f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -33,12 +33,12 @@ jobs: echo 'base_ref: ${{ github.base_ref }}' echo 'event.before: ${{ github.event.before }}' echo 'event.after: ${{ github.event.after }}' - - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - name: Determine latest commit in the Galaxy repo id: get-galaxy-sha run: echo "::set-output name=galaxy_head_sha::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1)" + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip @@ -85,26 +85,26 @@ jobs: run: echo "COMMIT_RANGE=HEAD~.." >> $GITHUB_ENV - id: get-commit-range run: echo "::set-output name=commit_range::$COMMIT_RANGE" - - name: Planemo ci_find_repos - run: planemo ci_find_repos --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_repositories.list - - name: Show repo list - run: cat changed_repositories.list + - name: Planemo ci_find_repos for the commit range + run: planemo ci_find_repos --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output repository_list.txt + - name: Show repository list + run: cat repository_list.txt - uses: actions/upload-artifact@v2 with: name: Workflow artifacts - path: changed_repositories.list - - name: Planemo ci_find_tools for the changed repos + path: repository_list.txt + - name: Planemo ci_find_tools for the repository list run: | - touch changed_tools.list - if [ -s changed_repositories.list ]; then - planemo ci_find_tools --output changed_tools.list $(cat changed_repositories.list) + touch tool_list.txt + if [ -s repository_list.txt ]; then + planemo ci_find_tools --output tool_list.txt $(cat repository_list.txt) fi - name: Show tool list - run: cat changed_tools.list + run: cat tool_list.txt - name: Compute chunks id: get-chunks run: | - nchunks=$(wc -l < changed_tools.list) + nchunks=$(wc -l < tool_list.txt) if [ "$nchunks" -gt "$MAX_CHUNKS" ]; then nchunks=$MAX_CHUNKS elif [ "$nchunks" -eq 0 ]; then @@ -148,19 +148,19 @@ jobs: set -e while read -r DIR; do planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; - done < ../workflow_artifacts/changed_repositories.list + done < ../workflow_artifacts/repository_list.txt - name: Planemo ci_find_tools for the commit range - run: planemo ci_find_tools --changed_in_commit_range ${{ needs.setup.outputs.commit_range }} --exclude packages --exclude deprecated --exclude_from .tt_skip --output changed_tools.list + run: planemo ci_find_tools --changed_in_commit_range ${{ needs.setup.outputs.commit_range }} --exclude packages --exclude deprecated --exclude_from .tt_skip --output tool_list.txt - name: Check if each changed tool is in the list of changed repositories run: | set -e while read -r TOOL; do # Check if any changed repo dir is a substring of $TOOL - if ! echo $TOOL | grep -qf ../workflow_artifacts/changed_repositories.list; then + if ! echo $TOOL | grep -qf ../workflow_artifacts/repository_list.txt; then echo "Tool $TOOL not in changed repositories list: .shed.yml file missing" >&2 exit 1 fi - done < changed_tools.list + done < tool_list.txt # flake8 of Python scripts in the changed repositories flake8: @@ -194,8 +194,8 @@ jobs: run: pip install flake8 flake8-import-order - name: Flake8 run: | - if [ -s ../workflow_artifacts/changed_repositories.list ]; then - flake8 $(cat ../workflow_artifacts/changed_repositories.list) + if [ -s ../workflow_artifacts/repository_list.txt ]; then + flake8 $(cat ../workflow_artifacts/repository_list.txt) fi lintr: @@ -234,7 +234,7 @@ jobs: run: | library(lintr) linters <- with_defaults(line_length_linter = NULL, cyclocomp_linter = NULL, object_usage_linter = NULL) - con <- file("../workflow_artifacts/changed_repositories.list", "r") + con <- file("../workflow_artifacts/repository_list.txt", "r") status <- 0 while (TRUE) { repo <- readLines(con, n = 1) @@ -301,14 +301,14 @@ jobs: key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} - name: Install Planemo run: pip install planemo - - name: Planemo ci_find_tools for the change repos, chunked + - name: Planemo ci_find_tools for the repository list, chunked run: | - touch changed_tools_chunk.list - if [ -s ../workflow_artifacts/changed_repositories.list ]; then - planemo ci_find_tools --chunk_count ${{ needs.setup.outputs.nchunks }} --chunk ${{ matrix.chunk }} --group_tools --output changed_tools_chunk.list $(cat ../workflow_artifacts/changed_repositories.list) + touch tool_list_chunk.txt + if [ -s ../workflow_artifacts/repository_list.txt ]; then + planemo ci_find_tools --chunk_count ${{ needs.setup.outputs.nchunks }} --chunk ${{ matrix.chunk }} --group_tools --output tool_list_chunk.txt $(cat ../workflow_artifacts/repository_list.txt) fi - - name: Show changed tools chunk list - run: cat changed_tools_chunk.list + - name: Show tool list chunk + run: cat tool_list_chunk.txt - name: Planemo test tools run: | mkdir json_output @@ -322,8 +322,8 @@ jobs: json=$(mktemp -u -p json_output --suff .json) PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$json" $TOOL_GROUP || true docker system prune --all --force --volumes || true - done < changed_tools_chunk.list - if [ ! -s changed_tools_chunk.list ]; then + done < tool_list_chunk.txt + if [ ! -s tool_list_chunk.txt ]; then echo '{"tests":[]}' > "$(mktemp -u -p json_output --suff .json)" fi - name: Merge tool_test_output.json files @@ -336,7 +336,7 @@ jobs: mv tool_test_output.json tool_test_output.html upload/ - uses: actions/upload-artifact@v2 with: - name: 'Tool test output ${{ matrix.chunk }}' + name: 'Tool test output ${{ matrix.chunk }}' path: upload # - combine the results of the test chunks (which will never fail due @@ -422,7 +422,7 @@ jobs: run: | while read -r DIR; do planemo shed_update --shed_target testtoolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; - done < ../workflow_artifacts/changed_repositories.list + done < ../workflow_artifacts/repository_list.txt continue-on-error: true - name: Deploy on toolshed env: @@ -430,4 +430,4 @@ jobs: run: | while read -r DIR; do planemo shed_update --shed_target toolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; - done < ../workflow_artifacts/changed_repositories.list + done < ../workflow_artifacts/repository_list.txt From ab4b3138e80d4c35c255bf9ea0c734049d314109 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 13 Jan 2021 20:03:19 +0100 Subject: [PATCH 065/142] use planemo-ci-action for PR and CI workflow Co-authored-by: Marius van den Beek --- .github/workflows/pr.yaml | 267 ++++++++++++-------------------------- 1 file changed, 85 insertions(+), 182 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6a11304f..38f780d6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -1,8 +1,8 @@ name: Galaxy Tool Linting and Tests for push and PR on: [push, pull_request] env: - GALAXY_REPO: https://github.com/galaxyproject/galaxy - GALAXY_RELEASE: release_20.09 + GALAXY_FORK: galaxyproject + GALAXY_BRANCH: release_20.09 MAX_CHUNKS: 4 jobs: # the setup job does two things: @@ -16,10 +16,12 @@ jobs: name: Setup cache and determine changed repositories runs-on: ubuntu-latest outputs: - galaxy_head_sha: ${{ steps.get-galaxy-sha.outputs.galaxy_head_sha }} - commit_range: ${{ steps.get-commit-range.outputs.commit_range }} - nchunks: ${{ steps.get-chunks.outputs.nchunks }} - chunk_list: ${{ steps.get-chunks.outputs.chunk_list }} + galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} + commit-range: ${{ steps.discover.outputs.commit-range }} + repositories: ${{ steps.discover.outputs.repositories }} + tools: ${{ steps.discover.outputs.tools }} + chunk-count: ${{ steps.discover.outputs.chunk-count }} + chunk-list: ${{ steps.discover.outputs.chunk-list }} strategy: matrix: python-version: [3.7] @@ -35,7 +37,7 @@ jobs: echo 'event.after: ${{ github.event.after }}' - name: Determine latest commit in the Galaxy repo id: get-galaxy-sha - run: echo "::set-output name=galaxy_head_sha::$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1)" + run: echo "::set-output name=galaxy-head-sha::$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -44,79 +46,44 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy_head_sha }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} - name: Cache .planemo uses: actions/cache@v2 id: cache-planemo with: path: ~/.planemo - key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy_head_sha }} + key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} # Install the `wheel` package so that when installing other packages which # are not available as wheels, pip will build a wheel for them, which can be cached. - - name: Install wheel - run: pip install wheel - - name: Install Planemo and flake8 - run: pip install planemo flake8 flake8-import-order - - name: Fake a Planemo run to update cache - if: steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' - run: | - touch tool.xml - PIP_QUIET=1 planemo test --galaxy_python_version ${{ matrix.python-version }} --no_conda_auto_init --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE + - name: Install flake8 + run: pip install wheel flake8 flake8-import-order - uses: actions/checkout@v2 with: fetch-depth: 0 - # The range of commits to check for changes is: - # - `origin/master...` for all events happening on a feature branch - # - for events on the master branch we compare against the sha before the event - # (note that this does not work for feature branch events since we want all - # commits on the feature branch and not just the commits of the last event) - # - for pull requests we compare against the 1st ancestor, given the current - # HEAD is the merge between the PR branch and the base branch - - name: Set commit range (push to the feature branch) - if: github.ref != 'refs/heads/master' && github.event_name == 'push' - run: | - git fetch origin master - echo "COMMIT_RANGE=origin/master..." >> $GITHUB_ENV - - name: Set commit range (push to the master branch, e.g. merge) - if: github.ref == 'refs/heads/master' && github.event_name == 'push' - run: echo "COMMIT_RANGE=${{ github.event.before }}.." >> $GITHUB_ENV - - name: Set commit range (pull request) - if: github.event_name == 'pull_request' - run: echo "COMMIT_RANGE=HEAD~.." >> $GITHUB_ENV - - id: get-commit-range - run: echo "::set-output name=commit_range::$COMMIT_RANGE" - - name: Planemo ci_find_repos for the commit range - run: planemo ci_find_repos --changed_in_commit_range $COMMIT_RANGE --exclude packages --exclude deprecated --exclude_from .tt_skip --output repository_list.txt - - name: Show repository list - run: cat repository_list.txt - - uses: actions/upload-artifact@v2 + - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks + uses: galaxyproject/planemo-ci-action@master + id: discover with: - name: Workflow artifacts - path: repository_list.txt - - name: Planemo ci_find_tools for the repository list - run: | - touch tool_list.txt - if [ -s repository_list.txt ]; then - planemo ci_find_tools --output tool_list.txt $(cat repository_list.txt) - fi + create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }} + galaxy-fork: ${{ env.GALAXY_FORK }} + galaxy-branch: ${{ env.GALAXY_BRANCH }} + max-chunks: ${{ env.MAX_CHUNKS }} + python-version: ${{ matrix.python-version }} + - name: Show commit range + run: echo ${{ steps.discover.outputs.commit-range }} + - name: Show repository list + run: echo "${{ steps.discover.outputs.repositories }}" - name: Show tool list - run: cat tool_list.txt - - name: Compute chunks - id: get-chunks + run: echo "${{ steps.discover.outputs.tools }}" + - name: Show chunks run: | - nchunks=$(wc -l < tool_list.txt) - if [ "$nchunks" -gt "$MAX_CHUNKS" ]; then - nchunks=$MAX_CHUNKS - elif [ "$nchunks" -eq 0 ]; then - nchunks=1 - fi - echo "::set-output name=nchunks::$nchunks" - echo "::set-output name=chunk_list::'['$(seq -s ", " 0 $(($nchunks - 1)))']'" + echo 'Using ${{ steps.discover.outputs.chunk-count }} chunks (${{ steps.discover.outputs.chunk-list }})' # Planemo lint the changed repositories lint: name: Lint tools needs: setup + if: needs.setup.outputs.repositories != '' runs-on: ubuntu-latest strategy: fail-fast: false @@ -127,45 +94,29 @@ jobs: # and use it as the current working directory - uses: actions/checkout@v2 with: - fetch-depth: 0 + fetch-depth: 1 - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - uses: actions/download-artifact@v2 - with: - name: Workflow artifacts - path: ../workflow_artifacts/ - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} - - name: Install Planemo - run: pip install planemo - - name: Planemo lint - run: | - set -e - while read -r DIR; do - planemo shed_lint --tools --ensure_metadata --urls --report_level warn --fail_level error --recursive "$DIR"; - done < ../workflow_artifacts/repository_list.txt - - name: Planemo ci_find_tools for the commit range - run: planemo ci_find_tools --changed_in_commit_range ${{ needs.setup.outputs.commit_range }} --exclude packages --exclude deprecated --exclude_from .tt_skip --output tool_list.txt - - name: Check if each changed tool is in the list of changed repositories - run: | - set -e - while read -r TOOL; do - # Check if any changed repo dir is a substring of $TOOL - if ! echo $TOOL | grep -qf ../workflow_artifacts/repository_list.txt; then - echo "Tool $TOOL not in changed repositories list: .shed.yml file missing" >&2 - exit 1 - fi - done < tool_list.txt + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} + - name: Planemo lint + uses: galaxyproject/planemo-ci-action@master + id: lint + with: + lint-tools: true + repositories: ${{ needs.setup.outputs.repositories }} + tools: ${{ needs.setup.outputs.tools }} # flake8 of Python scripts in the changed repositories flake8: name: Lint Python scripts needs: setup + if: needs.setup.outputs.repositories != '' runs-on: ubuntu-latest strategy: fail-fast: false @@ -180,27 +131,21 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - uses: actions/download-artifact@v2 - with: - name: Workflow artifacts - path: ../workflow_artifacts/ - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - name: Install flake8 run: pip install flake8 flake8-import-order - name: Flake8 - run: | - if [ -s ../workflow_artifacts/repository_list.txt ]; then - flake8 $(cat ../workflow_artifacts/repository_list.txt) - fi + run: flake8 ${{ needs.setup.outputs.repositories }} lintr: name: Lint R scripts needs: setup + if: needs.setup.outputs.repositories != '' runs-on: ubuntu-latest strategy: matrix: @@ -211,10 +156,6 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 1 - - uses: actions/download-artifact@v2 - with: - name: Workflow artifacts - path: ../workflow_artifacts/ - uses: r-lib/actions/setup-r@master with: r-version: ${{ matrix.r-version }} @@ -230,11 +171,13 @@ jobs: install.packages('remotes') remotes::install_cran("lintr") shell: Rscript {0} + - name: Save repositories to file + run: echo ${{ needs.setup.outputs.repositories }} > repository_list.txt - name: lintr run: | library(lintr) linters <- with_defaults(line_length_linter = NULL, cyclocomp_linter = NULL, object_usage_linter = NULL) - con <- file("../workflow_artifacts/repository_list.txt", "r") + con <- file("repository_list.txt", "r") status <- 0 while (TRUE) { repo <- readLines(con, n = 1) @@ -260,10 +203,11 @@ jobs: # This job runs on Linux runs-on: ubuntu-latest needs: setup + if: needs.setup.outputs.repositories != '' strategy: fail-fast: false matrix: - chunk: ${{ fromJson(needs.setup.outputs.chunk_list) }} + chunk: ${{ fromJson(needs.setup.outputs.chunk-list) }} python-version: [3.7] services: postgres: @@ -283,57 +227,29 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - uses: actions/download-artifact@v2 - with: - name: Workflow artifacts - path: ../workflow_artifacts/ - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - name: Cache .planemo uses: actions/cache@v2 id: cache-planemo with: path: ~/.planemo - key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} - - name: Install Planemo - run: pip install planemo - - name: Planemo ci_find_tools for the repository list, chunked - run: | - touch tool_list_chunk.txt - if [ -s ../workflow_artifacts/repository_list.txt ]; then - planemo ci_find_tools --chunk_count ${{ needs.setup.outputs.nchunks }} --chunk ${{ matrix.chunk }} --group_tools --output tool_list_chunk.txt $(cat ../workflow_artifacts/repository_list.txt) - fi - - name: Show tool list chunk - run: cat tool_list_chunk.txt - - name: Planemo test tools - run: | - mkdir json_output - while read -r TOOL_GROUP; do - # Check if any of the lines in .tt_biocontainer_skip is a substring of $TOOL_GROUP - if echo $TOOL_GROUP | grep -qf .tt_biocontainer_skip; then - PLANEMO_OPTIONS="" - else - PLANEMO_OPTIONS="--biocontainers --no_dependency_resolution --no_conda_auto_init" - fi - json=$(mktemp -u -p json_output --suff .json) - PIP_QUIET=1 planemo test --database_connection postgresql://postgres:postgres@localhost:5432/galaxy $PLANEMO_OPTIONS --galaxy_source $GALAXY_REPO --galaxy_branch $GALAXY_RELEASE --galaxy_python_version ${{ matrix.python-version }} --test_output_json "$json" $TOOL_GROUP || true - docker system prune --all --force --volumes || true - done < tool_list_chunk.txt - if [ ! -s tool_list_chunk.txt ]; then - echo '{"tests":[]}' > "$(mktemp -u -p json_output --suff .json)" - fi - - name: Merge tool_test_output.json files - run: planemo merge_test_reports json_output/*.json tool_test_output.json - - name: Create tool_test_output.html - run: planemo test_reports tool_test_output.json --test_output tool_test_output.html - - name: Copy artifacts into place - run: | - mkdir upload - mv tool_test_output.json tool_test_output.html upload/ + key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} + + - name: Planemo test + uses: galaxyproject/planemo-ci-action@master + id: test + with: + test-tools: true + repositories: ${{ needs.setup.outputs.repositories }} + galaxy-fork: ${{ env.GALAXY_FORK }} + galaxy-branch: ${{ env.GALAXY_BRANCH }} + chunk: ${{ matrix.chunk }} + chunk-count: ${{ needs.setup.outputs.chunk-count }} - uses: actions/upload-artifact@v2 with: name: 'Tool test output ${{ matrix.chunk }}' @@ -364,29 +280,22 @@ jobs: id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} - - name: Install Planemo - run: pip install planemo - - name: Install jq - run: sudo apt-get install jq + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - name: Combine outputs - run: find artifacts/ -name tool_test_output.json -exec sh -c 'planemo merge_test_reports "$@" tool_test_output.json' sh {} + - - name: Create tool_test_output.html - run: planemo test_reports tool_test_output.json --test_output tool_test_output.html - - name: Copy artifacts into place - run: | - mkdir upload - mv tool_test_output.json tool_test_output.html upload/ + uses: galaxyproject/planemo-ci-action@master + id: combine + with: + combine-outputs: true + html-report: true - uses: actions/upload-artifact@v2 with: name: 'All tool test results' path: upload - - name: Check status of combined outputs - run: | - if jq '.["tests"][]["data"]["status"]' upload/tool_test_output.json | grep -v "success"; then - echo "Unsuccessful tests found, inspect the 'All tool test results' artifact for details." - exit 1 - fi + - name: Check outputs + uses: galaxyproject/planemo-ci-action@master + id: check + with: + check-outputs: true # deploy the tools to the toolsheds (first TTS for testing) deploy: @@ -400,34 +309,28 @@ jobs: steps: - uses: actions/checkout@v2 with: - fetch-depth: 1 + fetch-depth: 0 - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - uses: actions/download-artifact@v2 - with: - name: Workflow artifacts - path: ../workflow_artifacts/ - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy_head_sha }} - - name: Install Planemo - run: pip install planemo + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} +# - name: Install Planemo +# run: pip install planemo - name: Deploy on testtoolshed - env: - SHED_KEY: ${{ secrets.tts_api_key }} - run: | - while read -r DIR; do - planemo shed_update --shed_target testtoolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; - done < ../workflow_artifacts/repository_list.txt + uses: galaxyproject/planemo-ci-action@master + with: + deploy-tools: true + shed-target: testtoolshed + shed-key: ${{ secrets.TTS_API_KEY }} continue-on-error: true - name: Deploy on toolshed - env: - SHED_KEY: ${{ secrets.ts_api_key }} - run: | - while read -r DIR; do - planemo shed_update --shed_target toolshed --shed_key "${{ env.SHED_KEY }}" --force_repository_creation "$DIR" || exit 1; - done < ../workflow_artifacts/repository_list.txt + uses: galaxyproject/planemo-ci-action@master + with: + deploy-tools: true + shed-target: toolshed + shed-key: ${{ secrets.TS_API_KEY }} From 5ec57654276e4a0895a47dd65724a91924519e25 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 19 Jan 2021 14:00:07 +0100 Subject: [PATCH 066/142] adapt to changes to the action that occured while making the action workflow ready --- .github/workflows/pr.yaml | 48 +++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 38f780d6..a8148f0d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,9 +17,8 @@ jobs: runs-on: ubuntu-latest outputs: galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} - commit-range: ${{ steps.discover.outputs.commit-range }} - repositories: ${{ steps.discover.outputs.repositories }} - tools: ${{ steps.discover.outputs.tools }} + repository-list: ${{ steps.discover.outputs.repository-list }} + tool-list: ${{ steps.discover.outputs.tool-list }} chunk-count: ${{ steps.discover.outputs.chunk-count }} chunk-list: ${{ steps.discover.outputs.chunk-list }} strategy: @@ -55,8 +54,10 @@ jobs: key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} # Install the `wheel` package so that when installing other packages which # are not available as wheels, pip will build a wheel for them, which can be cached. + - name: Install wheel + run: pip install wheel - name: Install flake8 - run: pip install wheel flake8 flake8-import-order + run: pip install flake8 flake8-import-order - uses: actions/checkout@v2 with: fetch-depth: 0 @@ -72,18 +73,18 @@ jobs: - name: Show commit range run: echo ${{ steps.discover.outputs.commit-range }} - name: Show repository list - run: echo "${{ steps.discover.outputs.repositories }}" + run: echo "${{ steps.discover.outputs.repository-list }}" - name: Show tool list - run: echo "${{ steps.discover.outputs.tools }}" + run: echo "${{ steps.discover.outputs.tool-list }}" - name: Show chunks run: | echo 'Using ${{ steps.discover.outputs.chunk-count }} chunks (${{ steps.discover.outputs.chunk-list }})' # Planemo lint the changed repositories lint: - name: Lint tools + name: Lint tool-list needs: setup - if: needs.setup.outputs.repositories != '' + if: needs.setup.outputs.repository-list != '' runs-on: ubuntu-latest strategy: fail-fast: false @@ -104,19 +105,19 @@ jobs: with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Planemo lint + - name: Planemo lint uses: galaxyproject/planemo-ci-action@master id: lint with: - lint-tools: true - repositories: ${{ needs.setup.outputs.repositories }} - tools: ${{ needs.setup.outputs.tools }} + lint-mode: true + repository-list: ${{ needs.setup.outputs.repository-list }} + tool-list: ${{ needs.setup.outputs.tool-list }} # flake8 of Python scripts in the changed repositories flake8: name: Lint Python scripts needs: setup - if: needs.setup.outputs.repositories != '' + if: needs.setup.outputs.repository-list != '' runs-on: ubuntu-latest strategy: fail-fast: false @@ -140,12 +141,12 @@ jobs: - name: Install flake8 run: pip install flake8 flake8-import-order - name: Flake8 - run: flake8 ${{ needs.setup.outputs.repositories }} + run: flake8 ${{ needs.setup.outputs.repository-list }} lintr: name: Lint R scripts needs: setup - if: needs.setup.outputs.repositories != '' + if: needs.setup.outputs.repository-list != '' runs-on: ubuntu-latest strategy: matrix: @@ -172,7 +173,7 @@ jobs: remotes::install_cran("lintr") shell: Rscript {0} - name: Save repositories to file - run: echo ${{ needs.setup.outputs.repositories }} > repository_list.txt + run: echo ${{ needs.setup.outputs.repository-list }} > repository_list.txt - name: lintr run: | library(lintr) @@ -203,7 +204,7 @@ jobs: # This job runs on Linux runs-on: ubuntu-latest needs: setup - if: needs.setup.outputs.repositories != '' + if: needs.setup.outputs.repository-list != '' strategy: fail-fast: false matrix: @@ -239,13 +240,12 @@ jobs: with: path: ~/.planemo key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Planemo test uses: galaxyproject/planemo-ci-action@master id: test with: - test-tools: true - repositories: ${{ needs.setup.outputs.repositories }} + test-mode: true + repository-list: ${{ needs.setup.outputs.repository-list }} galaxy-fork: ${{ env.GALAXY_FORK }} galaxy-branch: ${{ env.GALAXY_BRANCH }} chunk: ${{ matrix.chunk }} @@ -285,7 +285,7 @@ jobs: uses: galaxyproject/planemo-ci-action@master id: combine with: - combine-outputs: true + combine-mode: true html-report: true - uses: actions/upload-artifact@v2 with: @@ -295,7 +295,7 @@ jobs: uses: galaxyproject/planemo-ci-action@master id: check with: - check-outputs: true + check-mode: true # deploy the tools to the toolsheds (first TTS for testing) deploy: @@ -309,7 +309,7 @@ jobs: steps: - uses: actions/checkout@v2 with: - fetch-depth: 0 + fetch-depth: 1 - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} @@ -331,6 +331,6 @@ jobs: - name: Deploy on toolshed uses: galaxyproject/planemo-ci-action@master with: - deploy-tools: true + deploy-mode: true shed-target: toolshed shed-key: ${{ secrets.TS_API_KEY }} From f9274415387898695f6f70c11c32b0e5de32003b Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 29 Jan 2021 10:52:15 +0100 Subject: [PATCH 067/142] update to latest version of the planemo-ci-action --- .github/workflows/pr.yaml | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a8148f0d..b335d443 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -62,7 +62,7 @@ jobs: with: fetch-depth: 0 - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks - uses: galaxyproject/planemo-ci-action@master + uses: galaxyproject/planemo-ci-action@v1 id: discover with: create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }} @@ -106,10 +106,10 @@ jobs: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - name: Planemo lint - uses: galaxyproject/planemo-ci-action@master + uses: galaxyproject/planemo-ci-action@v1 id: lint with: - lint-mode: true + mode: lint repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} @@ -241,10 +241,10 @@ jobs: path: ~/.planemo key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - name: Planemo test - uses: galaxyproject/planemo-ci-action@master + uses: galaxyproject/planemo-ci-action@v1 id: test with: - test-mode: true + mode: test repository-list: ${{ needs.setup.outputs.repository-list }} galaxy-fork: ${{ env.GALAXY_FORK }} galaxy-branch: ${{ env.GALAXY_BRANCH }} @@ -282,20 +282,20 @@ jobs: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - name: Combine outputs - uses: galaxyproject/planemo-ci-action@master + uses: galaxyproject/planemo-ci-action@v1 id: combine with: - combine-mode: true + mode: combine html-report: true - uses: actions/upload-artifact@v2 with: name: 'All tool test results' path: upload - name: Check outputs - uses: galaxyproject/planemo-ci-action@master + uses: galaxyproject/planemo-ci-action@v1 id: check with: - check-mode: true + mode: check # deploy the tools to the toolsheds (first TTS for testing) deploy: @@ -319,18 +319,16 @@ jobs: with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} -# - name: Install Planemo -# run: pip install planemo - name: Deploy on testtoolshed - uses: galaxyproject/planemo-ci-action@master + uses: galaxyproject/planemo-ci-action@v1 with: - deploy-tools: true + mode: deploy shed-target: testtoolshed shed-key: ${{ secrets.TTS_API_KEY }} continue-on-error: true - name: Deploy on toolshed - uses: galaxyproject/planemo-ci-action@master + uses: galaxyproject/planemo-ci-action@v1 with: - deploy-mode: true + mode: deploy shed-target: toolshed shed-key: ${{ secrets.TS_API_KEY }} From bd434996c1b9fb516041bd9dea27c83e7fcdc227 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 1 Feb 2021 10:14:19 +0100 Subject: [PATCH 068/142] add repository list to deploy --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b335d443..56216f2c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -323,6 +323,7 @@ jobs: uses: galaxyproject/planemo-ci-action@v1 with: mode: deploy + repository-list: ${{ needs.setup.outputs.repository-list }} shed-target: testtoolshed shed-key: ${{ secrets.TTS_API_KEY }} continue-on-error: true @@ -330,5 +331,6 @@ jobs: uses: galaxyproject/planemo-ci-action@v1 with: mode: deploy + repository-list: ${{ needs.setup.outputs.repository-list }} shed-target: toolshed shed-key: ${{ secrets.TS_API_KEY }} From 0caf82a62f6df5310ffc84d8a99d01026368e14a Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 5 Feb 2021 20:16:41 +0000 Subject: [PATCH 069/142] Fix quotes in GH workflows --- .github/workflows/pr.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 56216f2c..36b63522 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -71,11 +71,11 @@ jobs: max-chunks: ${{ env.MAX_CHUNKS }} python-version: ${{ matrix.python-version }} - name: Show commit range - run: echo ${{ steps.discover.outputs.commit-range }} + run: echo '${{ steps.discover.outputs.commit-range }}' - name: Show repository list - run: echo "${{ steps.discover.outputs.repository-list }}" + run: echo '${{ steps.discover.outputs.repository-list }}' - name: Show tool list - run: echo "${{ steps.discover.outputs.tool-list }}" + run: echo '${{ steps.discover.outputs.tool-list }}' - name: Show chunks run: | echo 'Using ${{ steps.discover.outputs.chunk-count }} chunks (${{ steps.discover.outputs.chunk-list }})' @@ -141,7 +141,7 @@ jobs: - name: Install flake8 run: pip install flake8 flake8-import-order - name: Flake8 - run: flake8 ${{ needs.setup.outputs.repository-list }} + run: echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' flake8 lintr: name: Lint R scripts @@ -173,7 +173,7 @@ jobs: remotes::install_cran("lintr") shell: Rscript {0} - name: Save repositories to file - run: echo ${{ needs.setup.outputs.repository-list }} > repository_list.txt + run: echo '${{ needs.setup.outputs.repository-list }}' > repository_list.txt - name: lintr run: | library(lintr) From b9c8dfa8165a330ab570d7d554093abff7047592 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 9 Feb 2021 17:03:22 +0100 Subject: [PATCH 070/142] switch workflows to Galaxy release 21.01 --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 36b63522..152ed316 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -2,7 +2,7 @@ name: Galaxy Tool Linting and Tests for push and PR on: [push, pull_request] env: GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_20.09 + GALAXY_BRANCH: release_21.01 MAX_CHUNKS: 4 jobs: # the setup job does two things: From d90b75094e1ffb020f76e7bae885fd2c38b7380e Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 18 Mar 2021 16:14:35 +0000 Subject: [PATCH 071/142] Make R package cache key dependent on OS version Also quote version string (could be mistaken for floats by YAML). --- .github/workflows/pr.yaml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 152ed316..cea8f426 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -23,7 +23,7 @@ jobs: chunk-list: ${{ steps.discover.outputs.chunk-list }} strategy: matrix: - python-version: [3.7] + python-version: ['3.7'] steps: - name: Print github context properties run: | @@ -89,7 +89,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7] + python-version: ['3.7'] steps: # checkout the repository # and use it as the current working directory @@ -122,7 +122,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7] + python-version: ['3.7'] steps: # checkout the repository to master # and use it as the current working directory @@ -147,10 +147,11 @@ jobs: name: Lint R scripts needs: setup if: needs.setup.outputs.repository-list != '' - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: - r-version: [4.0.1] + os: [ubuntu-20.04] + r-version: ['4.0.1'] steps: # checkout the repository to master # and use it as the current working directory @@ -164,7 +165,7 @@ jobs: uses: actions/cache@v2 with: path: ${{ env.R_LIBS_USER }} - key: r_cache_${{ matrix.r-version }} + key: r_cache_${{ matrix.os }}_${{ matrix.r-version }} - name: Install non-R lintr dependencies run: sudo apt-get install libcurl4-openssl-dev - name: Install lintr @@ -209,7 +210,7 @@ jobs: fail-fast: false matrix: chunk: ${{ fromJson(needs.setup.outputs.chunk-list) }} - python-version: [3.7] + python-version: ['3.7'] services: postgres: image: postgres:11 @@ -265,7 +266,7 @@ jobs: needs: [setup, test] strategy: matrix: - python-version: [3.7] + python-version: ['3.7'] # This job runs on Linux runs-on: ubuntu-latest steps: @@ -303,7 +304,7 @@ jobs: needs: [setup, lint, flake8, lintr, combine_outputs] strategy: matrix: - python-version: [3.7] + python-version: ['3.7'] runs-on: ubuntu-latest if: github.ref == 'refs/heads/master' && github.repository_owner == 'galaxyproject' steps: From 37b7484bf059d97c73f6790948b8736bbe62e5c7 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 13 May 2021 12:17:20 +0200 Subject: [PATCH 072/142] update CI/PR workflow to 21.05 --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index cea8f426..27a4d26c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -2,7 +2,7 @@ name: Galaxy Tool Linting and Tests for push and PR on: [push, pull_request] env: GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_21.01 + GALAXY_BRANCH: release_21.05 MAX_CHUNKS: 4 jobs: # the setup job does two things: From 36a190454212dd29892b4f626551bad205d4f914 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 13 May 2021 15:35:38 +0200 Subject: [PATCH 073/142] add a job to PR workflow that indicates success if no tools were changed --- .github/workflows/pr.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 27a4d26c..074fd885 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -335,3 +335,16 @@ jobs: repository-list: ${{ needs.setup.outputs.repository-list }} shed-target: toolshed shed-key: ${{ secrets.TS_API_KEY }} + + determine-success: + name: check workflow success + runs-on: ubuntu-latest + if: always() + needs: [setup, combine] + strategy: + matrix: + python-version: ['3.7'] + steps: + - name: Indicate failure + if: needs.setup.outputs.tool-list != '' and needs.combine.result != 'success' + run: exit 1 \ No newline at end of file From d81e224a497e73f47cb00e76a7e8a61207d4a013 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 13 May 2021 19:48:30 +0200 Subject: [PATCH 074/142] and --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 074fd885..deb0ca9a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -346,5 +346,5 @@ jobs: python-version: ['3.7'] steps: - name: Indicate failure - if: needs.setup.outputs.tool-list != '' and needs.combine.result != 'success' + if: needs.setup.outputs.tool-list != '' && needs.combine.result != 'success' run: exit 1 \ No newline at end of file From a10272ffcad457e324b07a633e767ab810c0efbc Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 13 May 2021 19:50:11 +0200 Subject: [PATCH 075/142] _outputs --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index deb0ca9a..354ced1b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -340,7 +340,7 @@ jobs: name: check workflow success runs-on: ubuntu-latest if: always() - needs: [setup, combine] + needs: [setup, combine_outputs] strategy: matrix: python-version: ['3.7'] From 7fae9018cb280c059a87cb101bccd99e7540e240 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 14 May 2021 13:52:02 +0200 Subject: [PATCH 076/142] also check linter jobs --- .github/workflows/pr.yaml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 354ced1b..2b088e35 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -337,14 +337,23 @@ jobs: shed-key: ${{ secrets.TS_API_KEY }} determine-success: - name: check workflow success + name: Check workflow success runs-on: ubuntu-latest if: always() - needs: [setup, combine_outputs] + needs: [setup, lint, flake8, lintr, combine_outputs] strategy: matrix: python-version: ['3.7'] steps: - - name: Indicate failure + - name: Indicate tool test failure if: needs.setup.outputs.tool-list != '' && needs.combine.result != 'success' + run: exit 1 + - name: Indicate tool lint failure + if: needs.lint.result != 'success' + run: exit 1 + - name: Indicate python script lint failure + if: needs.flake8.result != 'success' + run: exit 1 + - name: Indicate R script lint failure + if: needs.lintr.result != 'success' run: exit 1 \ No newline at end of file From 55d91a3745e7a5b410f00dd533bff5769ae0c3e0 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 18 May 2021 16:28:03 +0200 Subject: [PATCH 077/142] do not require empty repo list for lint --- .github/workflows/pr.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2b088e35..3ac5990b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -84,7 +84,6 @@ jobs: lint: name: Lint tool-list needs: setup - if: needs.setup.outputs.repository-list != '' runs-on: ubuntu-latest strategy: fail-fast: false From 744f00458fd76ee47dcd7551337e347412f909cd Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 19 May 2021 01:50:19 +0100 Subject: [PATCH 078/142] Fail if previous job results are not success or skipped Also: - Skip `combine_outputs` if no repo was modified - Fix `combine_outputs` job name inside `determine-success` - Small reorderings - Quote if conditions --- .github/workflows/pr.yaml | 42 ++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3ac5990b..ce5089be 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -116,7 +116,7 @@ jobs: flake8: name: Lint Python scripts needs: setup - if: needs.setup.outputs.repository-list != '' + if: ${{ needs.setup.outputs.repository-list != '' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -145,7 +145,7 @@ jobs: lintr: name: Lint R scripts needs: setup - if: needs.setup.outputs.repository-list != '' + if: ${{ needs.setup.outputs.repository-list != '' }} runs-on: ${{ matrix.os }} strategy: matrix: @@ -201,10 +201,9 @@ jobs: # containing HTML and JSON reports for the executed tests test: name: Test tools - # This job runs on Linux - runs-on: ubuntu-latest needs: setup - if: needs.setup.outputs.repository-list != '' + if: ${{ needs.setup.outputs.repository-list != '' }} + runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -263,11 +262,11 @@ jobs: combine_outputs: name: Combine chunked test results needs: [setup, test] + if: ${{ needs.setup.outputs.repository-list != '' }} + runs-on: ubuntu-latest strategy: matrix: python-version: ['3.7'] - # This job runs on Linux - runs-on: ubuntu-latest steps: - uses: actions/download-artifact@v2 with: @@ -301,11 +300,11 @@ jobs: deploy: name: Deploy needs: [setup, lint, flake8, lintr, combine_outputs] + if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'galaxyproject' }} + runs-on: ubuntu-latest strategy: matrix: python-version: ['3.7'] - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/master' && github.repository_owner == 'galaxyproject' steps: - uses: actions/checkout@v2 with: @@ -337,22 +336,19 @@ jobs: determine-success: name: Check workflow success - runs-on: ubuntu-latest - if: always() needs: [setup, lint, flake8, lintr, combine_outputs] - strategy: - matrix: - python-version: ['3.7'] + if: ${{ always() && github.ref != 'refs/heads/master' }} + runs-on: ubuntu-latest steps: - - name: Indicate tool test failure - if: needs.setup.outputs.tool-list != '' && needs.combine.result != 'success' + - name: Check tool lint status + if: ${{ needs.lint.result != 'success' }} + run: exit 1 + - name: Indicate Python script lint status + if: ${{ needs.flake8.result != 'success' && needs.flake8.result != 'skipped' }} run: exit 1 - - name: Indicate tool lint failure - if: needs.lint.result != 'success' + - name: Indicate R script lint status + if: ${{ needs.lintr.result != 'success' && needs.lintr.result != 'skipped' }} run: exit 1 - - name: Indicate python script lint failure - if: needs.flake8.result != 'success' + - name: Check tool test status + if: ${{ needs.combine_outputs.result != 'success' && needs.combine_outputs.result != 'skipped' }} run: exit 1 - - name: Indicate R script lint failure - if: needs.lintr.result != 'success' - run: exit 1 \ No newline at end of file From 435125aedcbd4a2f9409ff6194db313a7086aadb Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 19 May 2021 13:57:54 +0200 Subject: [PATCH 079/142] lint only if tools or repos --- .github/workflows/pr.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ce5089be..c54f8348 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -84,6 +84,7 @@ jobs: lint: name: Lint tool-list needs: setup + if: ${{ needs.setup.outputs.repository-list != '' || needs.setup.outputs.tool-list != '' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -341,7 +342,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check tool lint status - if: ${{ needs.lint.result != 'success' }} + if: ${{ needs.lint.result != 'success' && needs.flake8.result != 'skipped' }} run: exit 1 - name: Indicate Python script lint status if: ${{ needs.flake8.result != 'success' && needs.flake8.result != 'skipped' }} From b113ef21bfbdd63e5b840fb6fe8470c7515b09e9 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Sun, 23 May 2021 18:49:55 +0100 Subject: [PATCH 080/142] Prepare CI for default branch rename from master to main. xref. https://github.com/galaxyproject/tools-iuc/issues/3691 --- .github/workflows/pr.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c54f8348..cf0426b1 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -124,8 +124,6 @@ jobs: matrix: python-version: ['3.7'] steps: - # checkout the repository to master - # and use it as the current working directory - uses: actions/checkout@v2 with: fetch-depth: 1 @@ -153,8 +151,6 @@ jobs: os: [ubuntu-20.04] r-version: ['4.0.1'] steps: - # checkout the repository to master - # and use it as the current working directory - uses: actions/checkout@v2 with: fetch-depth: 1 @@ -301,7 +297,7 @@ jobs: deploy: name: Deploy needs: [setup, lint, flake8, lintr, combine_outputs] - if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'galaxyproject' }} + if: ${{ (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) && github.repository_owner == 'galaxyproject' }} runs-on: ubuntu-latest strategy: matrix: @@ -338,7 +334,7 @@ jobs: determine-success: name: Check workflow success needs: [setup, lint, flake8, lintr, combine_outputs] - if: ${{ always() && github.ref != 'refs/heads/master' }} + if: ${{ always() && github.ref != 'refs/heads/master' && github.ref == 'refs/heads/main' }} runs-on: ubuntu-latest steps: - name: Check tool lint status From dbb558ee82e2db70d77e99f6842ad51960ff72b5 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 15 Mar 2021 12:54:51 +0100 Subject: [PATCH 081/142] CI: provide linter output as artifacts --- .github/workflows/pr.yaml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index cf0426b1..4b70ccc9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -112,6 +112,10 @@ jobs: mode: lint repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} + - uses: actions/upload-artifact@v2 + with: + name: 'Tool linting output' + path: lint_report.txt # flake8 of Python scripts in the changed repositories flake8: @@ -139,7 +143,11 @@ jobs: - name: Install flake8 run: pip install flake8 flake8-import-order - name: Flake8 - run: echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' flake8 + run: echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' flake8 --output-file y --tee + - uses: actions/upload-artifact@v2 + with: + name: 'Python linting output' + path: pylint_report.txt lintr: name: Lint R scripts @@ -187,12 +195,16 @@ jobs: status <- 1 for (l in lnt) { rel_path <- paste(repo, l$filename, sep="/") - write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message), stderr()) + write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message), stdout()) } } } quit(status = status) - shell: Rscript {0} + shell: Rscript {0} | tee rlint_report.txt + - uses: actions/upload-artifact@v2 + with: + name: 'R linting output' + path: rlint_report.txt # Planemo test the changed repositories, each chunk creates an artifact # containing HTML and JSON reports for the executed tests From 79205f5e37471e349bc887ae7661b04946d67afa Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 15 Mar 2021 13:10:37 +0100 Subject: [PATCH 082/142] fix path --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4b70ccc9..7d135224 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -143,7 +143,7 @@ jobs: - name: Install flake8 run: pip install flake8 flake8-import-order - name: Flake8 - run: echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' flake8 --output-file y --tee + run: echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' flake8 --output-file pylint_report.txt --tee - uses: actions/upload-artifact@v2 with: name: 'Python linting output' From ff591df3c1ea14c76712ce926aed631e021fdddb Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 15 Mar 2021 13:21:08 +0100 Subject: [PATCH 083/142] always upload linter artifacts --- .github/workflows/pr.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7d135224..edb3d878 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -113,6 +113,7 @@ jobs: repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} - uses: actions/upload-artifact@v2 + if: always() with: name: 'Tool linting output' path: lint_report.txt @@ -145,6 +146,7 @@ jobs: - name: Flake8 run: echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' flake8 --output-file pylint_report.txt --tee - uses: actions/upload-artifact@v2 + if: always() with: name: 'Python linting output' path: pylint_report.txt @@ -202,6 +204,7 @@ jobs: quit(status = status) shell: Rscript {0} | tee rlint_report.txt - uses: actions/upload-artifact@v2 + if: always() with: name: 'R linting output' path: rlint_report.txt From e21c9e0a259fd7dbdbf1cb50a05236a5782d08ad Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 15 Mar 2021 13:48:37 +0100 Subject: [PATCH 084/142] replace tee --- .github/workflows/pr.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index edb3d878..0b126694 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -197,12 +197,13 @@ jobs: status <- 1 for (l in lnt) { rel_path <- paste(repo, l$filename, sep="/") - write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message), stdout()) + write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message), stderr()) + write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message), "rlint_report.txt") } } } quit(status = status) - shell: Rscript {0} | tee rlint_report.txt + shell: Rscript {0} - uses: actions/upload-artifact@v2 if: always() with: From 2ce8be91f5815e467db92e92f5eaf19a737141ab Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 25 May 2021 10:25:32 +0200 Subject: [PATCH 085/142] append lintr output --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0b126694..ec74b280 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -198,7 +198,7 @@ jobs: for (l in lnt) { rel_path <- paste(repo, l$filename, sep="/") write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message), stderr()) - write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message), "rlint_report.txt") + write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message), "rlint_report.txt", append=TRUE) } } } From a90da4bfb8adf5ce4bbe06420e69896b9d126c6e Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 25 May 2021 23:15:24 +0200 Subject: [PATCH 086/142] provide linter artifacts only in case of error Co-authored-by: Nicola Soranzo --- .github/workflows/pr.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ec74b280..fd0039c2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -113,7 +113,7 @@ jobs: repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} - uses: actions/upload-artifact@v2 - if: always() + if: ${{ failure() }} with: name: 'Tool linting output' path: lint_report.txt @@ -146,7 +146,7 @@ jobs: - name: Flake8 run: echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' flake8 --output-file pylint_report.txt --tee - uses: actions/upload-artifact@v2 - if: always() + if: ${{ failure() }} with: name: 'Python linting output' path: pylint_report.txt @@ -205,7 +205,7 @@ jobs: quit(status = status) shell: Rscript {0} - uses: actions/upload-artifact@v2 - if: always() + if: ${{ failure() }} with: name: 'R linting output' path: rlint_report.txt From 7a7a9d2ba46f756aef742a8f649ea34afb965c08 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 26 May 2021 14:00:07 +0200 Subject: [PATCH 087/142] fix typo in PR --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index fd0039c2..d44cd5fa 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -350,7 +350,7 @@ jobs: determine-success: name: Check workflow success needs: [setup, lint, flake8, lintr, combine_outputs] - if: ${{ always() && github.ref != 'refs/heads/master' && github.ref == 'refs/heads/main' }} + if: ${{ always() && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/main' }} runs-on: ubuntu-latest steps: - name: Check tool lint status From 00daf579450b2bcea6422389863594e453a531e1 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Sat, 11 Sep 2021 07:59:24 +0100 Subject: [PATCH 088/142] Cancel in-progress GitHub workflows when a PR is updated (#3939) --- .github/workflows/pr.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d44cd5fa..37d07f2e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -4,6 +4,9 @@ env: GALAXY_FORK: galaxyproject GALAXY_BRANCH: release_21.05 MAX_CHUNKS: 4 +concurrency: + group: pr-${{ github.ref }} + cancel-in-progress: true jobs: # the setup job does two things: # 1. cache the pip cache and .planemo From 8ab0660cb91354447ff2229fd5be19ac2a1f2fc2 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Tue, 21 Sep 2021 10:32:55 +0200 Subject: [PATCH 089/142] Prevent cancelling of in-progress toolshed uploads --- .github/workflows/pr.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 37d07f2e..510edcf9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -5,7 +5,9 @@ env: GALAXY_BRANCH: release_21.05 MAX_CHUNKS: 4 concurrency: - group: pr-${{ github.ref }} + # group runs by PR, but keep runs on master separate + # because we do not want to cancel toolshed uploads + group: pr-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }} cancel-in-progress: true jobs: # the setup job does two things: From f09c50cd5828c8e1f72facf8c74def13ff49c0f8 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Tue, 21 Sep 2021 11:06:39 +0200 Subject: [PATCH 090/142] Update .github/workflows/pr.yaml Co-authored-by: Nicola Soranzo --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 510edcf9..0a68e1a9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -7,7 +7,7 @@ env: concurrency: # group runs by PR, but keep runs on master separate # because we do not want to cancel toolshed uploads - group: pr-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }} + group: pr-${{ (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main') && github.run_number || github.ref }} cancel-in-progress: true jobs: # the setup job does two things: From ee9bbb2b0f413ae89a383f4d2efa40722c90bede Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Mon, 27 Sep 2021 22:25:43 +0200 Subject: [PATCH 091/142] Test against release 21.09 (#3990) --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0a68e1a9..58766e3f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -2,7 +2,7 @@ name: Galaxy Tool Linting and Tests for push and PR on: [push, pull_request] env: GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_21.05 + GALAXY_BRANCH: release_21.09 MAX_CHUNKS: 4 concurrency: # group runs by PR, but keep runs on master separate From 66628ca7999b9be451d06fa574dda5a4804830fb Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 6 Oct 2021 13:44:49 +0200 Subject: [PATCH 092/142] Let PR workflow check file sizes (#4005) * let PR workflow check file sizes * rename step * Apply suggestions from code review Co-authored-by: Nicola Soranzo * temp bump * Update .github/workflows/pr.yaml Co-authored-by: Nicola Soranzo * add comment in case of size violation * use up-to-date version of the action * try * just use another action for comment * Apply suggestions from code review Co-authored-by: Nicola Soranzo * use peter-evans/create-or-update-comment * try without * Revert "try without" This reverts commit 56409e01076f62382fbcdb1c39f565c1e63161de. * do not comment seems not possible * make final status independent of file size status Co-authored-by: Nicola Soranzo --- .github/workflows/pr.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 58766e3f..70993055 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -4,6 +4,7 @@ env: GALAXY_FORK: galaxyproject GALAXY_BRANCH: release_21.09 MAX_CHUNKS: 4 + MAX_FILE_SIZE: 500k concurrency: # group runs by PR, but keep runs on master separate # because we do not want to cancel toolshed uploads @@ -215,6 +216,35 @@ jobs: name: 'R linting output' path: rlint_report.txt + file_sizes: + name: Check file sizes + needs: setup + if: ${{ github.event_name == 'pull_request' && needs.setup.outputs.repository-list != '' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - name: Save repositories to file + run: echo '${{ needs.setup.outputs.repository-list }}' > repository_list.txt + - name: Check file sizes + run: | + while read repo; do + find "$repo" -size +${{ env.MAX_FILE_SIZE }} + done < repository_list.txt > file_size_report.txt + if [[ -s file_size_report.txt ]]; then + echo "Files larger than ${{ env.MAX_FILE_SIZE }} found" + cat file_size_report.txt + exit 1 + fi + - uses: actions/upload-artifact@v2 + if: ${{ failure() }} + with: + name: 'File size report' + path: file_size_report.txt + # Planemo test the changed repositories, each chunk creates an artifact # containing HTML and JSON reports for the executed tests test: From 9390ce453131d0f3289793325bb1e8fbe89aa1f3 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 14 Dec 2021 12:19:41 +0100 Subject: [PATCH 093/142] CI: Check file sizes only for changed files (#4291) * check file sizes only for changed files as pointed out here: https://github.com/galaxyproject/tools-iuc/pull/3973#issuecomment-944247753 reducing the size of test data does not decrease the repo size if the files are already in. so we should only check for the sizes of changed files and be strict with them * Update .github/workflows/pr.yaml Co-authored-by: Nicola Soranzo Co-authored-by: Nicola Soranzo --- .github/workflows/pr.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 70993055..917119e0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -27,6 +27,7 @@ jobs: tool-list: ${{ steps.discover.outputs.tool-list }} chunk-count: ${{ steps.discover.outputs.chunk-count }} chunk-list: ${{ steps.discover.outputs.chunk-list }} + commit-range: ${{ steps.discover.outputs.commit-range }} strategy: matrix: python-version: ['3.7'] @@ -226,14 +227,10 @@ jobs: steps: - uses: actions/checkout@v2 with: - fetch-depth: 1 - - name: Save repositories to file - run: echo '${{ needs.setup.outputs.repository-list }}' > repository_list.txt + fetch-depth: 0 - name: Check file sizes run: | - while read repo; do - find "$repo" -size +${{ env.MAX_FILE_SIZE }} - done < repository_list.txt > file_size_report.txt + find $(git diff --name-only ${{ needs.setup.outputs.commit-range }}) -type f -size +${{ env.MAX_FILE_SIZE }} > file_size_report.txt if [[ -s file_size_report.txt ]]; then echo "Files larger than ${{ env.MAX_FILE_SIZE }} found" cat file_size_report.txt @@ -384,7 +381,7 @@ jobs: determine-success: name: Check workflow success - needs: [setup, lint, flake8, lintr, combine_outputs] + needs: [setup, lint, flake8, lintr, file_sizes, combine_outputs] if: ${{ always() && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/main' }} runs-on: ubuntu-latest steps: @@ -397,6 +394,9 @@ jobs: - name: Indicate R script lint status if: ${{ needs.lintr.result != 'success' && needs.lintr.result != 'skipped' }} run: exit 1 + - name: Indicate file size check status + if: ${{ needs.file_sizes.result != 'success' && needs.file_sizes.result != 'skipped' }} + run: exit 1 - name: Check tool test status if: ${{ needs.combine_outputs.result != 'success' && needs.combine_outputs.result != 'skipped' }} run: exit 1 From b8b89c2c092f50b244384e31bc2f701f2bfd3778 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 15 Dec 2021 17:37:21 +0100 Subject: [PATCH 094/142] only check added files otherwise find fails for deleted files --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 917119e0..c925f9bc 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -230,7 +230,7 @@ jobs: fetch-depth: 0 - name: Check file sizes run: | - find $(git diff --name-only ${{ needs.setup.outputs.commit-range }}) -type f -size +${{ env.MAX_FILE_SIZE }} > file_size_report.txt + find $(git diff --diff-filter A --name-only ${{ needs.setup.outputs.commit-range }}) -type f -size +${{ env.MAX_FILE_SIZE }} > file_size_report.txt if [[ -s file_size_report.txt ]]; then echo "Files larger than ${{ env.MAX_FILE_SIZE }} found" cat file_size_report.txt From e88a0bd6b03a23fd831ef820b18fa7a2da4bf177 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 5 Jan 2022 10:27:32 +0100 Subject: [PATCH 095/142] only call find if there is a diff --- .github/workflows/pr.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c925f9bc..73b3cc1d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -230,7 +230,11 @@ jobs: fetch-depth: 0 - name: Check file sizes run: | - find $(git diff --diff-filter A --name-only ${{ needs.setup.outputs.commit-range }}) -type f -size +${{ env.MAX_FILE_SIZE }} > file_size_report.txt + touch file_size_report.txt + GITDIFF=$(git diff --diff-filter A --name-only ${{ needs.setup.outputs.commit-range }}) + if [ -n "$GITDIFF" ]; then + find "$GITDIFF" -type f -size +${{ env.MAX_FILE_SIZE }} > file_size_report.txt + fi if [[ -s file_size_report.txt ]]; then echo "Files larger than ${{ env.MAX_FILE_SIZE }} found" cat file_size_report.txt From df7ec0ea0dda60a470b7311907cdd7682670aa8b Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 5 Jan 2022 11:02:48 +0100 Subject: [PATCH 096/142] only exclude deletions from git diff --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 73b3cc1d..f3097f17 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -231,7 +231,7 @@ jobs: - name: Check file sizes run: | touch file_size_report.txt - GITDIFF=$(git diff --diff-filter A --name-only ${{ needs.setup.outputs.commit-range }}) + GITDIFF=$(git diff --diff-filter=d --name-only ${{ needs.setup.outputs.commit-range }}) if [ -n "$GITDIFF" ]; then find "$GITDIFF" -type f -size +${{ env.MAX_FILE_SIZE }} > file_size_report.txt fi From 1459d1627ca6171d7c8cfa03a5625610a978726d Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 5 Jan 2022 11:26:08 +0100 Subject: [PATCH 097/142] replace new line --- .github/workflows/pr.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f3097f17..8f3a4900 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -231,7 +231,8 @@ jobs: - name: Check file sizes run: | touch file_size_report.txt - GITDIFF=$(git diff --diff-filter=d --name-only ${{ needs.setup.outputs.commit-range }}) + GITDIFF=$(git diff --diff-filter=d --name-only ${{ needs.setup.outputs.commit-range }} | tr '\n' ' ') + echo GITDIFF "$GITDIFF" if [ -n "$GITDIFF" ]; then find "$GITDIFF" -type f -size +${{ env.MAX_FILE_SIZE }} > file_size_report.txt fi From c1c6fa38b2f06b927a2d8cc239897d9ec8eec8d6 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 5 Jan 2022 14:39:00 +0100 Subject: [PATCH 098/142] try to save git diff to file --- .github/workflows/pr.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8f3a4900..83887dc7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -231,11 +231,10 @@ jobs: - name: Check file sizes run: | touch file_size_report.txt - GITDIFF=$(git diff --diff-filter=d --name-only ${{ needs.setup.outputs.commit-range }} | tr '\n' ' ') - echo GITDIFF "$GITDIFF" - if [ -n "$GITDIFF" ]; then - find "$GITDIFF" -type f -size +${{ env.MAX_FILE_SIZE }} > file_size_report.txt - fi + git diff --diff-filter=d --name-only ${{ needs.setup.outputs.commit-range }} > git.diff + while read line; do + find "$line" -type f -size +${{ env.MAX_FILE_SIZE }} >> file_size_report.txt + done < git.diff if [[ -s file_size_report.txt ]]; then echo "Files larger than ${{ env.MAX_FILE_SIZE }} found" cat file_size_report.txt From 5459f2f9ff9700acfe3f8cdbb66609a4945798f7 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 17 Jan 2022 09:39:38 +0000 Subject: [PATCH 099/142] Increase `MAX_FILE_SIZE` to 1Mb (#4327) To be in line with the IUC Standards recommendation: https://galaxy-iuc-standards.readthedocs.io/en/latest/best_practices/integration_checklist.html#testing-your-tool --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 83887dc7..ab86378a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -4,7 +4,7 @@ env: GALAXY_FORK: galaxyproject GALAXY_BRANCH: release_21.09 MAX_CHUNKS: 4 - MAX_FILE_SIZE: 500k + MAX_FILE_SIZE: 1M concurrency: # group runs by PR, but keep runs on master separate # because we do not want to cancel toolshed uploads From a1071ddccaa2e1b27fc0ac8edc6ef590976c053b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 4 Feb 2022 11:40:23 +0100 Subject: [PATCH 100/142] Test against 22.01 --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ab86378a..feca74ca 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -2,7 +2,7 @@ name: Galaxy Tool Linting and Tests for push and PR on: [push, pull_request] env: GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_21.09 + GALAXY_BRANCH: release_22.01 MAX_CHUNKS: 4 MAX_FILE_SIZE: 1M concurrency: From a9ffdca54a5577e75c1d6bb2a2f6c3a66416e9bb Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 28 Feb 2022 19:19:04 +0000 Subject: [PATCH 101/142] Sync CI workflows with tools-devteam (#4433) --- .github/workflows/pr.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index feca74ca..daf9a723 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -6,8 +6,8 @@ env: MAX_CHUNKS: 4 MAX_FILE_SIZE: 1M concurrency: - # group runs by PR, but keep runs on master separate - # because we do not want to cancel toolshed uploads + # Group runs by PR, but keep runs on the default branch separate + # because we do not want to cancel ToolShed uploads group: pr-${{ (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main') && github.run_number || github.ref }} cancel-in-progress: true jobs: @@ -98,8 +98,6 @@ jobs: matrix: python-version: ['3.7'] steps: - # checkout the repository - # and use it as the current working directory - uses: actions/checkout@v2 with: fetch-depth: 1 @@ -222,8 +220,6 @@ jobs: needs: setup if: ${{ github.event_name == 'pull_request' && needs.setup.outputs.repository-list != '' }} runs-on: ubuntu-latest - strategy: - fail-fast: false steps: - uses: actions/checkout@v2 with: @@ -268,8 +264,6 @@ jobs: ports: - 5432:5432 steps: - # checkout the repository - # and use it as the current working directory - uses: actions/checkout@v2 with: fetch-depth: 1 From de9dffa373451f4563c7a4a395be46d39e4d2de5 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 19 Apr 2022 11:23:41 +0200 Subject: [PATCH 102/142] use the available cores for Galaxy jobs (#4490) relies on https://github.com/galaxyproject/planemo-ci-action/pull/25 --- .github/workflows/pr.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index daf9a723..9b2059fe 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -282,6 +282,9 @@ jobs: with: path: ~/.planemo key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v1 + id: cpu-cores - name: Planemo test uses: galaxyproject/planemo-ci-action@v1 id: test @@ -292,6 +295,7 @@ jobs: galaxy-branch: ${{ env.GALAXY_BRANCH }} chunk: ${{ matrix.chunk }} chunk-count: ${{ needs.setup.outputs.chunk-count }} + galaxy-slots: ${{ steps.cpu-cores.outputs.count }} - uses: actions/upload-artifact@v2 with: name: 'Tool test output ${{ matrix.chunk }}' From 56580e1e457ab4dcc07dccaffff49bb3073d54e3 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Thu, 30 Jun 2022 23:20:46 +0200 Subject: [PATCH 103/142] Target release_22.05 (#4625) --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9b2059fe..afd3d616 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -2,7 +2,7 @@ name: Galaxy Tool Linting and Tests for push and PR on: [push, pull_request] env: GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_22.01 + GALAXY_BRANCH: release_22.05 MAX_CHUNKS: 4 MAX_FILE_SIZE: 1M concurrency: From 06c2b7a4f3acb3a0247dd8eb7ddc0e642aea64ab Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 24 Aug 2022 19:39:44 +0200 Subject: [PATCH 104/142] CI: lintr updates (#4743) - fix function that emits a deprecation warning - also output line (can't hurt) --- .github/workflows/pr.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index afd3d616..3b8c5a3f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -189,7 +189,7 @@ jobs: - name: lintr run: | library(lintr) - linters <- with_defaults(line_length_linter = NULL, cyclocomp_linter = NULL, object_usage_linter = NULL) + linters <- linters_with_defaults(line_length_linter = NULL, cyclocomp_linter = NULL, object_usage_linter = NULL) con <- file("repository_list.txt", "r") status <- 0 while (TRUE) { @@ -202,8 +202,8 @@ jobs: status <- 1 for (l in lnt) { rel_path <- paste(repo, l$filename, sep="/") - write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message), stderr()) - write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message), "rlint_report.txt", append=TRUE) + write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message, paste("(", l$line, ")")), stderr()) + write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message, paste("(", l$line, ")")), "rlint_report.txt", append=TRUE) } } } From 75232f08462fdd7fe9c9b1352fb28d78e933794d Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 13 Sep 2022 11:26:51 +0200 Subject: [PATCH 105/142] apply test timeout also to PRs as in CI .. otherwise we will just have failing CI jobs for long running tools --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3b8c5a3f..4ccb8b80 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -296,6 +296,8 @@ jobs: chunk: ${{ matrix.chunk }} chunk-count: ${{ needs.setup.outputs.chunk-count }} galaxy-slots: ${{ steps.cpu-cores.outputs.count }} + # Limit each test to 15 minutes + test_timeout: 900 - uses: actions/upload-artifact@v2 with: name: 'Tool test output ${{ matrix.chunk }}' From fef1343bb5c577da262cfea3e37a8e353ff9833e Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 20 Oct 2022 01:10:06 +0100 Subject: [PATCH 106/142] Update version of GitHub actions used (#4850) Fix: ``` Error: Unable to resolve action `r-lib/actions@master`, unable to find version `master` ``` Also update R version to 'release'. --- .github/workflows/pr.yaml | 62 +++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4ccb8b80..39bfff52 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -44,17 +44,17 @@ jobs: - name: Determine latest commit in the Galaxy repo id: get-galaxy-sha run: echo "::set-output name=galaxy-head-sha::$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-pip with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} - name: Cache .planemo - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-planemo with: path: ~/.planemo @@ -65,7 +65,7 @@ jobs: run: pip install wheel - name: Install flake8 run: pip install flake8 flake8-import-order - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks @@ -98,14 +98,14 @@ jobs: matrix: python-version: ['3.7'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-pip with: path: ~/.cache/pip @@ -117,7 +117,7 @@ jobs: mode: lint repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: ${{ failure() }} with: name: 'Tool linting output' @@ -134,14 +134,14 @@ jobs: matrix: python-version: ['3.7'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-pip with: path: ~/.cache/pip @@ -150,7 +150,7 @@ jobs: run: pip install flake8 flake8-import-order - name: Flake8 run: echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' flake8 --output-file pylint_report.txt --tee - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: ${{ failure() }} with: name: 'Python linting output' @@ -164,16 +164,16 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - r-version: ['4.0.1'] + r-version: ['release'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1 - - uses: r-lib/actions/setup-r@master + - uses: r-lib/actions/setup-r@v2 with: r-version: ${{ matrix.r-version }} - name: Cache R packages - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ env.R_LIBS_USER }} key: r_cache_${{ matrix.os }}_${{ matrix.r-version }} @@ -209,7 +209,7 @@ jobs: } quit(status = status) shell: Rscript {0} - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: ${{ failure() }} with: name: 'R linting output' @@ -221,7 +221,7 @@ jobs: if: ${{ github.event_name == 'pull_request' && needs.setup.outputs.repository-list != '' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Check file sizes @@ -236,7 +236,7 @@ jobs: cat file_size_report.txt exit 1 fi - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: ${{ failure() }} with: name: 'File size report' @@ -264,20 +264,20 @@ jobs: ports: - 5432:5432 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-pip with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - name: Cache .planemo - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-planemo with: path: ~/.planemo @@ -298,7 +298,7 @@ jobs: galaxy-slots: ${{ steps.cpu-cores.outputs.count }} # Limit each test to 15 minutes test_timeout: 900 - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: 'Tool test output ${{ matrix.chunk }}' path: upload @@ -317,14 +317,14 @@ jobs: matrix: python-version: ['3.7'] steps: - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: path: artifacts - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-pip with: path: ~/.cache/pip @@ -335,7 +335,7 @@ jobs: with: mode: combine html-report: true - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: 'All tool test results' path: upload @@ -355,14 +355,14 @@ jobs: matrix: python-version: ['3.7'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 1 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-pip with: path: ~/.cache/pip From c8ce89eb18690e5c3d9c92641bb4c221326c6bc9 Mon Sep 17 00:00:00 2001 From: Lucille Delisle Date: Tue, 1 Nov 2022 23:02:09 +0100 Subject: [PATCH 107/142] remove deprecated set-output syntax --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 39bfff52..be48ecea 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -43,7 +43,7 @@ jobs: echo 'event.after: ${{ github.event.after }}' - name: Determine latest commit in the Galaxy repo id: get-galaxy-sha - run: echo "::set-output name=galaxy-head-sha::$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" + run: echo "galaxy-head-sha=$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" >> $GITHUB_OUTPUT - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 03bd2b96c01e232f2db44b71ada1137bd91ff10b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Gallardo?= Date: Mon, 9 Jan 2023 16:26:03 +0100 Subject: [PATCH 108/142] Update pr.yaml --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index be48ecea..498a6009 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -297,7 +297,7 @@ jobs: chunk-count: ${{ needs.setup.outputs.chunk-count }} galaxy-slots: ${{ steps.cpu-cores.outputs.count }} # Limit each test to 15 minutes - test_timeout: 900 + test_timeout: 1800 - uses: actions/upload-artifact@v3 with: name: 'Tool test output ${{ matrix.chunk }}' From 04df4ca109984eb26e53cf216904fef7d48765c0 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 11 Jan 2023 12:06:05 +0100 Subject: [PATCH 109/142] report deployment failure --- .github/workflows/pr.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 498a6009..5e306dda 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -382,6 +382,24 @@ jobs: repository-list: ${{ needs.setup.outputs.repository-list }} shed-target: toolshed shed-key: ${{ secrets.TS_API_KEY }} + # report to the PR if deployment failed + - name: get PR number + uses: 8BitJonny/gh-get-current-pr@2.2.0 + id: getpr + if: ${{ failure() }} + with: + github-token: ${{secrets.GITHUB_TOKEN}} + sha: ${{ github.event.after }} + - name: Create comment + uses: peter-evans/create-or-update-comment@v2 + if: ${{ failure() }} + with: + token: ${{ secrets.PAT }} + issue-number: ${{ steps.getpr.outputs.number }} + body: | + Attention: deployment failed! + + https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} determine-success: name: Check workflow success From a53f79f5e6b6039890e4f13e8228a5f7b5685d2d Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sun, 15 Jan 2023 12:20:13 +0100 Subject: [PATCH 110/142] Apply suggestions from code review Co-authored-by: Nicola Soranzo --- .github/workflows/pr.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5e306dda..ef1a1008 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -383,12 +383,11 @@ jobs: shed-target: toolshed shed-key: ${{ secrets.TS_API_KEY }} # report to the PR if deployment failed - - name: get PR number + - name: Get PR object uses: 8BitJonny/gh-get-current-pr@2.2.0 id: getpr if: ${{ failure() }} with: - github-token: ${{secrets.GITHUB_TOKEN}} sha: ${{ github.event.after }} - name: Create comment uses: peter-evans/create-or-update-comment@v2 From 5d2be410f9c0b5df5f37af9560505714b2ca1c63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Gallardo?= Date: Thu, 19 Jan 2023 17:30:18 +0100 Subject: [PATCH 111/142] Fix tests --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ef1a1008..e5b708bb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -189,7 +189,7 @@ jobs: - name: lintr run: | library(lintr) - linters <- linters_with_defaults(line_length_linter = NULL, cyclocomp_linter = NULL, object_usage_linter = NULL) + linters <- linters_with_defaults(line_length_linter = NULL, cyclocomp_linter = NULL, object_usage_linter = NULL, object_name_linter = NULL) con <- file("repository_list.txt", "r") status <- 0 while (TRUE) { From 4b1ec74bbf7d90e9a5f7a1b225f251009c17a102 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 20 Jan 2023 11:55:03 +0100 Subject: [PATCH 112/142] Topic/bump test (#5068) * trigger deployment message on anything but success * test bump --- .github/workflows/pr.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e5b708bb..9a498f03 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -382,21 +382,25 @@ jobs: repository-list: ${{ needs.setup.outputs.repository-list }} shed-target: toolshed shed-key: ${{ secrets.TS_API_KEY }} + + deploy-report: + name: Report deploy status + needs: [deploy] + if: ${{ jobs.deploy.result != "success" and (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) && github.repository_owner == 'galaxyproject' }} + runs-on: ubuntu-latest # report to the PR if deployment failed - name: Get PR object uses: 8BitJonny/gh-get-current-pr@2.2.0 id: getpr - if: ${{ failure() }} with: sha: ${{ github.event.after }} - name: Create comment uses: peter-evans/create-or-update-comment@v2 - if: ${{ failure() }} with: token: ${{ secrets.PAT }} issue-number: ${{ steps.getpr.outputs.number }} body: | - Attention: deployment failed! + Attention: deployment ${{ jobs.deploy.result }}! https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} From 44970641e735be3cafb59ab233e48765adb3e0b6 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 20 Jan 2023 11:58:37 +0100 Subject: [PATCH 113/142] Revert "Topic/bump test (#5068)" (#5069) This reverts commit 356f921c6fb6b9c68f5df46510b23038fe1058fa. merged on accident --- .github/workflows/pr.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9a498f03..e5b708bb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -382,25 +382,21 @@ jobs: repository-list: ${{ needs.setup.outputs.repository-list }} shed-target: toolshed shed-key: ${{ secrets.TS_API_KEY }} - - deploy-report: - name: Report deploy status - needs: [deploy] - if: ${{ jobs.deploy.result != "success" and (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) && github.repository_owner == 'galaxyproject' }} - runs-on: ubuntu-latest # report to the PR if deployment failed - name: Get PR object uses: 8BitJonny/gh-get-current-pr@2.2.0 id: getpr + if: ${{ failure() }} with: sha: ${{ github.event.after }} - name: Create comment uses: peter-evans/create-or-update-comment@v2 + if: ${{ failure() }} with: token: ${{ secrets.PAT }} issue-number: ${{ steps.getpr.outputs.number }} body: | - Attention: deployment ${{ jobs.deploy.result }}! + Attention: deployment failed! https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} From 7716193b6b832baf9d2c97c4c938126c8218eb8f Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 20 Jan 2023 11:56:12 +0100 Subject: [PATCH 114/142] trigger deployment message on anything but success --- .github/workflows/pr.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e5b708bb..3b8a3a1c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -382,21 +382,26 @@ jobs: repository-list: ${{ needs.setup.outputs.repository-list }} shed-target: toolshed shed-key: ${{ secrets.TS_API_KEY }} + + deploy-report: + name: Report deploy status + needs: [deploy] + if: ${{ always() && needs.deploy.result != 'success' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) && github.repository_owner == 'galaxyproject' }} + runs-on: ubuntu-latest + steps: # report to the PR if deployment failed - name: Get PR object uses: 8BitJonny/gh-get-current-pr@2.2.0 id: getpr - if: ${{ failure() }} with: sha: ${{ github.event.after }} - name: Create comment uses: peter-evans/create-or-update-comment@v2 - if: ${{ failure() }} with: token: ${{ secrets.PAT }} issue-number: ${{ steps.getpr.outputs.number }} body: | - Attention: deployment failed! + Attention: deployment ${{ needs.deploy.result }}! https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} From f17d40d10c19f9ebfb9a2bfeb2a4f428de593be3 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Mon, 27 Mar 2023 14:08:28 +0200 Subject: [PATCH 115/142] Test commits pushed to main, not other branches ... but keep testing PR commits as before. --- .github/workflows/pr.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3b8a3a1c..17b4becf 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -1,5 +1,9 @@ name: Galaxy Tool Linting and Tests for push and PR -on: [push, pull_request] +on: + pull_request: + push: + - main + - master env: GALAXY_FORK: galaxyproject GALAXY_BRANCH: release_22.05 From 6de57d3701433032036f7aa9cf83a257cfe46a79 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Mon, 27 Mar 2023 17:01:18 +0200 Subject: [PATCH 116/142] Add missing branches keyword --- .github/workflows/pr.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 17b4becf..adaae9ab 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -2,8 +2,9 @@ name: Galaxy Tool Linting and Tests for push and PR on: pull_request: push: - - main - - master + branches: + - main + - master env: GALAXY_FORK: galaxyproject GALAXY_BRANCH: release_22.05 From 5dd0ff7cf9ceaddef04a23c65f7dfef77576f2be Mon Sep 17 00:00:00 2001 From: Dannon Date: Wed, 5 Apr 2023 14:56:31 -0400 Subject: [PATCH 117/142] Target release_23.0 (#5229) --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index adaae9ab..4f0ebb05 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -7,7 +7,7 @@ on: - master env: GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_22.05 + GALAXY_BRANCH: release_23.0 MAX_CHUNKS: 4 MAX_FILE_SIZE: 1M concurrency: From 6e11e7875250e45e5599747f50a7d24b8aacfdf8 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 12 Apr 2023 16:34:00 +0200 Subject: [PATCH 118/142] Circos and PR workflow fixes (#5242) * add test 3 * try to fix https://github.com/galaxyproject/tools-iuc/issues/5240 * try always * add weekly CI job checking for missing containers * circos: test for a resolvable environment * circos: fix test by sorting output * circos: remove python and gawk requirements python restrictions should come via conda, removing it seems to allow to resolve the environment. also gawk seems not to be used in the tools, so it should come via conda. Co-authored-by: Nicola Soranzo * circos: bump --------- Co-authored-by: Helena Rasche Co-authored-by: Nicola Soranzo --- .github/workflows/pr.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4f0ebb05..424e3ba3 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -316,7 +316,7 @@ jobs: combine_outputs: name: Combine chunked test results needs: [setup, test] - if: ${{ needs.setup.outputs.repository-list != '' }} + if: ${{ always() && needs.setup.outputs.repository-list != '' }} runs-on: ubuntu-latest strategy: matrix: @@ -334,6 +334,12 @@ jobs: with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} + - name: Check if all test chunks succeeded + run: | + NFILES=$(ls artifacts/ | wc -l) + if [[ "${{ needs.setup.outputs.chunk-count }}" != "$NFILES" ]]; then + exit 1 + fi - name: Combine outputs uses: galaxyproject/planemo-ci-action@v1 id: combine From 9b689ef2288f7f6f513a76c0151c34668406d0e7 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Thu, 13 Apr 2023 13:42:58 +0200 Subject: [PATCH 119/142] fix PR workflow (#5246) - do not count tool linting artifact - and move test to the end such that the combined tool test output artifact is uploaded in any case --- .github/workflows/pr.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 424e3ba3..84dfc156 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -334,12 +334,6 @@ jobs: with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - name: Check if all test chunks succeeded - run: | - NFILES=$(ls artifacts/ | wc -l) - if [[ "${{ needs.setup.outputs.chunk-count }}" != "$NFILES" ]]; then - exit 1 - fi - name: Combine outputs uses: galaxyproject/planemo-ci-action@v1 id: combine @@ -355,6 +349,12 @@ jobs: id: check with: mode: check + - name: Check if all test chunks succeeded + run: | + NFILES=$(ls artifacts/ | grep "Tool test output" | wc -l) + if [[ "${{ needs.setup.outputs.chunk-count }}" != "$NFILES" ]]; then + exit 1 + fi # deploy the tools to the toolsheds (first TTS for testing) deploy: From 38b5b47851ffe7b54613c1b3f48051f75cc11b61 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 19 Apr 2023 20:25:56 +0200 Subject: [PATCH 120/142] PR workflow: fail on warning xref https://github.com/galaxyproject/galaxy/pull/15962#issuecomment-1515169606 --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 84dfc156..fcad2e39 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -120,6 +120,7 @@ jobs: id: lint with: mode: lint + fail-level: warn repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} - uses: actions/upload-artifact@v3 From aec27f0ca26419e3b5e48a011c4041a153576e69 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Tue, 9 May 2023 15:08:27 +0200 Subject: [PATCH 121/142] Do not initiate tool tests if no tool was touched (#5219) * Do not initiate tool tests if no tool folder was changed * Provide fallback workflow with required job for PRs * Fix indentation of comment lines * Add closing single quote --- .github/workflows/pr.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index fcad2e39..44b8dfa2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -1,10 +1,20 @@ name: Galaxy Tool Linting and Tests for push and PR on: pull_request: + paths-ignore: + - '.github/**' + - 'deprecated/**' + - 'docs/**' + - '*' push: branches: - main - master + paths-ignore: + - '.github/**' + - 'deprecated/**' + - 'docs/**' + - '*' env: GALAXY_FORK: galaxyproject GALAXY_BRANCH: release_23.0 From 430253c739920edc27959ffe7117902684be4963 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 7 Aug 2023 10:57:41 +0200 Subject: [PATCH 122/142] bump CI workflows to 23.1 and clean dornet folder for consistency with weekly CI --- .github/workflows/pr.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 44b8dfa2..2b7400f0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,7 +17,7 @@ on: - '*' env: GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_23.0 + GALAXY_BRANCH: release_23.1 MAX_CHUNKS: 4 MAX_FILE_SIZE: 1M concurrency: @@ -301,6 +301,8 @@ jobs: - name: Get number of CPU cores uses: SimenB/github-actions-cpu-cores@v1 id: cpu-cores + - name: Clean dotnet folder for space + run: rm -Rf /usr/share/dotnet - name: Planemo test uses: galaxyproject/planemo-ci-action@v1 id: test From 852ab59e6d142aff94286697f2a8bb5891b7963d Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 7 Aug 2023 14:35:41 +0200 Subject: [PATCH 123/142] Skip lint on merge (#5429) * merge: skip python and R linting and use fail level error * test bump * Revert bump --- .github/workflows/pr.yaml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2b7400f0..85cee20c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -125,12 +125,21 @@ jobs: with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} + + - name: Set fail level for pull request + if: ${{ github.event_name == 'pull_request' }} + run: + echo "FAIL_LEVEL=warn" >> "$GITHUB_ENV" + - name: Set fail level for merge + if: ${{ github.event_name != 'pull_request' }} + run: + echo "FAIL_LEVEL=error" >> "$GITHUB_ENV" - name: Planemo lint uses: galaxyproject/planemo-ci-action@v1 id: lint with: mode: lint - fail-level: warn + fail-level: ${{ env.FAIL_LEVEL }} repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} - uses: actions/upload-artifact@v3 @@ -143,7 +152,7 @@ jobs: flake8: name: Lint Python scripts needs: setup - if: ${{ needs.setup.outputs.repository-list != '' }} + if: ${{ github.event_name == 'pull_request' && needs.setup.outputs.repository-list != '' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -175,7 +184,7 @@ jobs: lintr: name: Lint R scripts needs: setup - if: ${{ needs.setup.outputs.repository-list != '' }} + if: ${{ github.event_name == 'pull_request' && needs.setup.outputs.repository-list != '' }} runs-on: ${{ matrix.os }} strategy: matrix: @@ -372,7 +381,7 @@ jobs: # deploy the tools to the toolsheds (first TTS for testing) deploy: name: Deploy - needs: [setup, lint, flake8, lintr, combine_outputs] + needs: [setup, lint, combine_outputs] if: ${{ (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) && github.repository_owner == 'galaxyproject' }} runs-on: ubuntu-latest strategy: From 069f1b90e92c6442a23d84798cef910b22c67271 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 15 Dec 2023 18:09:29 +0100 Subject: [PATCH 124/142] replace lintr by styler styler the R code linter/styler from the tidyverse. in contrast to lintr it can also fix the found problems (added a script that can be used for checking and fixing) --- .github/workflows/pr.yaml | 37 +++++++++---------------------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 85cee20c..a66b14fb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -202,38 +202,19 @@ jobs: with: path: ${{ env.R_LIBS_USER }} key: r_cache_${{ matrix.os }}_${{ matrix.r-version }} - - name: Install non-R lintr dependencies - run: sudo apt-get install libcurl4-openssl-dev - - name: Install lintr - run: | - install.packages('remotes') - remotes::install_cran("lintr") - shell: Rscript {0} + - name: Install packages + uses: r-lib/actions/setup-r-dependencies@v2 + with: + packages: | + any::argparse + any::styler - name: Save repositories to file run: echo '${{ needs.setup.outputs.repository-list }}' > repository_list.txt - name: lintr run: | - library(lintr) - linters <- linters_with_defaults(line_length_linter = NULL, cyclocomp_linter = NULL, object_usage_linter = NULL, object_name_linter = NULL) - con <- file("repository_list.txt", "r") - status <- 0 - while (TRUE) { - repo <- readLines(con, n = 1) - if (length(repo) == 0) { - break - } - lnt <- lint_dir(repo, relative_path=T, linters=linters) - if (length(lnt) > 0) { - status <- 1 - for (l in lnt) { - rel_path <- paste(repo, l$filename, sep="/") - write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message, paste("(", l$line, ")")), stderr()) - write(paste(paste(rel_path, l$line_number, l$column_number, sep=":"), l$message, paste("(", l$line, ")")), "rlint_report.txt", append=TRUE) - } - } - } - quit(status = status) - shell: Rscript {0} + set -eo pipefail + echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' .github/styler.R --dry off + git diff --exit-code | tee rlint_report.txt - uses: actions/upload-artifact@v3 if: ${{ failure() }} with: From 167ef559c27421862d22fd0d291ddb44875da129 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 15 Dec 2023 19:39:47 +0100 Subject: [PATCH 125/142] make lintr job (aka styler) a string requirement that is do not deploy if failing --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a66b14fb..73557167 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -184,7 +184,7 @@ jobs: lintr: name: Lint R scripts needs: setup - if: ${{ github.event_name == 'pull_request' && needs.setup.outputs.repository-list != '' }} + if: ${{ needs.setup.outputs.repository-list != '' }} runs-on: ${{ matrix.os }} strategy: matrix: From ec50f557e588b62937c1668b78a8fc1ddf81ae03 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 15 Dec 2023 19:54:27 +0100 Subject: [PATCH 126/142] debug --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 73557167..b8e3ad8a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -214,6 +214,7 @@ jobs: run: | set -eo pipefail echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' .github/styler.R --dry off + git status git diff --exit-code | tee rlint_report.txt - uses: actions/upload-artifact@v3 if: ${{ failure() }} From 28133c0537989326f71c80a412ee6eb3ceffcde1 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 15 Dec 2023 20:08:42 +0100 Subject: [PATCH 127/142] fix xargs --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b8e3ad8a..026b0186 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -213,7 +213,7 @@ jobs: - name: lintr run: | set -eo pipefail - echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' .github/styler.R --dry off + echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' -n 1 ./.github/styler.R --dry off git status git diff --exit-code | tee rlint_report.txt - uses: actions/upload-artifact@v3 From 532dbe08b3396919b417101e151d580119a3ba9f Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 18 Dec 2023 10:29:09 +0100 Subject: [PATCH 128/142] Remove unneded step Co-authored-by: Nicola Soranzo --- .github/workflows/pr.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 026b0186..bae5bf36 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -208,8 +208,6 @@ jobs: packages: | any::argparse any::styler - - name: Save repositories to file - run: echo '${{ needs.setup.outputs.repository-list }}' > repository_list.txt - name: lintr run: | set -eo pipefail From 23c4a91936f52a1d7e2aa04af51d13f33f9c1472 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 10 Jan 2024 23:48:24 +0100 Subject: [PATCH 129/142] try comments (#5707) --- .github/workflows/pr.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index bae5bf36..f37b2289 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -342,6 +342,7 @@ jobs: with: mode: combine html-report: true + markdown-report: true - uses: actions/upload-artifact@v3 with: name: 'All tool test results' @@ -357,6 +358,24 @@ jobs: if [[ "${{ needs.setup.outputs.chunk-count }}" != "$NFILES" ]]; then exit 1 fi + - name: Get PR object + if: failure() + uses: 8BitJonny/gh-get-current-pr@2.2.0 + id: failed_pr_number + with: + sha: ${{ github.event.pull_request.head.sha }} + - name: Debug GetPR output + if: failure() + run: | + echo ${{ steps.failed_pr_number.outputs.number }} + - name: Post comment with test report + if: failure() + uses: peter-evans/create-or-update-comment@v3 + with: + token: ${{ secrets.PAT }} + issue-number: ${{ steps.failed_pr_number.outputs.number }} + body-path: upload/tool_test_output.md + # deploy the tools to the toolsheds (first TTS for testing) deploy: From bc9149faa0d116edf96d5b723309568686937198 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 11 Jan 2024 17:17:26 +0100 Subject: [PATCH 130/142] Update python version to 3.11 and galaxy branch to 23.2 --- .github/workflows/pr.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f37b2289..9957256c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,7 +17,7 @@ on: - '*' env: GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_23.1 + GALAXY_BRANCH: release_23.2 MAX_CHUNKS: 4 MAX_FILE_SIZE: 1M concurrency: @@ -45,7 +45,7 @@ jobs: commit-range: ${{ steps.discover.outputs.commit-range }} strategy: matrix: - python-version: ['3.7'] + python-version: ['3.11'] steps: - name: Print github context properties run: | @@ -111,7 +111,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.7'] + python-version: ['3.11'] steps: - uses: actions/checkout@v3 with: @@ -157,7 +157,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.7'] + python-version: ['3.11'] steps: - uses: actions/checkout@v3 with: @@ -258,7 +258,7 @@ jobs: fail-fast: false matrix: chunk: ${{ fromJson(needs.setup.outputs.chunk-list) }} - python-version: ['3.7'] + python-version: ['3.11'] services: postgres: image: postgres:11 @@ -322,7 +322,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7'] + python-version: ['3.11'] steps: - uses: actions/download-artifact@v3 with: @@ -385,7 +385,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7'] + python-version: ['3.11'] steps: - uses: actions/checkout@v3 with: From 0948ed9c218761731bfed8a92763bb95cc5b45a4 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Thu, 11 Jan 2024 17:23:11 +0100 Subject: [PATCH 131/142] print summary below the artifacts (#5711) --- .github/workflows/pr.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9957256c..115ae8a5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -358,6 +358,10 @@ jobs: if [[ "${{ needs.setup.outputs.chunk-count }}" != "$NFILES" ]]; then exit 1 fi + - run: cat upload/tool_test_output.md >> $GITHUB_STEP_SUMMARY + # the following only works for PRs opened from the main fork + # which we should not do (but lets leave this here until we have a + # solution) - name: Get PR object if: failure() uses: 8BitJonny/gh-get-current-pr@2.2.0 From 007aaefc8f8d1e80c3c73a2a676b1fb3d087a49b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 17 Jan 2024 17:17:17 +0100 Subject: [PATCH 132/142] Remove PR comment portion that doesn't work --- .github/workflows/pr.yaml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 115ae8a5..082eda42 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -359,27 +359,6 @@ jobs: exit 1 fi - run: cat upload/tool_test_output.md >> $GITHUB_STEP_SUMMARY - # the following only works for PRs opened from the main fork - # which we should not do (but lets leave this here until we have a - # solution) - - name: Get PR object - if: failure() - uses: 8BitJonny/gh-get-current-pr@2.2.0 - id: failed_pr_number - with: - sha: ${{ github.event.pull_request.head.sha }} - - name: Debug GetPR output - if: failure() - run: | - echo ${{ steps.failed_pr_number.outputs.number }} - - name: Post comment with test report - if: failure() - uses: peter-evans/create-or-update-comment@v3 - with: - token: ${{ secrets.PAT }} - issue-number: ${{ steps.failed_pr_number.outputs.number }} - body-path: upload/tool_test_output.md - # deploy the tools to the toolsheds (first TTS for testing) deploy: From a4271a659baf313846c7c942b3faa7a228ed31b6 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 17 Jan 2024 17:21:21 +0100 Subject: [PATCH 133/142] Post results before running success check --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 082eda42..3a3a6d9f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -347,6 +347,7 @@ jobs: with: name: 'All tool test results' path: upload + - run: cat upload/tool_test_output.md >> $GITHUB_STEP_SUMMARY - name: Check outputs uses: galaxyproject/planemo-ci-action@v1 id: check @@ -358,7 +359,6 @@ jobs: if [[ "${{ needs.setup.outputs.chunk-count }}" != "$NFILES" ]]; then exit 1 fi - - run: cat upload/tool_test_output.md >> $GITHUB_STEP_SUMMARY # deploy the tools to the toolsheds (first TTS for testing) deploy: From 022d08cce14ffa88fb78a3721cc8f715e1ae8312 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sun, 11 Feb 2024 11:27:08 +0100 Subject: [PATCH 134/142] bump workflow actions --- .github/workflows/pr.yaml | 62 +++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3a3a6d9f..0da33407 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -59,17 +59,17 @@ jobs: - name: Determine latest commit in the Galaxy repo id: get-galaxy-sha run: echo "galaxy-head-sha=$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" >> $GITHUB_OUTPUT - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-pip with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }} - name: Cache .planemo - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-planemo with: path: ~/.planemo @@ -80,7 +80,7 @@ jobs: run: pip install wheel - name: Install flake8 run: pip install flake8 flake8-import-order - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks @@ -113,14 +113,14 @@ jobs: matrix: python-version: ['3.11'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 1 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-pip with: path: ~/.cache/pip @@ -142,7 +142,7 @@ jobs: fail-level: ${{ env.FAIL_LEVEL }} repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: ${{ failure() }} with: name: 'Tool linting output' @@ -159,14 +159,14 @@ jobs: matrix: python-version: ['3.11'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 1 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-pip with: path: ~/.cache/pip @@ -175,7 +175,7 @@ jobs: run: pip install flake8 flake8-import-order - name: Flake8 run: echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' flake8 --output-file pylint_report.txt --tee - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: ${{ failure() }} with: name: 'Python linting output' @@ -191,14 +191,14 @@ jobs: os: [ubuntu-20.04] r-version: ['release'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 1 - uses: r-lib/actions/setup-r@v2 with: r-version: ${{ matrix.r-version }} - name: Cache R packages - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.R_LIBS_USER }} key: r_cache_${{ matrix.os }}_${{ matrix.r-version }} @@ -214,7 +214,7 @@ jobs: echo '${{ needs.setup.outputs.repository-list }}' | xargs -d '\n' -n 1 ./.github/styler.R --dry off git status git diff --exit-code | tee rlint_report.txt - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: ${{ failure() }} with: name: 'R linting output' @@ -226,7 +226,7 @@ jobs: if: ${{ github.event_name == 'pull_request' && needs.setup.outputs.repository-list != '' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Check file sizes @@ -241,7 +241,7 @@ jobs: cat file_size_report.txt exit 1 fi - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: ${{ failure() }} with: name: 'File size report' @@ -269,26 +269,26 @@ jobs: ports: - 5432:5432 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 1 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-pip with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - name: Cache .planemo - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-planemo with: path: ~/.planemo key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v1 + uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - name: Clean dotnet folder for space run: rm -Rf /usr/share/dotnet @@ -305,7 +305,7 @@ jobs: galaxy-slots: ${{ steps.cpu-cores.outputs.count }} # Limit each test to 15 minutes test_timeout: 1800 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: 'Tool test output ${{ matrix.chunk }}' path: upload @@ -324,14 +324,14 @@ jobs: matrix: python-version: ['3.11'] steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: path: artifacts - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-pip with: path: ~/.cache/pip @@ -343,7 +343,7 @@ jobs: mode: combine html-report: true markdown-report: true - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: 'All tool test results' path: upload @@ -370,14 +370,14 @@ jobs: matrix: python-version: ['3.11'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 1 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache .cache/pip - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-pip with: path: ~/.cache/pip @@ -411,7 +411,7 @@ jobs: with: sha: ${{ github.event.after }} - name: Create comment - uses: peter-evans/create-or-update-comment@v2 + uses: peter-evans/create-or-update-comment@v4 with: token: ${{ secrets.PAT }} issue-number: ${{ steps.getpr.outputs.number }} From 395f27bc06d813c2f30ae843cd5ad80c3abdfcc3 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 7 Mar 2024 21:24:32 +0100 Subject: [PATCH 135/142] bump workflows to 24.0 --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0da33407..0f5f2155 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,7 +17,7 @@ on: - '*' env: GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_23.2 + GALAXY_BRANCH: release_24.0 MAX_CHUNKS: 4 MAX_FILE_SIZE: 1M concurrency: From 5b361936a9ed001e8ea347abec2678fd82e1178e Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 8 Mar 2024 19:27:27 +0100 Subject: [PATCH 136/142] do not ignore .github xref https://github.com/galaxyproject/tools-devteam/pull/618#discussion_r1517536601 --- .github/workflows/pr.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0f5f2155..fc4e5216 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -2,7 +2,6 @@ name: Galaxy Tool Linting and Tests for push and PR on: pull_request: paths-ignore: - - '.github/**' - 'deprecated/**' - 'docs/**' - '*' @@ -11,7 +10,6 @@ on: - main - master paths-ignore: - - '.github/**' - 'deprecated/**' - 'docs/**' - '*' From a02b3931479b4ce81b1aad9013fa92e44c3f219a Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 20 Jun 2024 12:14:40 -0400 Subject: [PATCH 137/142] Bump workflows to 24.1 --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index fc4e5216..eccab2f5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ on: - '*' env: GALAXY_FORK: galaxyproject - GALAXY_BRANCH: release_24.0 + GALAXY_BRANCH: release_24.1 MAX_CHUNKS: 4 MAX_FILE_SIZE: 1M concurrency: From f5ad16816434413ceefdf356f8cbd90130fb29be Mon Sep 17 00:00:00 2001 From: heylf Date: Sun, 11 Aug 2024 22:16:06 +0200 Subject: [PATCH 138/142] ci.yaml and pr.yaml update (#6218) * update ci.yaml * update pr.yaml * forget - --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index eccab2f5..f2e13b5f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -290,6 +290,7 @@ jobs: id: cpu-cores - name: Clean dotnet folder for space run: rm -Rf /usr/share/dotnet + - run: python -m pip install pillow tifffile - name: Planemo test uses: galaxyproject/planemo-ci-action@v1 id: test From 51ddf548f4f4336c8a112d780e0c3985b6528bd0 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Wed, 21 Aug 2024 09:41:39 +0200 Subject: [PATCH 139/142] Revert "ci.yaml and pr.yaml update (#6218)" (#6228) This reverts commit 30843d2e3fd504a29c09e8b28229572810c9db25. --- .github/workflows/pr.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f2e13b5f..eccab2f5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -290,7 +290,6 @@ jobs: id: cpu-cores - name: Clean dotnet folder for space run: rm -Rf /usr/share/dotnet - - run: python -m pip install pillow tifffile - name: Planemo test uses: galaxyproject/planemo-ci-action@v1 id: test From 5df2a695474c4f113cc1ad1615e46d6df2e87d64 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sat, 27 Jul 2024 19:59:31 +0200 Subject: [PATCH 140/142] add linter exceptions for all linters that are currently failing --- .github/workflows/pr.yaml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index eccab2f5..87249ab8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -123,21 +123,12 @@ jobs: with: path: ~/.cache/pip key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-head-sha }} - - - name: Set fail level for pull request - if: ${{ github.event_name == 'pull_request' }} - run: - echo "FAIL_LEVEL=warn" >> "$GITHUB_ENV" - - name: Set fail level for merge - if: ${{ github.event_name != 'pull_request' }} - run: - echo "FAIL_LEVEL=error" >> "$GITHUB_ENV" - name: Planemo lint uses: galaxyproject/planemo-ci-action@v1 id: lint with: mode: lint - fail-level: ${{ env.FAIL_LEVEL }} + fail-level: warn repository-list: ${{ needs.setup.outputs.repository-list }} tool-list: ${{ needs.setup.outputs.tool-list }} - uses: actions/upload-artifact@v4 From 335104265c8d53dcd094acd023da7e9c0092d5de Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 15 Dec 2023 18:09:29 +0100 Subject: [PATCH 141/142] replace lintr by styler styler the R code linter/styler from the tidyverse. in contrast to lintr it can also fix the found problems (added a script that can be used for checking and fixing) --- .github/styler.R | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100755 .github/styler.R diff --git a/.github/styler.R b/.github/styler.R new file mode 100755 index 00000000..489227c9 --- /dev/null +++ b/.github/styler.R @@ -0,0 +1,36 @@ +#!/usr/bin/env Rscript + +library("argparse") +library("styler") + +parser <- ArgumentParser(description = "Call styler") +parser$add_argument("dir", + metavar = "DIR", type = "character", + help = "File to parse" +) +parser$add_argument("--dry", + choices = c("off", "on"), default = "on" +) +args <- parser$parse_args() + +file_info <- file.info(args$dir) +is_directory <- file_info$isdir + +if(is_directory){ + captured_output <- capture.output({ + result <- style_dir(args$dir, indent_by = 4, dry = args$dry, recursive = TRUE) + }) +}else{ + captured_output <- capture.output({ + result <- style_file(args$dir, indent_by = 4, dry = args$dry) + }) +} + +n <- nrow(subset(result, changed == TRUE)) +if(n > 0){ + if(args$dry == "off"){ + print(paste("Changed", n, "files")) + }else{ + stop(paste("Linting failed for", n, "files")) + } +} \ No newline at end of file From 85b9ab7788aa8adaabae2eda2e8fff6cfa663184 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 15 Dec 2023 19:38:10 +0100 Subject: [PATCH 142/142] fix styling of all R scripts --- .github/styler.R | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/styler.R b/.github/styler.R index 489227c9..c5c0cb5d 100755 --- a/.github/styler.R +++ b/.github/styler.R @@ -5,32 +5,32 @@ library("styler") parser <- ArgumentParser(description = "Call styler") parser$add_argument("dir", - metavar = "DIR", type = "character", - help = "File to parse" + metavar = "DIR", type = "character", + help = "File to parse" ) parser$add_argument("--dry", - choices = c("off", "on"), default = "on" + choices = c("off", "on"), default = "on" ) args <- parser$parse_args() file_info <- file.info(args$dir) is_directory <- file_info$isdir -if(is_directory){ +if (is_directory) { captured_output <- capture.output({ result <- style_dir(args$dir, indent_by = 4, dry = args$dry, recursive = TRUE) }) -}else{ +} else { captured_output <- capture.output({ result <- style_file(args$dir, indent_by = 4, dry = args$dry) }) } n <- nrow(subset(result, changed == TRUE)) -if(n > 0){ - if(args$dry == "off"){ +if (n > 0) { + if (args$dry == "off") { print(paste("Changed", n, "files")) - }else{ + } else { stop(paste("Linting failed for", n, "files")) } -} \ No newline at end of file +}