Add reusable workflows

.flake8_requirements

@@ -0,0 +1,2 @@
+flake8
+flake8-import-order

.github/workflows/tools.yaml

@@ -19,7 +19,7 @@ jobs:
       chunk-list: ${{ steps.discover.outputs.chunk-list }}
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: ['3.7']
     steps:
     - name: Print github context properties
       run: |
@@ -85,198 +85,90 @@ jobs:
 
   setup-ci-tools:
     name: Setup as in CI for tools
+    if: ${{ github.repository_owner == 'galaxyproject' }}
+    uses: ./.github/workflows/wf_setup.yaml
+    with:
+      # default-galaxy-fork: galaxyproject
+      # default-galaxy-branch: release_22.05
+      max-chunks: 40
+      github-event-name-override: 'schedule'
+    secrets:
+      PAT: ${{ secrets.PAT }}
+
+  check-setup-ci-tools:
+    name: Check setup
+    needs: setup-ci-tools
     runs-on: ubuntu-latest
-    outputs:
-      galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
-      fork: ${{ steps.get-fork-branch.outputs.fork }}
-      branch: ${{ steps.get-fork-branch.outputs.branch }}
-      repository-list: ${{ steps.discover.outputs.repository-list }}
-      chunk-count: ${{ steps.discover.outputs.chunk-count }}
-      chunk-list: ${{ steps.discover.outputs.chunk-list }}
-    strategy:
-      matrix:
-        python-version: [3.7]
     steps:
-    - name: Determine latest commit in the Galaxy repo
-      id: get-galaxy-sha
-      run: echo "galaxy-head-sha=$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" >> $GITHUB_OUTPUT
-    - uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Cache .cache/pip
-      uses: actions/cache@v3
-      id: cache-pip
-      with:
-        path: ~/.cache/pip
-        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
-    # Install the `wheel` package so that when installing other packages which
-    # are not available as wheels, pip will build a wheel for them, which can be cached.
-    - name: Install wheel
-      run: pip install wheel
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 1
-    - name: Artificially exclude workflows (because ci_find_repos discovers tool and workflow repos)
-      run: echo 'test/workflows/' > .tt_skip
-    - name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks
-      uses: ./
-      id: discover
-      env:
-        GITHUB_EVENT_NAME: schedule
-      with:
-        create-cache: true
-        galaxy-fork: ${{ env.GALAXY_FORK }}
-        galaxy-branch: ${{ env.GALAXY_BRANCH }}
-        max-chunks: ${{ env.MAX_CHUNKS }}
-        python-version: ${{ matrix.python-version }}
-        github-event-name-override: 'schedule'
     - name: Check for empty commit range
-      run: if [ -n "${{ steps.discover.outputs.commit-range }}" ]; then exit 1; fi
+      run: if [ -n "${{ needs.setup-ci-tools.outputs.commit-range }}" ]; then exit 1; fi
     # note: ci_find_repos will find all tools and workflows
     - name: Check that all tools an workflows are in the repository list
       run: |
-        if ! grep -q "tool1" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "tool1 must be in the repo list"; exit 1; fi
-        if ! grep -q "tool2" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "tool2 must be in the repo list"; exit 1; fi
-        if grep -q "example3" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "example3 must not be in the repo list"; exit 1; fi
-        if grep -q "example4" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "example4 must not be in the repo list"; exit 1; fi
+        if ! grep -q "tool1" <<<$(echo "${{ needs.setup-ci-tools.outputs.repository-list }}"); then echo "tool1 must be in the repo list"; exit 1; fi
+        if ! grep -q "tool2" <<<$(echo "${{ needs.setup-ci-tools.outputs.repository-list }}"); then echo "tool2 must be in the repo list"; exit 1; fi
+        if grep -q "example3" <<<$(echo "${{ needs.setup-ci-tools.outputs.repository-list }}"); then echo "example3 must not be in the repo list"; exit 1; fi
+        if grep -q "example4" <<<$(echo "${{ needs.setup-ci-tools.outputs.repository-list }}"); then echo "example4 must not be in the repo list"; exit 1; fi
     # note: ci_find_tools will find onlytools
     - name: Check that all tools are in the tool list
       run: |
-        if ! grep -q "tool1.xml" <<<$(echo "${{ steps.discover.outputs.tool-list }}"); then echo "tool1.xml must be in the repo list"; exit 1; fi
-        if ! grep -q "tool2.xml" <<<$(echo "${{ steps.discover.outputs.tool-list }}"); then echo "tool2.xml must be in the repo list"; exit 1; fi
+        if ! grep -q "tool1.xml" <<<$(echo "${{ needs.setup-ci-tools.outputs.tool-list }}"); then echo "tool1.xml must be in the repo list"; exit 1; fi
+        if ! grep -q "tool2.xml" <<<$(echo "${{ needs.setup-ci-tools.outputs.tool-list }}"); then echo "tool2.xml must be in the repo list"; exit 1; fi
     # chunk-count will only consider tools/workflows depending on the workflow input to the action
     - name: Check that the number of chunks is three, i.e. the number of tools
-      run: if [ "${{ steps.discover.outputs.chunk-count }}" != "3" ]; then exit 1; fi
+      run: if [ "${{ needs.setup-ci-tools.outputs.chunk-count }}" != "3" ]; then exit 1; fi
 
   lint:
-    name: Test linting of tools
-    needs: [setup-ci-tools]
+    name: Lint
+    needs: setup-ci-tools
+    uses: ./.github/workflows/wf_lint.yaml
+    with:
+      repository-list: ${{ needs.setup-ci-tools.outputs.repository-list }}
+      tool-list: ${{ needs.setup-ci-tools.outputs.tool-list }}
+      galaxy-head-sha: ${{ needs.setup-ci-tools.outputs.galaxy-head-sha }}
+      commit-range: ${{ needs.setup-ci-tools.outputs.commit-range }}
+      planemo-version: ${{ needs.setup-ci-tools.outputs.planemo-version }}
+      fail-level: error
+
+  check-lint:
+    name: Check linting results
+    needs: lint
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [3.7]
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 1
-    - uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Cache .cache/pip
-      uses: actions/cache@v3
-      id: cache-pip
-      with:
-        path: ~/.cache/pip
-        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup-ci-tools.outputs.galaxy-head-sha }}
-    - name: Install wheel
-      run: pip install wheel
-    - name: Planemo lint tools
-      uses: ./
+    steps:  
+    - uses: actions/download-artifact@v3
       with:
-        mode: lint
-        report-level: all
-        fail-level: error
-        repository-list: ${{ needs.setup-ci-tools.outputs.repository-list }}
-        tool-list: ${{ needs.setup-ci-tools.outputs.tool-list }}
-        additional-planemo-options: --report_level all
-      continue-on-error: true
+        path: lint_report.txt
+    # TODO check result of lint job (ie make sure it failed)
     - name: check if all test tools were linted
       run: |
         grep tool1 lint_report.txt
         grep tool2 lint_report.txt
         grep "ERROR: Error 'HTTPConnectionPool" lint_report.txt
         grep "Applying linter tests... CHECK" lint_report.txt
+
   test-tools:
-    name: Test testing of tools
-    needs: [setup-ci-tools]
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        chunk: ${{ fromJson(needs.setup-ci-tools.outputs.chunk-list) }}
-        python-version: [3.7]
-    services:
-      postgres:
-        image: postgres:11
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: postgres
-        ports:
-          - 5432:5432
-    steps:
-    # checkout the repository
-    # and use it as the current working directory
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 1
-    - uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Cache .cache/pip
-      uses: actions/cache@v3
-      id: cache-pip
-      with:
-        path: ~/.cache/pip
-        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup-ci-tools.outputs.galaxy-head-sha }}
-    - name: Get number of CPU cores
-      uses: SimenB/github-actions-cpu-cores@v1
-      id: cpu-cores
-    - name: Planemo test tools
-      uses: ./
-      id: test-tools
-      with:
-        mode: test
-        repository-list: ${{ needs.setup-ci-tools.outputs.repository-list }}
-        galaxy-fork: ${{ env.GALAXY_FORK }}
-        galaxy-branch: ${{ env.GALAXY_BRANCH }}
-        chunk: ${{ matrix.chunk }}
-        chunk-count: ${{ needs.setup-ci-tools.outputs.chunk-count }}
-        additional-planemo-options: --simultaneous_uploads --check_uploads_ok
-        galaxy-slots: ${{ steps.cpu-cores.outputs.count }}
-    - uses: actions/upload-artifact@v3
-      with:
-        name: 'Tool test output ${{ matrix.chunk }}'
-        path: upload
+    name: Test tools
+    needs: setup-ci-tools
+    uses: ./.github/workflows/wf_test.yaml
+    with:
+      galaxy-fork: ${{ needs.setup-ci-tools.outputs.galaxy-fork }}
+      galaxy-branch: ${{ needs.setup-ci-tools.outputs.galaxy-branch }}
+      planemo-version: ${{ needs.setup-ci-tools.outputs.planemo-version }}
+      repository-list: ${{ needs.setup-ci-tools.outputs.repository-list }}
+      chunk-count: ${{ needs.setup-ci-tools.outputs.chunk-count }}
+      chunk-list: ${{ needs.setup-ci-tools.outputs.chunk-list }}
+      galaxy-head-sha: ${{ needs.setup-ci-tools.outputs.galaxy-head-sha }}
+      # set these values if you want to use different values than the IUC defaults
+      # test-timeout: 900
 
-  combine_outputs:
-    name: Test combining 'chunked' test results
+  check-test:
+    name: Check testing results
     needs: [setup-ci-tools, test-tools]
-    strategy:
-      matrix:
-        python-version: [3.7]
-    # This job runs on Linux
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 1
-    - uses: actions/download-artifact@v3
-      with:
-        path: artifacts
-    - uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Cache .cache/pip
-      uses: actions/cache@v3
-      id: cache-pip
-      with:
-        path: ~/.cache/pip
-        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup-ci-tools.outputs.galaxy-head-sha }}
-    - name: Combine outputs
-      uses: ./
-      id: combine
-      with:
-        mode: combine
-        html-report: true
     - name: Check statistics
-      run: if ! grep -q "3\s\+success" <<<$(echo ${{ steps.combine.outputs.statistics }}); then echo "wrong statistics"; exit 1; fi
+      run: if ! grep -q "3\s\+success" <<<$(echo ${{ needs.test-tools.outputs.statistics }}); then echo "wrong statistics"; exit 1; fi
     - uses: actions/upload-artifact@v3
       with:
         name: 'All tool test results'
         path: upload
-    - name: Check outputs
-      uses: ./
-      id: check
-      with:
-        mode: check

.github/workflows/wf_deploy.yaml

@@ -0,0 +1,89 @@
+name: Deploy Tools
+on:
+  workflow_call:
+    inputs:
+      galaxy-head-sha:
+        description: 'hash of the latest commit in the Galaxy repo'
+        required: true
+        type: string
+      repository-list:
+        description: 'list of repositories to deploy'
+        required: true
+        type: string
+      python-version-list:
+        description: 'Python versions (stringified JSON array)'
+        default: "[\"3.7\"]"
+        required: false
+        type: string
+      planemo-version:
+        description: 'Planemo version to use'
+        default: "planemo"
+        required: false
+        type: string
+    secrets:
+      TTS_API_KEY:
+        required: true
+      TS_API_KEY:
+        required: true
+
+jobs:
+  # deploy the tools to the toolsheds (first TTS for testing)
+  deploy:
+    name: Deploy
+    if: ${{ (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ${{ fromJson(inputs.python-version-list) }}
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 1
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Cache .cache/pip
+      uses: actions/cache@v3
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ inputs.galaxy-head-sha }}_planemo_${{ steps.get-galaxy-sha.outputs.planemo-version }}
+    - name: Deploy on testtoolshed
+      uses: ./
+      with:
+        mode: deploy
+        repository-list: ${{ inputs.repository-list }}
+        shed-target: testtoolshed
+        shed-key: ${{ secrets.TTS_API_KEY }}
+        planemo-version: ${{ inputs.planemo-version }}
+      continue-on-error: true
+    - name: Deploy on toolshed
+      uses: ./
+      with:
+        mode: deploy
+        repository-list: ${{ inputs.repository-list }}
+        shed-target: toolshed
+        shed-key: ${{ secrets.TS_API_KEY }}
+        planemo-version: ${{ inputs.planemo-version }}
+
+  deploy-report:
+    name: Report deploy status
+    needs: [deploy]
+    if: ${{ always() && needs.deploy.result != 'success' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) }}
+    runs-on: ubuntu-latest
+    steps:
+    # report to the PR if deployment failed
+    - name: Get PR object
+      uses: 8BitJonny/[email protected]
+      id: getpr
+      with:
+        sha: ${{ github.event.after }}
+    - name: Create comment
+      uses: peter-evans/create-or-update-comment@v2
+      with:
+        token: ${{ secrets.PAT }}
+        issue-number: ${{ steps.getpr.outputs.number }}
+        body: |
+          Attention: deployment ${{ needs.deploy.result }}!
+
+          https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}