Merge branch 'master' of github.com:nf-core/modules into fresh-ci #18277
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run pytest-workflow | ||
| on: | ||
| push: | ||
| branches: | ||
| # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging | ||
| - "renovate/**" # branches Renovate creates | ||
| pull_request: | ||
| branches: [master] | ||
| merge_group: | ||
| types: [checks_requested] | ||
| branches: [master] | ||
| workflow_dispatch: | ||
| inputs: | ||
| runners: | ||
| description: "Runners to test on" | ||
| type: choice | ||
| options: | ||
| - "ubuntu-latest" | ||
| - "self-hosted" | ||
| default: "self-hosted" | ||
| # Cancel if a newer run is started | ||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | ||
| cancel-in-progress: true | ||
| env: | ||
| NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity | ||
| NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity | ||
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| jobs: | ||
| pytest-changes: | ||
| name: pytest-changes | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| tags: ${{ steps.filter.outputs.changes }} | ||
| modules: ${{ steps.tags.outputs.modules }} | ||
| subworkflows: ${{ steps.tags.outputs.subworkflows }} | ||
| steps: | ||
| - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 | ||
| with: | ||
| fetch-depth: 2 # To retrieve the preceding commit. | ||
| # TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented | ||
| - uses: mirpedrol/paths-filter@main | ||
| id: filter | ||
| with: | ||
| filters: "tests/config/pytest_modules.yml" | ||
| token: "" | ||
| - name: Fetch module tags | ||
| id: tags | ||
| run: | | ||
| echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT | ||
| echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT | ||
| - name: debug | ||
| run: | | ||
| echo ${{ steps.tags.outputs.modules }} | ||
| echo ${{ steps.tags.outputs.subworkflows }} | ||
| pytest: | ||
| runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} | ||
| name: pytest | ||
| needs: [pytest-changes] | ||
| if: needs.pytest-changes.outputs.tags != '[]' | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"] | ||
| profile: [conda, docker, singularity] | ||
| exclude: | ||
| exclude: | ||
| - tags: nf-test | ||
| - profile: conda | ||
| tags: backsub | ||
| - profile: conda | ||
| tags: bases2fastq | ||
| - profile: singularity | ||
| tags: bases2fastq | ||
| - profile: conda | ||
| tags: basicpy | ||
| - profile: conda | ||
| tags: bcl2fastq | ||
| - profile: conda | ||
| tags: bclconvert | ||
| - profile: conda | ||
| tags: bwa/aln | ||
| - profile: conda | ||
| tags: bwa/index | ||
| - profile: conda | ||
| tags: bwa/mem | ||
| - profile: conda | ||
| tags: bwa/sampe | ||
| - profile: conda | ||
| tags: bwa/samse | ||
| - profile: conda | ||
| tags: cellpose | ||
| - profile: conda | ||
| tags: cellrangerarc/count | ||
| - profile: conda | ||
| tags: cellrangerarc/mkfastq | ||
| - profile: conda | ||
| tags: cellrangerarc/mkref | ||
| - profile: conda | ||
| tags: cellrangeratac/count | ||
| - profile: conda | ||
| tags: cellrangeratac/mkfastq | ||
| - profile: conda | ||
| tags: cellrangeratac/mkref | ||
| - profile: conda | ||
| tags: checkm2/databasedownload | ||
| - profile: conda | ||
| tags: checkm2/predict | ||
| - profile: conda | ||
| tags: controlfreec/makegraph2 | ||
| - profile: conda | ||
| tags: coreograph | ||
| - profile: conda | ||
| tags: deepcell/mesmer | ||
| - profile: conda | ||
| tags: deepvariant | ||
| - profile: conda | ||
| tags: fastani | ||
| - profile: conda | ||
| tags: fastk/fastk | ||
| - profile: conda | ||
| tags: fastk/histex | ||
| - profile: conda | ||
| tags: fastk/merge | ||
| - profile: conda | ||
| tags: fcs/fcsadaptor | ||
| - profile: conda | ||
| tags: fcs/fcsgx | ||
| - profile: conda | ||
| tags: gatk4/cnnscorevariants | ||
| - profile: conda | ||
| tags: gatk4/determinegermlinecontigploidy | ||
| - profile: singularity | ||
| tags: gatk4/determinegermlinecontigploidy | ||
| - profile: conda | ||
| tags: gatk4/germlinecnvcaller | ||
| - profile: conda | ||
| tags: gatk4/postprocessgermlinecnvcalls | ||
| - profile: conda | ||
| tags: genescopefk | ||
| - profile: conda | ||
| tags: happy/sompy | ||
| - profile: conda | ||
| tags: hlala/preparegraph | ||
| - profile: conda | ||
| tags: ilastik/multicut | ||
| - profile: conda | ||
| tags: ilastik/pixelclassification | ||
| - profile: conda | ||
| tags: imputeme/vcftoprs | ||
| - profile: conda | ||
| tags: islandpath | ||
| - profile: conda | ||
| tags: manta/convertinversion | ||
| - profile: conda | ||
| tags: mcstaging/imc2mc | ||
| - profile: conda | ||
| tags: mcquant | ||
| - profile: conda | ||
| tags: medaka | ||
| - profile: conda | ||
| tags: merquryfk/katcomp | ||
| - profile: conda | ||
| tags: merquryfk/katgc | ||
| - profile: conda | ||
| tags: merquryfk/merquryfk | ||
| - profile: conda | ||
| tags: merquryfk/ploidyplot | ||
| - profile: conda | ||
| tags: minimap2/align | ||
| - profile: conda | ||
| tags: mitohifi/findmitoreference | ||
| - profile: conda | ||
| tags: mitohifi/mitohifi | ||
| - profile: conda | ||
| tags: nanoplot | ||
| - profile: conda | ||
| tags: ncbitools/vecscreen | ||
| - profile: conda | ||
| tags: parabricks/applybqsr | ||
| - profile: conda | ||
| tags: parabricks/dbsnp | ||
| - profile: conda | ||
| tags: parabricks/deepvariant | ||
| - profile: conda | ||
| tags: parabricks/fq2bam | ||
| - profile: conda | ||
| tags: parabricks/genotypegvcf | ||
| - profile: conda | ||
| tags: parabricks/haplotypecaller | ||
| - profile: conda | ||
| tags: parabricks/indexgvcf | ||
| - profile: conda | ||
| tags: parabricks/mutectcaller | ||
| - profile: conda | ||
| tags: picard/collecthsmetrics | ||
| - profile: conda | ||
| tags: picard/collectwgsmetrics | ||
| - profile: conda | ||
| tags: sentieon/applyvarcal | ||
| - profile: conda | ||
| tags: sentieon/datametrics | ||
| - profile: conda | ||
| tags: sentieon/dnamodelapply | ||
| - profile: conda | ||
| tags: sentieon/dnascope | ||
| - profile: conda | ||
| tags: sentieon/readwriter | ||
| - profile: conda | ||
| tags: sentieon/tnfilter | ||
| - profile: conda | ||
| tags: sentieon/tnhaplotyper2 | ||
| - profile: conda | ||
| tags: sentieon/tnscope | ||
| - profile: conda | ||
| tags: sentieon/varcal | ||
| - profile: conda | ||
| tags: sentieon/wgsmetrics | ||
| - profile: conda | ||
| tags: subworkflows/bam_qc_picard | ||
| - profile: conda | ||
| tags: subworkflows/bcl_demultiplex | ||
| - profile: conda | ||
| tags: subworkflows/fasta_clean_fcs | ||
| - profile: conda | ||
| tags: svanalyzer/svbenchmark | ||
| - profile: conda | ||
| tags: universc | ||
| - profile: singularity | ||
| tags: universc | ||
| - profile: conda | ||
| tags: vt/decompose | ||
| env: | ||
| NXF_ANSI_LOG: false | ||
| steps: | ||
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 | ||
| - name: Set up Python | ||
| uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 | ||
| with: | ||
| python-version: "3.11" | ||
| - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 | ||
| id: cache-pip-pytest | ||
| with: | ||
| path: ~/.cache/pip | ||
| key: ${{ runner.os }}-pip-pytest | ||
| restore-keys: | | ||
| ${{ runner.os }}-pip-pytest | ||
| - name: Install Python dependencies | ||
| run: python -m pip install --upgrade pip pytest-workflow cryptography | ||
| - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 | ||
| with: | ||
| distribution: "temurin" | ||
| java-version: "17" | ||
| - name: Setup Nextflow ${{ matrix.NXF_VER }} | ||
| uses: nf-core/setup-nextflow@v2 | ||
| with: | ||
| version: "${{ matrix.NXF_VER }}" | ||
| - name: Setup apptainer | ||
| if: matrix.profile == 'singularity' | ||
| uses: eWaterCycle/setup-apptainer@main | ||
| - name: Set up Singularity | ||
| if: matrix.profile == 'singularity' | ||
| run: | | ||
| mkdir -p $NXF_SINGULARITY_CACHEDIR | ||
| mkdir -p $NXF_SINGULARITY_LIBRARYDIR | ||
| - name: Set up miniconda | ||
| uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3 | ||
| with: | ||
| miniconda-version: "latest" | ||
| channels: conda-forge,bioconda | ||
| python-version: ${{ matrix.python-version }} | ||
| - name: Conda setup | ||
| run: | | ||
| conda clean -a | ||
| conda install -n base conda-libmamba-solver | ||
| conda config --set solver libmamba | ||
| echo $(realpath $CONDA)/condabin >> $GITHUB_PATH | ||
| echo $(realpath python) >> $GITHUB_PATH | ||
| # Test the module | ||
| - name: Run pytest-workflow | ||
| # only use one thread for pytest-workflow to avoid race condition on conda cache. | ||
| run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes | ||
| - name: Output log on failure | ||
| if: failure() | ||
| run: | | ||
| sudo apt-get update > /dev/null | ||
| sudo apt-get install bat > /dev/null | ||
| batcat --decorations=always --color=always /home/ubuntu/pytest_workflow_*/*/log.{out,err} | ||
| - name: Setting global variables | ||
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 | ||
| id: parsed | ||
| with: | ||
| script: | | ||
| return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"') | ||
| result-encoding: string | ||
| - name: Upload logs on failure | ||
| if: failure() | ||
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4 | ||
| with: | ||
| name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} | ||
| path: | | ||
| /home/ubuntu/pytest_workflow_*/*/.nextflow.log | ||
| /home/ubuntu/pytest_workflow_*/*/log.out | ||
| /home/ubuntu/pytest_workflow_*/*/log.err | ||
| /home/ubuntu/pytest_workflow_*/*/work | ||
| !/home/ubuntu/pytest_workflow_*/*/work/conda | ||
| !/home/ubuntu/pytest_workflow_*/*/work/singularity | ||
| !${{ github.workspace }}/.singularity | ||
| confirm-pass: | ||
| runs-on: ubuntu-latest | ||
| needs: [pytest-changes, pytest] | ||
| if: always() | ||
| steps: | ||
| - name: All tests ok | ||
| if: ${{ success() || !contains(needs.*.result, 'failure') }} | ||
| run: exit 0 | ||
| - name: One or more tests failed | ||
| if: ${{ contains(needs.*.result, 'failure') }} | ||
| run: exit 1 | ||
| - name: debug-print | ||
| if: always() | ||
| run: | | ||
| echo "toJSON(needs) = ${{ toJSON(needs) }}" | ||
| echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" | ||
