Adding pipeline nf-tests to nf-core/mag

.github/workflows/ci.yml

@@ -1,112 +1,124 @@
-name: nf-core CI
 # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
+name: nf-core CI
 on:
-  push:
-    branches:
-      - dev
   pull_request:
   release:
     types: [published]
+  merge_group:
+    types:
+      - checks_requested
+    branches:
+      - master
+      - dev
 
 env:
   NXF_ANSI_LOG: false
+  NFT_VER: "0.8.3"
+  NFT_WORKDIR: "~"
+  NFT_DIFF: "pdiff"
+  NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2"
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
   cancel-in-progress: true
 
 jobs:
-  test:
-    name: Run pipeline with test data
-    # Only run on push if this is the nf-core dev branch (merged PRs)
-    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/mag') }}"
+  changes:
+    name: Check for changes
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        NXF_VER:
-          - "23.04.0"
-          - "latest-everything"
+    outputs:
+      # Expose matched filters as job 'tags' output variable
+      tags: ${{ steps.filter.outputs.changes }}
     steps:
-      - name: Free some space
-        run: |
-          sudo rm -rf "/usr/local/share/boost"
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-
-      - name: Check out pipeline code
-        uses: actions/checkout@v4
-
-      - name: Install Nextflow
-        uses: nf-core/setup-nextflow@v1
+      - uses: actions/checkout@v3
+      - name: Combine all tags.yml files
+        id: get_username
+        run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml
+      - name: debug
+        run: cat .github/tags.yml
+      - uses: dorny/paths-filter@v2
+        id: filter
         with:
-          version: "${{ matrix.NXF_VER }}"
+          filters: ".github/tags.yml"
 
-      - name: Run pipeline with test data
+  define_nxf_versions:
+    name: Choose nextflow versions to test against depending on target branch
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.nxf_versions.outputs.matrix }}
+    steps:
+      - id: nxf_versions
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results
+          if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then
+            echo matrix='["latest-everything"]' | tee -a $GITHUB_OUTPUT
+          else
+            echo matrix='["latest-everything", "23.04.0"]' | tee -a $GITHUB_OUTPUT
+          fi
 
-  profiles:
-    name: Run workflow profile
-    # Only run on push if this is the nf-core dev branch (merged PRs)
-    if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/mag') }}
+  test:
+    name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }}
+    needs: [changes, define_nxf_versions]
+    if: needs.changes.outputs.tags != '[]'
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        # Run remaining test profiles with minimum nextflow version
+        NXF_VER: ${{ fromJson(needs.define_nxf_versions.outputs.matrix) }}
+        tags: ["${{ fromJson(needs.changes.outputs.tags) }}"]
         profile:
-          [
-            test_host_rm,
-            test_hybrid,
-            test_hybrid_host_rm,
-            test_busco_auto,
-            test_ancient_dna,
-            test_adapterremoval,
-            test_binrefinement,
-            test_virus_identification,
-          ]
-    steps:
-      - name: Free some space
-        run: |
-          sudo rm -rf "/usr/local/share/boost"
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          - "docker"
 
+    steps:
       - name: Check out pipeline code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install Nextflow
-        run: |
-          wget -qO- get.nextflow.io | bash
-          sudo mv nextflow /usr/local/bin/
-
-      - name: Run pipeline with ${{ matrix.profile }} test profile
-        run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.profile }},docker --outdir ./results
+        uses: nf-core/setup-nextflow@v1
+        with:
+          version: "${{ matrix.NXF_VER }}"
 
-  checkm:
-    name: Run single test to checkm due to database download
-    # Only run on push if this is the nf-core dev branch (merged PRs)
-    if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/mag') }}
-    runs-on: ubuntu-latest
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          architecture: "x64"
 
-    steps:
-      - name: Free some space
+      - name: Install pdiff to see diff between nf-test snapshots
         run: |
-          sudo rm -rf "/usr/local/share/boost"
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          python -m pip install --upgrade pip
+          pip install pdiff
 
-      - name: Check out pipeline code
-        uses: actions/checkout@v2
+      - name: Cache nf-test installation
+        id: cache-software
+        uses: actions/cache@v3
+        with:
+          path: |
+            /usr/local/bin/nf-test
+            /home/runner/.nf-test/nf-test.jar
+          key: ${{ runner.os }}-${{ env.NFT_VER }}-nftest
 
-      - name: Install Nextflow
+      - name: Install nf-test
+        if: steps.cache-software.outputs.cache-hit != 'true'
         run: |
-          wget -qO- get.nextflow.io | bash
-          sudo mv nextflow /usr/local/bin/
+          wget -qO- https://code.askimed.com/install/nf-test | bash
+          sudo mv nf-test /usr/local/bin/
 
-      - name: Download and prepare CheckM database
+      - name: Run nf-test
         run: |
-          mkdir -p databases/checkm
-          wget https://data.ace.uq.edu.au/public/CheckM_databases/checkm_data_2015_01_16.tar.gz -P databases/checkm
-          tar xzvf databases/checkm/checkm_data_2015_01_16.tar.gz -C databases/checkm/
+          nf-test test --verbose --tag ${{ matrix.tags }} --profile +"${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap
+
+      - uses: pcolby/tap-summary@v1
+        with:
+          path: >-
+            test.tap
 
-      - name: Run pipeline with ${{ matrix.profile }} test profile
+      - name: Output log on failure
+        if: failure()
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results --binqc_tool checkm --checkm_db databases/checkm
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v3
+        if: always() # always run even if the previous step fails
+        with:
+          report_paths: test.xml

.gitignore

@@ -6,3 +6,6 @@ results/
 testing/
 testing*
 *.pyc
+.nf-tests/
+.nf-test.log
+.nf-test/

.nf-core.yml

@@ -7,3 +7,4 @@ lint:
     - config_defaults:
         - params.phix_reference
         - params.lambda_reference
+  actions_ci: false

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### `Added`
 
+- [#543](https://github.com/nf-core/mag/pull/543) - Automatic samplesheet generation for nf-core/phageannotator (@CarsonJM)
+
 ### `Changed`
 
 - [#581](https://github.com/nf-core/mag/pull/581) - Added explicit licence text to headers of all custom scripts (reported by @FriederikeHanssen and @maxibor, fix by @jfy133)

README.md

@@ -38,6 +38,7 @@ The pipeline then:
 - Performs ancient DNA validation and repair with [pyDamage](https://github.com/maxibor/pydamage) and [freebayes](https://github.com/freebayes/freebayes)
 - optionally refines bins with [DAS Tool](https://github.com/cmks/DAS_Tool)
 - assigns taxonomy to bins using [GTDB-Tk](https://github.com/Ecogenomics/GTDBTk) and/or [CAT](https://github.com/dutilh/CAT) and optionally identifies viruses in assemblies using [geNomad](https://github.com/apcamargo/genomad), or Eukaryotes with [Tiara](https://github.com/ibe-uw/tiara)
+- generates a samplesheet that can be used as input for other nf-core pipelines. Currently, [phageannotator](https://github.com/nf-core/phageannotator) is supported.
 
 Furthermore, the pipeline creates various reports in the results directory specified, including a [MultiQC](https://multiqc.info/) report summarizing some of the findings and software versions.
 

bin/combine_tables.py

@@ -188,6 +188,8 @@ def main(args=None):
             how="outer",
         )
 
+    # sort results for reproducibility
+    results.sort_values(by="bin", inplace=True, ignore_index=True)
     results.to_csv(args.out, sep="\t")
 
 

bin/summary_busco.py

@@ -219,6 +219,8 @@ def main(args=None):
     else:
         df_final = df_specific.append(df_failed)
 
+    # sort output file for reproducibility
+    df_final.sort_values(by="GenomeBin", inplace=True)
     df_final.to_csv(args.out, sep="\t", index=False)
 
 

conf/base.config

@@ -122,6 +122,20 @@ process {
         memory = { check_max (128.GB * task.attempt, 'memory' ) }
         time   = { check_max (12.h   * task.attempt, 'time'   ) }
     }
+    //bowtie2 returns exit code 250 when running out of memory
+    withName: BOWTIE2_HOST_REMOVAL_ALIGN {
+        cpus          = { check_bowtie2_cpus (8, task.attempt  ) }
+        memory        = { check_max (40.GB * task.attempt, 'memory' ) }
+        time          = { check_max (16.h  * task.attempt, 'time'   ) }
+        errorStrategy = { task.exitStatus in [143,137,104,134,139,250] ? 'retry' : 'finish' }
+    }
+    //bowtie2 returns exit code 250 when running out of memory
+    withName: BOWTIE2_PHIX_REMOVAL_ALIGN {
+        cpus          = { check_bowtie2_cpus (8, task.attempt  ) }
+        memory        = { check_max (40.GB * task.attempt, 'memory' ) }
+        time          = { check_max (16.h  * task.attempt, 'time'   ) }
+        errorStrategy = { task.exitStatus in [143,137,104,134,139,250] ? 'retry' : 'finish' }
+    }
     //MEGAHIT returns exit code 250 when running out of memory
     withName: MEGAHIT {
         cpus          = { check_megahit_cpus (8, task.attempt  ) }
@@ -147,7 +161,7 @@ process {
     }
     //returns exit code 247 when running out of memory
     withName: BOWTIE2_ASSEMBLY_ALIGN {
-        cpus          = { check_max (2    * task.attempt, 'cpus'   ) }
+        cpus          = { check_bowtie2_cpus (8, 'cpus'   ) }
         memory        = { check_max (8.GB * task.attempt, 'memory' ) }
         time          = { check_max (8.h  * task.attempt, 'time'   ) }
         errorStrategy = { task.exitStatus in [143,137,104,134,139,247] ? 'retry' : 'finish' }

conf/modules.config

@@ -115,7 +115,9 @@ process {
     }
 
     withName: BOWTIE2_HOST_REMOVAL_ALIGN {
-        ext.args = params.host_removal_verysensitive ? "--very-sensitive" : "--sensitive"
+        ext.args = [
+            params.host_removal_verysensitive ? "--very-sensitive" : "--sensitive"
+        ].join(' ').trim()
         ext.args2 = params.host_removal_save_ids ? "--host_removal_save_ids" : ''
         ext.prefix = { "${meta.id}_run${meta.run}_host_removed" }
         publishDir = [

conf/test.config

@@ -19,16 +19,19 @@ params {
     max_memory = '6.GB'
     max_time   = '6.h'
 
-    // Input data
     input                         = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv'
     centrifuge_db                 = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz"
     kraken2_db                    = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz"
     skip_krona                    = true
-    min_length_unbinned_contigs   = 1
+    megahit_fix_cpu_1             = true
+    spades_fix_cpus               = 1
+    bowtie2_fix_cpu_1             = true
+    maxbin2_fix_cpu_1             = true
+    binning_map_mode              = 'own'
+    min_length_unbinned_contigs   = 1000000
     max_unbinned_contigs          = 2
     busco_db                      = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz"
     busco_clean                   = true
     skip_gtdbtk                   = true
-    gtdbtk_min_completeness       = 0
     skip_concoct                  = true
 }

conf/test_adapterremoval.config

@@ -21,16 +21,19 @@ params {
 
     // Input data
     input                       = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.euk.csv'
-    centrifuge_db               = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz"
-    kraken2_db                  = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz"
-    metaeuk_db                  = "https://github.com/nf-core/test-datasets/raw/modules/data/proteomics/database/yeast_UPS.fasta"
+    clip_tool                   = 'adapterremoval'
+    keep_phix                   = true
+    centrifuge_db               = null
+    kraken2_db                  = null
     skip_krona                  = true
-    min_length_unbinned_contigs = 1
-    max_unbinned_contigs        = 2
-    busco_db                    = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz"
+    skip_megahit                = true
+    skip_spades                 = true
+    skip_spadeshybrid           = true
+    skip_quast                  = true
+    skip_prodigal               = true
+    skip_binning                = true
+    skip_binqc                  = true
     skip_gtdbtk                 = true
-    gtdbtk_min_completeness     = 0
-    clip_tool                   = 'adapterremoval'
-    skip_concoct                = true
-    bin_domain_classification   = true
+    skip_prokka                 = true
+    skip_metaeuk                = true
 }

conf/test_ancient_dna.config

@@ -20,23 +20,30 @@ params {
     max_time   = '6.h'
 
     // Input data
-    input                                = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv'
-    centrifuge_db                        = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz"
-    kraken2_db                           = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz"
-    skip_krona                           = true
-    min_length_unbinned_contigs          = 1
-    max_unbinned_contigs                 = 2
-    busco_db                             = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz"
-    skip_gtdbtk                          = true
-    gtdbtk_min_completeness              = 0
-    ancient_dna                          = true
-    binning_map_mode                     = 'own'
-    skip_spades                          = false
-    skip_spadeshybrid                    = true
-    bcftools_view_high_variant_quality   = 0
-    bcftools_view_medium_variant_quality = 0
-    bcftools_view_minimal_allelesupport  = 3
-    refine_bins_dastool                  = true
-    refine_bins_dastool_threshold        = 0
-    skip_concoct                         = true
+    input                                   = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv'
+    skip_clipping                           = true
+    keep_phix                               = true
+    kraken2_db                              = null
+    centrifuge_db                           = null
+    skip_krona                              = true
+    megahit_fix_cpu_1                       = true
+    spades_fix_cpus                         = 1
+    skip_spadeshybrid                       = true
+    ancient_dna                             = true
+    skip_quast                              = true
+    skip_prodigal                           = true
+    bowtie2_fix_cpu_1                       = true
+    binning_map_mode                        = 'own'
+    maxbin2_fix_cpu_1                       = true
+    bcftools_view_high_variant_quality      = 0
+    bcftools_view_medium_variant_quality    = 0
+    bcftools_view_minimal_allelesupport     = 3
+    refine_bins_dastool                     = true
+    refine_bins_dastool_threshold           = 0
+    min_length_unbinned_contigs             = 1
+    max_unbinned_contigs                    = 2
+    skip_binqc                              = true
+    skip_gtdbtk                             = true
+    skip_prokka                             = true
+    skip_metaeuk                            = true
 }