diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4dde8d56..72c2b4ae 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -18,7 +18,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/fetc - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/fetchngs/tree/master/.github/CONTRIBUTING.md) - [ ] If necessary, also make a PR on the nf-core/fetchngs _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). -- [ ] Ensure the test suite passes (`nf-test test main.nf.test -profile test,docker`). +- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 073e1876..1fcafe88 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -14,13 +14,12 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 - cache: "pip" + python-version: "3.12" - name: Install pre-commit run: pip install pre-commit @@ -32,14 +31,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: "3.11" + python-version: "3.12" architecture: "x64" - name: Install dependencies @@ -60,7 +59,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: linting-logs path: | diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index b706875f..40acc23f 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@f6b0bace624032e30a85a8fd9c1a7f8f611f5737 # v3 + uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3 with: workflow: linting.yml workflow_conclusion: completed diff --git a/CHANGELOG.md b/CHANGELOG.md index dbfb81be..234bb84d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [PR #300](https://github.com/nf-core/fetchngs/pull/300) - Use file paths instead of tags for testing matrix, should make matrices more efficient - [PR #303](https://github.com/nf-core/fetchngs/pull/303) - Update wget container for SRA_FASTQ_FTP from 1.20.1 to 1.21.4 - [PR #305](https://github.com/nf-core/fetchngs/pull/305) - Update module sratools/prefetch for reliable download integrity check +- [PR #319](https://github.com/nf-core/fetchngs/pull/319) - Update subworkflows and modules to be TES compliance. + +### :warning: Major enhancements + +- All the subworkflows and modules have been updated to newest main branch of `nf-core/modules` +- Add support for tags +- Updated `multiqc_mappings_config` to have different matching pattern for `config.yml` vs `versions.yml` ### Software dependencies diff --git a/modules.json b/modules.json index c6b234bb..b37bf4e5 100644 --- a/modules.json +++ b/modules.json @@ -7,23 +7,23 @@ "nf-core": { "custom/sratoolsncbisettings": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "06c8865e36741e05ad32ef70ab3fac127486af48", "installed_by": ["fastq_download_prefetch_fasterqdump_sratools"] }, "sratools/fasterqdump": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "06c8865e36741e05ad32ef70ab3fac127486af48", "installed_by": ["fastq_download_prefetch_fasterqdump_sratools"], "patch": "modules/nf-core/sratools/fasterqdump/sratools-fasterqdump.diff" }, "sratools/prefetch": { "branch": "master", - "git_sha": "1fc29f92e439d5631fdf34b8ac4687297d70f5ec", + "git_sha": "368e6c90b91adbd171e7c0a1c85a700b86a915af", "installed_by": ["fastq_download_prefetch_fasterqdump_sratools"] }, "untar": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "06c8865e36741e05ad32ef70ab3fac127486af48", "installed_by": ["modules"] } } @@ -37,12 +37,12 @@ }, "utils_nextflow_pipeline": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "d20fb2a9cc3e2835e9d067d1046a63252eb17352", "installed_by": ["subworkflows"] }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "2fdce49d30c0254f76bc0f13c55c17455c1251ab", "installed_by": ["subworkflows"] }, "utils_nfvalidation_plugin": { diff --git a/modules/local/multiqc_mappings_config/main.nf b/modules/local/multiqc_mappings_config/main.nf index 8efe1caa..5d809e52 100644 --- a/modules/local/multiqc_mappings_config/main.nf +++ b/modules/local/multiqc_mappings_config/main.nf @@ -10,7 +10,7 @@ process MULTIQC_MAPPINGS_CONFIG { path csv output: - path "*yml" , emit: yml + path "*config.yml" , emit: yml path "versions.yml", emit: versions script: diff --git a/modules/nf-core/custom/sratoolsncbisettings/environment.yml b/modules/nf-core/custom/sratoolsncbisettings/environment.yml index 44a1b008..cd4c3fbe 100644 --- a/modules/nf-core/custom/sratoolsncbisettings/environment.yml +++ b/modules/nf-core/custom/sratoolsncbisettings/environment.yml @@ -1,7 +1,5 @@ -name: custom_sratoolsncbisettings channels: - conda-forge - bioconda - - defaults dependencies: - bioconda::sra-tools=3.0.8 diff --git a/modules/nf-core/custom/sratoolsncbisettings/tests/main.nf.test b/modules/nf-core/custom/sratoolsncbisettings/tests/main.nf.test index e9ea68dc..b8b95320 100644 --- a/modules/nf-core/custom/sratoolsncbisettings/tests/main.nf.test +++ b/modules/nf-core/custom/sratoolsncbisettings/tests/main.nf.test @@ -4,6 +4,10 @@ nextflow_process { script "../main.nf" process "CUSTOM_SRATOOLSNCBISETTINGS" config "modules/nf-core/custom/sratoolsncbisettings/tests/nextflow.config" + tag "modules" + tag "modules_nfcore" + tag "custom" + tag "custom/sratoolsncbisettings" test("Should run without failures") { diff --git a/modules/nf-core/custom/sratoolsncbisettings/tests/tags.yml b/modules/nf-core/custom/sratoolsncbisettings/tests/tags.yml new file mode 100644 index 00000000..fb4a08a7 --- /dev/null +++ b/modules/nf-core/custom/sratoolsncbisettings/tests/tags.yml @@ -0,0 +1,2 @@ +custom/sratoolsncbisettings: + - modules/nf-core/custom/sratoolsncbisettings/** diff --git a/modules/nf-core/sratools/fasterqdump/environment.yml b/modules/nf-core/sratools/fasterqdump/environment.yml index dd0faa56..970dfbc0 100644 --- a/modules/nf-core/sratools/fasterqdump/environment.yml +++ b/modules/nf-core/sratools/fasterqdump/environment.yml @@ -1,8 +1,6 @@ -name: sratools_fasterqdump channels: - conda-forge - bioconda - - defaults dependencies: - - bioconda::sra-tools=2.11.0 + - bioconda::sra-tools=3.0.8 - conda-forge::pigz=2.6 diff --git a/modules/nf-core/sratools/fasterqdump/main.nf b/modules/nf-core/sratools/fasterqdump/main.nf index e7cf157a..d5b65bab 100644 --- a/modules/nf-core/sratools/fasterqdump/main.nf +++ b/modules/nf-core/sratools/fasterqdump/main.nf @@ -4,8 +4,8 @@ process SRATOOLS_FASTERQDUMP { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-5f89fe0cd045cb1d615630b9261a1d17943a9b6a:6a9ff0e76ec016c3d0d27e0c0d362339f2d787e6-0' : - 'quay.io/biocontainers/mulled-v2-5f89fe0cd045cb1d615630b9261a1d17943a9b6a:6a9ff0e76ec016c3d0d27e0c0d362339f2d787e6-0' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-5f89fe0cd045cb1d615630b9261a1d17943a9b6a:2f4a4c900edd6801ff0068c2b3048b4459d119eb-0' : + 'biocontainers/mulled-v2-5f89fe0cd045cb1d615630b9261a1d17943a9b6a:2f4a4c900edd6801ff0068c2b3048b4459d119eb-0' }" input: tuple val(meta), path(sra) @@ -23,7 +23,10 @@ process SRATOOLS_FASTERQDUMP { def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def outfile = meta.single_end ? "${prefix}.fastq" : prefix + def outfile = "${prefix}.fastq" + def exclude_third = meta.single_end ? '' : "mv $outfile $prefix || echo 'No third file'" + // Excludes the "${prefix}.fastq" file from output `reads` channel for paired end cases and + // avoids the '.' in the path bug: https://github.com/ncbi/sra-tools/issues/865 def key_file = '' if (certificate.toString().endsWith('.jwt')) { key_file += " --perm ${certificate}" @@ -40,6 +43,51 @@ process SRATOOLS_FASTERQDUMP { ${key_file} \\ ${sra} + $exclude_third + + pigz \\ + $args2 \\ + --no-name \\ + --processes $task.cpus \\ + *.fastq + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + sratools: \$(fasterq-dump --version 2>&1 | grep -Eo '[0-9.]+') + pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def args2 = task.ext.args2 ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def outfile = "${prefix}.fastq" + def exclude_third = meta.single_end ? '' : "mv $outfile $prefix || echo 'No third file'" + // Excludes the "${prefix}.fastq" file from output `reads` channel for paired end cases and + // avoids the '.' in the path bug: https://github.com/ncbi/sra-tools/issues/865 + def key_file = '' + if (certificate.toString().endsWith('.jwt')) { + key_file += " --perm ${certificate}" + } else if (certificate.toString().endsWith('.ngc')) { + key_file += " --ngc ${certificate}" + } + def touch_outfiles = meta.single_end ? "${prefix}.fastq" : "${prefix}_1.fastq ${prefix}_2.fastq" + """ + touch $touch_outfiles + + export NCBI_SETTINGS="\$PWD/${ncbi_settings}" + + echo \\ + "fasterq-dump \\ + $args \\ + --threads $task.cpus \\ + --outfile $outfile \\ + ${key_file} \\ + ${sra}" + + $exclude_third + pigz \\ $args2 \\ --no-name \\ diff --git a/modules/nf-core/sratools/fasterqdump/meta.yml b/modules/nf-core/sratools/fasterqdump/meta.yml index b5e0175a..6a2151a8 100644 --- a/modules/nf-core/sratools/fasterqdump/meta.yml +++ b/modules/nf-core/sratools/fasterqdump/meta.yml @@ -51,3 +51,4 @@ authors: - "@Midnighter" maintainers: - "@Midnighter" + - "@gallvp" diff --git a/modules/nf-core/sratools/fasterqdump/tests/main.nf.test b/modules/nf-core/sratools/fasterqdump/tests/main.nf.test index 695394d4..6996cd18 100644 --- a/modules/nf-core/sratools/fasterqdump/tests/main.nf.test +++ b/modules/nf-core/sratools/fasterqdump/tests/main.nf.test @@ -1,10 +1,12 @@ nextflow_process { name "Test Process SRATOOLS_FASTERQDUMP" script "../main.nf" - config "./nextflow.config" process "SRATOOLS_FASTERQDUMP" - - tag "UNTAR" + tag "modules" + tag "modules_nfcore" + tag "untar" + tag "sratools" + tag "sratools/fasterqdump" test("Single-end") { @@ -67,4 +69,54 @@ nextflow_process { ) } } + + test("Single-end-stub") { + + options '-stub' + + when { + process { + """ + input[0] = [ + [ id:'test_single_end', single_end:true ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/sra/SRR13255544.tar.gz', checkIfExists: true) + ] + input[1] = file(params.modules_testdata_base_path + 'generic/config/ncbi_user_settings.mkfg', checkIfExists: true) + input[2] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + test("Paired-end-stub") { + + options '-stub' + + when { + process { + """ + input[0] = [ + [ id:'test_paired_end', single_end:false ], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/sra/SRR11140744.tar.gz', checkIfExists: true) + ] + input[1] = file(params.modules_testdata_base_path + 'generic/config/ncbi_user_settings.mkfg', checkIfExists: true) + input[2] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } } diff --git a/modules/nf-core/sratools/fasterqdump/tests/main.nf.test.snap b/modules/nf-core/sratools/fasterqdump/tests/main.nf.test.snap index ce0f9800..5d0c3e7c 100644 --- a/modules/nf-core/sratools/fasterqdump/tests/main.nf.test.snap +++ b/modules/nf-core/sratools/fasterqdump/tests/main.nf.test.snap @@ -12,7 +12,7 @@ ] ], "1": [ - "versions.yml:md5,6ff2d50b15c3f0eb9c72cd13a4a20295" + "versions.yml:md5,a3d61a9761e1606ef8459f0b68821d7a" ], "reads": [ [ @@ -24,15 +24,50 @@ ] ], "versions": [ - "versions.yml:md5,6ff2d50b15c3f0eb9c72cd13a4a20295" + "versions.yml:md5,a3d61a9761e1606ef8459f0b68821d7a" ] } ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.01.0" + "nextflow": "23.10.1" }, - "timestamp": "2024-02-28T15:25:52.837288" + "timestamp": "2024-05-17T22:01:43.486256" + }, + "Single-end-stub": { + "content": [ + { + "0": [ + [ + { + "id": "test_single_end", + "single_end": true + }, + "test_single_end.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,a3d61a9761e1606ef8459f0b68821d7a" + ], + "reads": [ + [ + { + "id": "test_single_end", + "single_end": true + }, + "test_single_end.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,a3d61a9761e1606ef8459f0b68821d7a" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-05-17T21:40:42.335786" }, "Paired-end": { "content": [ @@ -50,7 +85,7 @@ ] ], "1": [ - "versions.yml:md5,6ff2d50b15c3f0eb9c72cd13a4a20295" + "versions.yml:md5,a3d61a9761e1606ef8459f0b68821d7a" ], "reads": [ [ @@ -65,14 +100,55 @@ ] ], "versions": [ - "versions.yml:md5,6ff2d50b15c3f0eb9c72cd13a4a20295" + "versions.yml:md5,a3d61a9761e1606ef8459f0b68821d7a" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-05-17T22:09:55.6396" + }, + "Paired-end-stub": { + "content": [ + { + "0": [ + [ + { + "id": "test_paired_end", + "single_end": false + }, + [ + "test_paired_end_1.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e", + "test_paired_end_2.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + "versions.yml:md5,a3d61a9761e1606ef8459f0b68821d7a" + ], + "reads": [ + [ + { + "id": "test_paired_end", + "single_end": false + }, + [ + "test_paired_end_1.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e", + "test_paired_end_2.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "versions": [ + "versions.yml:md5,a3d61a9761e1606ef8459f0b68821d7a" ] } ], "meta": { "nf-test": "0.8.4", - "nextflow": "24.01.0" + "nextflow": "23.10.1" }, - "timestamp": "2024-02-28T15:26:42.466223" + "timestamp": "2024-05-17T20:35:44.782058" } } \ No newline at end of file diff --git a/modules/nf-core/sratools/fasterqdump/tests/nextflow.config b/modules/nf-core/sratools/fasterqdump/tests/nextflow.config deleted file mode 100644 index 23e4100b..00000000 --- a/modules/nf-core/sratools/fasterqdump/tests/nextflow.config +++ /dev/null @@ -1,5 +0,0 @@ -process { - withName: SRATOOLS_FASTERQDUMP { - ext.args = '' - } -} \ No newline at end of file diff --git a/modules/nf-core/sratools/fasterqdump/tests/tags.yml b/modules/nf-core/sratools/fasterqdump/tests/tags.yml new file mode 100644 index 00000000..5d1ddcb3 --- /dev/null +++ b/modules/nf-core/sratools/fasterqdump/tests/tags.yml @@ -0,0 +1,2 @@ +sratools/fasterqdump: + - modules/nf-core/sratools/fasterqdump/** diff --git a/modules/nf-core/sratools/prefetch/environment.yml b/modules/nf-core/sratools/prefetch/environment.yml index 0abad336..6596bc7c 100644 --- a/modules/nf-core/sratools/prefetch/environment.yml +++ b/modules/nf-core/sratools/prefetch/environment.yml @@ -1,8 +1,6 @@ -name: sratools_prefetch channels: - conda-forge - bioconda - - defaults dependencies: - bioconda::sra-tools=3.1.0 - conda-forge::curl=8.5.0 diff --git a/modules/nf-core/sratools/prefetch/main.nf b/modules/nf-core/sratools/prefetch/main.nf index 170f1753..74838d53 100644 --- a/modules/nf-core/sratools/prefetch/main.nf +++ b/modules/nf-core/sratools/prefetch/main.nf @@ -13,7 +13,7 @@ process SRATOOLS_PREFETCH { path certificate output: - tuple val(meta), path(id), emit: sra + tuple val(meta), path(id, type: 'dir'), emit: sra path 'versions.yml' , emit: versions when: @@ -32,4 +32,16 @@ process SRATOOLS_PREFETCH { } template 'retry_with_backoff.sh' + + stub: + """ + mkdir $id + touch $id/${id}.sra + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + sratools: \$(prefetch --version 2>&1 | grep -Eo '[0-9.]+') + curl: \$(curl --version | head -n 1 | sed 's/^curl //; s/ .*\$//') + END_VERSIONS + """ } diff --git a/modules/nf-core/sratools/prefetch/meta.yml b/modules/nf-core/sratools/prefetch/meta.yml index ff54229f..7ed42d49 100644 --- a/modules/nf-core/sratools/prefetch/meta.yml +++ b/modules/nf-core/sratools/prefetch/meta.yml @@ -54,3 +54,4 @@ authors: - "@Midnighter" maintainers: - "@Midnighter" + - "@gallvp" diff --git a/modules/nf-core/sratools/prefetch/tests/main.nf.test b/modules/nf-core/sratools/prefetch/tests/main.nf.test index f59de7cb..92034d40 100644 --- a/modules/nf-core/sratools/prefetch/tests/main.nf.test +++ b/modules/nf-core/sratools/prefetch/tests/main.nf.test @@ -46,4 +46,26 @@ nextflow_process { ) } } + + test("sratools/prefetch/stub") { + + options '-stub' + + when { + process { + """ + input[0] = Channel.of([ [ id:'test', single_end:false ], 'DRR000774' ]) + input[1] = file(params.modules_testdata_base_path + 'generic/config/ncbi_user_settings.mkfg', checkIfExists: true) + input[2] = [] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } } diff --git a/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap b/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap index 67795ebb..a1f82f45 100644 --- a/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap +++ b/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap @@ -1,4 +1,43 @@ { + "sratools/prefetch/stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + [ + "DRR000774.sra:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + "versions.yml:md5,83d1b23f5ff5b2ad1b96d17d7d7594ee" + ], + "sra": [ + [ + { + "id": "test", + "single_end": false + }, + [ + "DRR000774.sra:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "versions": [ + "versions.yml:md5,83d1b23f5ff5b2ad1b96d17d7d7594ee" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-05-17T20:07:31.627115" + }, "sratools/prefetch with sralite": { "content": [ { diff --git a/modules/nf-core/untar/environment.yml b/modules/nf-core/untar/environment.yml index 0c9cbb10..c7794856 100644 --- a/modules/nf-core/untar/environment.yml +++ b/modules/nf-core/untar/environment.yml @@ -1,11 +1,7 @@ -name: untar - channels: - conda-forge - bioconda - - defaults - dependencies: - conda-forge::grep=3.11 - - conda-forge::sed=4.7 + - conda-forge::sed=4.8 - conda-forge::tar=1.34 diff --git a/modules/nf-core/untar/main.nf b/modules/nf-core/untar/main.nf index 8a75bb95..9bd8f554 100644 --- a/modules/nf-core/untar/main.nf +++ b/modules/nf-core/untar/main.nf @@ -4,8 +4,8 @@ process UNTAR { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : - 'nf-core/ubuntu:20.04' }" + 'https://depot.galaxyproject.org/singularity/ubuntu:22.04' : + 'nf-core/ubuntu:22.04' }" input: tuple val(meta), path(archive) @@ -52,8 +52,29 @@ process UNTAR { stub: prefix = task.ext.prefix ?: ( meta.id ? "${meta.id}" : archive.toString().replaceFirst(/\.[^\.]+(.gz)?$/, "")) """ - mkdir $prefix - touch ${prefix}/file.txt + mkdir ${prefix} + ## Dry-run untaring the archive to get the files and place all in prefix + if [[ \$(tar -taf ${archive} | grep -o -P "^.*?\\/" | uniq | wc -l) -eq 1 ]]; then + for i in `tar -tf ${archive}`; + do + if [[ \$(echo "\${i}" | grep -E "/\$") == "" ]]; + then + touch \${i} + else + mkdir -p \${i} + fi + done + else + for i in `tar -tf ${archive}`; + do + if [[ \$(echo "\${i}" | grep -E "/\$") == "" ]]; + then + touch ${prefix}/\${i} + else + mkdir -p ${prefix}/\${i} + fi + done + fi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/untar/tests/main.nf.test b/modules/nf-core/untar/tests/main.nf.test index 98b769ad..c957517a 100644 --- a/modules/nf-core/untar/tests/main.nf.test +++ b/modules/nf-core/untar/tests/main.nf.test @@ -3,7 +3,10 @@ nextflow_process { name "Test Process UNTAR" script "../main.nf" process "UNTAR" - + tag "modules" + tag "modules_nfcore" + tag "untar" + test("test_untar") { when { @@ -17,10 +20,9 @@ nextflow_process { then { assertAll ( { assert process.success }, - { assert snapshot(process.out.untar).match("test_untar") }, + { assert snapshot(process.out).match() }, ) } - } test("test_untar_onlyfiles") { @@ -36,10 +38,48 @@ nextflow_process { then { assertAll ( { assert process.success }, - { assert snapshot(process.out.untar).match("test_untar_onlyfiles") }, + { assert snapshot(process.out).match() }, ) } + } + + test("test_untar - stub") { + + options "-stub" + when { + process { + """ + input[0] = [ [], file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/db/kraken2.tar.gz', checkIfExists: true) ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + ) + } } + test("test_untar_onlyfiles - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ [], file(params.modules_testdata_base_path + 'generic/tar/hello.tar.gz', checkIfExists: true) ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out).match() }, + ) + } + } } diff --git a/modules/nf-core/untar/tests/main.nf.test.snap b/modules/nf-core/untar/tests/main.nf.test.snap index 64550292..ceb91b79 100644 --- a/modules/nf-core/untar/tests/main.nf.test.snap +++ b/modules/nf-core/untar/tests/main.nf.test.snap @@ -1,42 +1,158 @@ { "test_untar_onlyfiles": { "content": [ - [ - [ + { + "0": [ [ - - ], + [ + + ], + [ + "hello.txt:md5,e59ff97941044f85df5297e1c302d260" + ] + ] + ], + "1": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" + ], + "untar": [ + [ + [ + + ], + [ + "hello.txt:md5,e59ff97941044f85df5297e1c302d260" + ] + ] + ], + "versions": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-10T12:04:28.231047" + }, + "test_untar_onlyfiles - stub": { + "content": [ + { + "0": [ + [ + [ + + ], + [ + "hello.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" + ], + "untar": [ [ - "hello.txt:md5,e59ff97941044f85df5297e1c302d260" + [ + + ], + [ + "hello.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ] + ], + "versions": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" ] - ] + } ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.3" }, - "timestamp": "2024-02-28T11:49:41.320643" + "timestamp": "2024-07-10T12:04:45.773103" + }, + "test_untar - stub": { + "content": [ + { + "0": [ + [ + [ + + ], + [ + "hash.k2d:md5,d41d8cd98f00b204e9800998ecf8427e", + "opts.k2d:md5,d41d8cd98f00b204e9800998ecf8427e", + "taxo.k2d:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "1": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" + ], + "untar": [ + [ + [ + + ], + [ + "hash.k2d:md5,d41d8cd98f00b204e9800998ecf8427e", + "opts.k2d:md5,d41d8cd98f00b204e9800998ecf8427e", + "taxo.k2d:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + "versions": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.04.3" + }, + "timestamp": "2024-07-10T12:04:36.777441" }, "test_untar": { "content": [ - [ - [ + { + "0": [ [ - - ], + [ + + ], + [ + "hash.k2d:md5,8b8598468f54a7087c203ad0190555d9", + "opts.k2d:md5,a033d00cf6759407010b21700938f543", + "taxo.k2d:md5,094d5891cdccf2f1468088855c214b2c" + ] + ] + ], + "1": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" + ], + "untar": [ [ - "hash.k2d:md5,8b8598468f54a7087c203ad0190555d9", - "opts.k2d:md5,a033d00cf6759407010b21700938f543", - "taxo.k2d:md5,094d5891cdccf2f1468088855c214b2c" + [ + + ], + [ + "hash.k2d:md5,8b8598468f54a7087c203ad0190555d9", + "opts.k2d:md5,a033d00cf6759407010b21700938f543", + "taxo.k2d:md5,094d5891cdccf2f1468088855c214b2c" + ] ] + ], + "versions": [ + "versions.yml:md5,6063247258c56fd271d076bb04dd7536" ] - ] + } ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.3" }, - "timestamp": "2024-02-28T11:49:33.795172" + "timestamp": "2024-07-10T12:04:19.377674" } } \ No newline at end of file diff --git a/modules/nf-core/untar/tests/tags.yml b/modules/nf-core/untar/tests/tags.yml new file mode 100644 index 00000000..feb6f15c --- /dev/null +++ b/modules/nf-core/untar/tests/tags.yml @@ -0,0 +1,2 @@ +untar: + - modules/nf-core/untar/** diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap index 0c0be2a4..1a34e0b8 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap @@ -40,8 +40,8 @@ "content": [ [ "versions.yml:md5,1a2218ff913fc33408bffccb081b5048", - "versions.yml:md5,2f3b3a13b36dabf13f09327613d5558d", - "versions.yml:md5,53d6e983afde3a28add2ffc6b7eba4f3" + "versions.yml:md5,53d6e983afde3a28add2ffc6b7eba4f3", + "versions.yml:md5,9c558ff624585a6eee82a19c8c0136db" ] ], "meta": { @@ -52,7 +52,7 @@ }, "test_pe_reads_1_size": { "content": [ - 2013376 + 2011460 ], "meta": { "nf-test": "0.8.4", diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf index ac31f28f..28e32b20 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -2,10 +2,6 @@ // Subworkflow with functionality that may be useful for any Nextflow pipeline // -import org.yaml.snakeyaml.Yaml -import groovy.json.JsonOutput -import nextflow.extension.FilesEx - /* ======================================================================================== SUBWORKFLOW DEFINITION @@ -58,7 +54,7 @@ workflow UTILS_NEXTFLOW_PIPELINE { // Generate version string // def getWorkflowVersion() { - String version_string = "" + def version_string = "" as String if (workflow.manifest.version) { def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' version_string += "${prefix_v}${workflow.manifest.version}" @@ -79,10 +75,10 @@ def dumpParametersToJSON(outdir) { def timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') def filename = "params_${timestamp}.json" def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") - def jsonStr = JsonOutput.toJson(params) - temp_pf.text = JsonOutput.prettyPrint(jsonStr) + def jsonStr = groovy.json.JsonOutput.toJson(params) + temp_pf.text = groovy.json.JsonOutput.prettyPrint(jsonStr) - FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") + nextflow.extension.FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") temp_pf.delete() } @@ -90,7 +86,7 @@ def dumpParametersToJSON(outdir) { // When running with -profile conda, warn if channels have not been set-up appropriately // def checkCondaChannels() { - Yaml parser = new Yaml() + def parser = new org.yaml.snakeyaml.Yaml() def channels = [] try { def config = parser.load("conda config --show channels".execute().text) @@ -102,14 +98,16 @@ def checkCondaChannels() { // Check that all channels are present // This channel list is ordered by required channel priority. - def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] + def required_channels_in_order = ['conda-forge', 'bioconda'] def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean // Check that they are in the right order def channel_priority_violation = false - def n = required_channels_in_order.size() - for (int i = 0; i < n - 1; i++) { - channel_priority_violation |= !(channels.indexOf(required_channels_in_order[i]) < channels.indexOf(required_channels_in_order[i+1])) + + required_channels_in_order.eachWithIndex { channel, index -> + if (index < required_channels_in_order.size() - 1) { + channel_priority_violation |= !(channels.indexOf(channel) < channels.indexOf(required_channels_in_order[index+1])) + } } if (channels_missing | channel_priority_violation) { diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test index e49d617f..68718e4f 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test @@ -4,7 +4,9 @@ nextflow_function { name "Test Functions" script "subworkflows/nf-core/utils_nextflow_pipeline/main.nf" config "subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config" - tag "UTILS_NEXTFLOW_PIPELINE" + tag 'subworkflows' + tag 'utils_nextflow_pipeline' + tag 'subworkflows/utils_nextflow_pipeline' test("Test Function getWorkflowVersion") { diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test index 13782037..ca964ce8 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test @@ -4,6 +4,9 @@ nextflow_workflow { script "../main.nf" config "subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config" workflow "UTILS_NEXTFLOW_PIPELINE" + tag 'subworkflows' + tag 'utils_nextflow_pipeline' + tag 'subworkflows/utils_nextflow_pipeline' test("Should run no inputs") { diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config index d0a926bf..a09572e5 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config @@ -3,7 +3,7 @@ manifest { author = """nf-core""" homePage = 'https://127.0.0.1' description = """Dummy pipeline""" - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=23.04.0' version = '9.9.9' doi = 'https://doi.org/10.5281/zenodo.5070524' } diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml b/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml new file mode 100644 index 00000000..f8476112 --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/utils_nextflow_pipeline: + - subworkflows/nf-core/utils_nextflow_pipeline/** diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index a8b55d6f..cbd8495b 100644 --- a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -2,9 +2,6 @@ // Subworkflow with utility functions specific to the nf-core pipeline template // -import org.yaml.snakeyaml.Yaml -import nextflow.extension.FilesEx - /* ======================================================================================== SUBWORKFLOW DEFINITION @@ -34,7 +31,7 @@ workflow UTILS_NFCORE_PIPELINE { // Warn if a -profile or Nextflow config has not been provided to run the pipeline // def checkConfigProvided() { - valid_config = true + def valid_config = true as Boolean if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { log.warn "[$workflow.manifest.name] You are attempting to run the pipeline without any custom configuration!\n\n" + "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + @@ -65,9 +62,17 @@ def checkProfileProvided(nextflow_cli_args) { // Citation string for pipeline // def workflowCitation() { + def temp_doi_ref = "" + def manifest_doi = workflow.manifest.doi.tokenize(",") + // Using a loop to handle multiple DOIs + // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers + // Removing ` ` since the manifest.doi is a string and not a proper list + manifest_doi.each { doi_ref -> + temp_doi_ref += " https://doi.org/${doi_ref.replace('https://doi.org/', '').replace(' ', '')}\n" + } return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + "* The pipeline\n" + - " ${workflow.manifest.doi}\n\n" + + temp_doi_ref + "\n" + "* The nf-core framework\n" + " https://doi.org/10.1038/s41587-020-0439-x\n\n" + "* Software dependencies\n" + @@ -78,7 +83,7 @@ def workflowCitation() { // Generate workflow version string // def getWorkflowVersion() { - String version_string = "" + def version_string = "" as String if (workflow.manifest.version) { def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' version_string += "${prefix_v}${workflow.manifest.version}" @@ -96,8 +101,8 @@ def getWorkflowVersion() { // Get software versions for pipeline // def processVersionsFromYAML(yaml_file) { - Yaml yaml = new Yaml() - versions = yaml.load(yaml_file).collectEntries { k, v -> [ k.tokenize(':')[-1], v ] } + def yaml = new org.yaml.snakeyaml.Yaml() + def versions = yaml.load(yaml_file).collectEntries { k, v -> [ k.tokenize(':')[-1], v ] } return yaml.dumpAsMap(versions).trim() } @@ -118,7 +123,7 @@ def workflowVersionToYAML() { def softwareVersionsToYAML(ch_versions) { return ch_versions .unique() - .map { processVersionsFromYAML(it) } + .map { version -> processVersionsFromYAML(version) } .unique() .mix(Channel.of(workflowVersionToYAML())) } @@ -128,19 +133,19 @@ def softwareVersionsToYAML(ch_versions) { // def paramsSummaryMultiqc(summary_params) { def summary_section = '' - for (group in summary_params.keySet()) { + summary_params.keySet().each { group -> def group_params = summary_params.get(group) // This gets the parameters of that particular group if (group_params) { summary_section += "

$group

\n" summary_section += "
\n" - for (param in group_params.keySet()) { + group_params.keySet().sort().each { param -> summary_section += "
$param
${group_params.get(param) ?: 'N/A'}
\n" } summary_section += "
\n" } } - String yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" + def yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" as String yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" @@ -155,7 +160,7 @@ def paramsSummaryMultiqc(summary_params) { // nf-core logo // def nfCoreLogo(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) + def colors = logColours(monochrome_logs) as Map String.format( """\n ${dashedLine(monochrome_logs)} @@ -174,7 +179,7 @@ def nfCoreLogo(monochrome_logs=true) { // Return dashed line // def dashedLine(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) + def colors = logColours(monochrome_logs) as Map return "-${colors.dim}----------------------------------------------------${colors.reset}-" } @@ -182,7 +187,7 @@ def dashedLine(monochrome_logs=true) { // ANSII colours used for terminal logging // def logColours(monochrome_logs=true) { - Map colorcodes = [:] + def colorcodes = [:] as Map // Reset / Meta colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" @@ -281,7 +286,7 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi } def summary = [:] - for (group in summary_params.keySet()) { + summary_params.keySet().sort().each { group -> summary << summary_params[group] } @@ -338,10 +343,10 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi def sendmail_html = sendmail_template.toString() // Send the HTML e-mail - Map colors = logColours(monochrome_logs) + def colors = logColours(monochrome_logs) as Map if (email_address) { try { - if (plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } + if (plaintext_email) { throw new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') } // Try to send HTML e-mail using sendmail def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") sendmail_tf.withWriter { w -> w << sendmail_html } @@ -358,13 +363,13 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi // Write summary e-mail HTML to a file def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") output_hf.withWriter { w -> w << email_html } - FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html"); + nextflow.extension.FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html"); output_hf.delete() // Write summary e-mail TXT to a file def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") output_tf.withWriter { w -> w << email_txt } - FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt"); + nextflow.extension.FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt"); output_tf.delete() } @@ -372,7 +377,7 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi // Print pipeline summary on completion // def completionSummary(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) + def colors = logColours(monochrome_logs) as Map if (workflow.success) { if (workflow.stats.ignoredCount == 0) { log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Pipeline completed successfully${colors.reset}-" @@ -389,7 +394,7 @@ def completionSummary(monochrome_logs=true) { // def imNotification(summary_params, hook_url) { def summary = [:] - for (group in summary_params.keySet()) { + summary_params.keySet().sort().each { group -> summary << summary_params[group] } diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml b/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml new file mode 100644 index 00000000..ac8523c9 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/utils_nfcore_pipeline: + - subworkflows/nf-core/utils_nfcore_pipeline/**