From d1e0ec7670fa77905a378627232566ce54c3c26d Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Thu, 26 Sep 2024 17:01:35 +0200 Subject: [PATCH 1/2] create indices when missing (#6697) * create indices when missing * fix meta yaml --- modules/nf-core/bcftools/concat/main.nf | 16 +- modules/nf-core/bcftools/concat/meta.yml | 27 ++- .../bcftools/concat/tests/main.nf.test | 18 +- .../bcftools/concat/tests/main.nf.test.snap | 158 +++++++++--------- 4 files changed, 116 insertions(+), 103 deletions(-) diff --git a/modules/nf-core/bcftools/concat/main.nf b/modules/nf-core/bcftools/concat/main.nf index e2337eff257..a94b28d86d7 100644 --- a/modules/nf-core/bcftools/concat/main.nf +++ b/modules/nf-core/bcftools/concat/main.nf @@ -11,18 +11,22 @@ process BCFTOOLS_CONCAT { tuple val(meta), path(vcfs), path(tbi) output: - tuple val(meta), path("*.gz") , emit: vcf - tuple val(meta), path("*.tbi"), emit: tbi, optional: true - tuple val(meta), path("*.csi"), emit: csi, optional: true - path "versions.yml" , emit: versions + tuple val(meta), path("${prefix}.vcf.gz") , emit: vcf + tuple val(meta), path("${prefix}.vcf.gz.tbi"), emit: tbi, optional: true + tuple val(meta), path("${prefix}.vcf.gz.csi"), emit: csi, optional: true + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when script: def args = task.ext.args ?: '' - def prefix = task.ext.prefix ?: "${meta.id}" + prefix = task.ext.prefix ?: "${meta.id}" + def tbi_names = tbi.findAll { file -> !(file instanceof List) }.collect { file -> file.name } + def create_input_index = vcfs.collect { vcf -> tbi_names.contains(vcf.name + ".tbi") ? "" : "tabix ${vcf}" }.join("\n ") """ + ${create_input_index} + bcftools concat \\ --output ${prefix}.vcf.gz \\ $args \\ @@ -37,7 +41,7 @@ process BCFTOOLS_CONCAT { stub: def args = task.ext.args ?: '' - def prefix = task.ext.prefix ?: "${meta.id}" + prefix = task.ext.prefix ?: "${meta.id}" def index = args.contains("--write-index=tbi") || args.contains("-W=tbi") ? "tbi" : args.contains("--write-index=csi") || args.contains("-W=csi") ? "csi" : args.contains("--write-index") || args.contains("-W") ? "csi" : diff --git a/modules/nf-core/bcftools/concat/meta.yml b/modules/nf-core/bcftools/concat/meta.yml index b5d1f5b2cef..d2565b289fb 100644 --- a/modules/nf-core/bcftools/concat/meta.yml +++ b/modules/nf-core/bcftools/concat/meta.yml @@ -37,9 +37,12 @@ output: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - "*.gz": - type: file - description: VCF concatenated output file + pattern: "*.{vcf.gz}" + - ${prefix}.vcf.gz: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] pattern: "*.{vcf.gz}" - tbi: - meta: @@ -47,9 +50,12 @@ output: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - "*.tbi": - type: file - description: Alternative VCF file index + pattern: "*.tbi" + - ${prefix}.vcf.gz.tbi: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] pattern: "*.tbi" - csi: - meta: @@ -57,9 +63,12 @@ output: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - "*.csi": - type: file - description: Default VCF file index + pattern: "*.csi" + - ${prefix}.vcf.gz.csi: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] pattern: "*.csi" - versions: - versions.yml: diff --git a/modules/nf-core/bcftools/concat/tests/main.nf.test b/modules/nf-core/bcftools/concat/tests/main.nf.test index cea386e3bd3..cb4642b29cd 100644 --- a/modules/nf-core/bcftools/concat/tests/main.nf.test +++ b/modules/nf-core/bcftools/concat/tests/main.nf.test @@ -10,7 +10,7 @@ nextflow_process { tag "bcftools/concat" - test("sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]]") { + test("homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]]") { config "./nextflow.config" @@ -41,7 +41,7 @@ nextflow_process { } - test("sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index") { + test("homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index") { config "./vcf_gz_index.config" @@ -78,7 +78,7 @@ nextflow_process { } - test("sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_csi") { + test("homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_csi") { config "./vcf_gz_index_csi.config" @@ -115,7 +115,7 @@ nextflow_process { } - test("sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_tbi") { + test("homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_tbi") { config "./vcf_gz_index_tbi.config" @@ -153,7 +153,7 @@ nextflow_process { } - test("sarscov2 - [[vcf1, vcf2], []]") { + test("homo_sapiens - [[vcf1, vcf2], []]") { config "./nextflow.config" @@ -181,7 +181,7 @@ nextflow_process { } - test("sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]] - stub") { + test("homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]] - stub") { config "./nextflow.config" options "-stub" @@ -213,7 +213,7 @@ nextflow_process { } - test("sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index - stub") { + test("homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index - stub") { config "./vcf_gz_index.config" options "-stub" @@ -246,7 +246,7 @@ nextflow_process { } - test("sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_csi - stub") { + test("homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_csi - stub") { config "./vcf_gz_index_csi.config" options "-stub" @@ -279,7 +279,7 @@ nextflow_process { } - test("sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_tbi - stub") { + test("homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_tbi - stub") { config "./vcf_gz_index_tbi.config" options "-stub" diff --git a/modules/nf-core/bcftools/concat/tests/main.nf.test.snap b/modules/nf-core/bcftools/concat/tests/main.nf.test.snap index 1182854f947..09e87cd3e5d 100644 --- a/modules/nf-core/bcftools/concat/tests/main.nf.test.snap +++ b/modules/nf-core/bcftools/concat/tests/main.nf.test.snap @@ -1,5 +1,5 @@ { - "sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index - stub": { + "homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index - stub": { "content": [ { "0": [ @@ -49,12 +49,12 @@ } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-06-05T08:09:13.734103412" + "timestamp": "2024-09-26T11:04:11.178539482" }, - "sarscov2 - [[vcf1, vcf2], []]": { + "homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]]": { "content": [ { "0": [ @@ -94,12 +94,12 @@ } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-06-04T15:19:09.213249578" + "timestamp": "2024-09-26T11:03:08.765639958" }, - "sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index": { + "homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index": { "content": [ [ [ @@ -125,12 +125,12 @@ ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-06-05T08:08:23.981388325" + "timestamp": "2024-09-26T11:03:21.607274757" }, - "sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_csi - stub": { + "homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_tbi - stub": { "content": [ { "0": [ @@ -142,29 +142,29 @@ ] ], "1": [ - - ], - "2": [ [ { "id": "test3" }, - "test3_vcf.vcf.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + "test3_vcf.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" ] + ], + "2": [ + ], "3": [ "versions.yml:md5,c6e19f105510a46af1c5da9064e2e659" ], "csi": [ + + ], + "tbi": [ [ { "id": "test3" }, - "test3_vcf.vcf.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + "test3_vcf.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" ] - ], - "tbi": [ - ], "vcf": [ [ @@ -180,12 +180,43 @@ } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-09-26T11:04:27.332133878" + }, + "homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_csi": { + "content": [ + [ + [ + { + "id": "test3" + }, + "test3_vcf.vcf.gz:md5,5f6796c3ae109a1a5b87353954693f5a" + ] + ], + [ + [ + { + "id": "test3" + }, + "test3_vcf.vcf.gz.csi" + ] + ], + [ + + ], + [ + "versions.yml:md5,c6e19f105510a46af1c5da9064e2e659" + ] + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-06-05T14:01:24.419027693" + "timestamp": "2024-09-26T11:03:36.575719606" }, - "sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]]": { + "homo_sapiens - [[vcf1, vcf2], []]": { "content": [ { "0": [ @@ -225,43 +256,12 @@ } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-06-04T15:19:03.597061078" + "timestamp": "2024-09-26T11:03:54.069826178" }, - "sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_csi": { - "content": [ - [ - [ - { - "id": "test3" - }, - "test3_vcf.vcf.gz:md5,5f6796c3ae109a1a5b87353954693f5a" - ] - ], - [ - [ - { - "id": "test3" - }, - "test3_vcf.vcf.gz.csi" - ] - ], - [ - - ], - [ - "versions.yml:md5,c6e19f105510a46af1c5da9064e2e659" - ] - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" - }, - "timestamp": "2024-06-05T14:00:10.868487669" - }, - "sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]] - stub": { + "homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]] - stub": { "content": [ { "0": [ @@ -301,12 +301,12 @@ } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-06-04T15:19:14.836256897" + "timestamp": "2024-09-26T11:04:02.45346063" }, - "sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_tbi": { + "homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_tbi": { "content": [ [ [ @@ -332,12 +332,12 @@ ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-06-05T14:00:31.061411617" + "timestamp": "2024-09-26T11:03:44.618596639" }, - "sarscov2 - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_tbi - stub": { + "homo_sapiens - [[vcf1, vcf2], [tbi1, tbi2]] - vcf_gz_index_csi - stub": { "content": [ { "0": [ @@ -349,29 +349,29 @@ ] ], "1": [ + + ], + "2": [ [ { "id": "test3" }, - "test3_vcf.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + "test3_vcf.vcf.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" ] - ], - "2": [ - ], "3": [ "versions.yml:md5,c6e19f105510a46af1c5da9064e2e659" ], "csi": [ - - ], - "tbi": [ [ { "id": "test3" }, - "test3_vcf.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + "test3_vcf.vcf.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" ] + ], + "tbi": [ + ], "vcf": [ [ @@ -387,9 +387,9 @@ } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-06-05T14:01:35.209746508" + "timestamp": "2024-09-26T11:04:19.745768656" } } \ No newline at end of file From a90a8863a3408e15d5a339b06413e57cf9ca9860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Schcolnicov?= <90359308+nschcolnicov@users.noreply.github.com> Date: Thu, 26 Sep 2024 18:32:48 -0300 Subject: [PATCH 2/2] Updated stub mkfastq (#6700) --- modules/nf-core/cellranger/mkfastq/main.nf | 27 +++++-------------- .../mkfastq/tests/main.nf.test.snap | 14 +++++++--- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/modules/nf-core/cellranger/mkfastq/main.nf b/modules/nf-core/cellranger/mkfastq/main.nf index 06c15bbc151..09480345aac 100644 --- a/modules/nf-core/cellranger/mkfastq/main.nf +++ b/modules/nf-core/cellranger/mkfastq/main.nf @@ -48,34 +48,19 @@ process CELLRANGER_MKFASTQ { } def prefix = task.ext.prefix ?: "${meta.id}" """ - mkdir -p "${prefix}_outs/outs/fastq_path/sample/files/" - # data with something to avoid breaking nf-test java I/O stream, fastq - cat <<-FAKE_FQ > ${prefix}_outs/outs/fastq_path/fake_file.fastq - @SEQ_ID - GATTTGGGGTTCAAAGCAGTATCGATCAAATAGTAAATCCATTTGTTCAACTCACAGTTT - + - !''*((((***+))%%%++)(%%%%).1***-+*''))**55CCF>>>>>>CCCCCCC65 - FAKE_FQ - gzip -n ${prefix}_outs/outs/fastq_path/fake_file.fastq - - # data with something to avoid breaking nf-test java I/O stream, fastq_undetermined - cat <<-FAKE_FQ > ${prefix}_outs/outs/fastq_path/sample/files/fake_file.fastq - @SEQ_ID - GATTTGGGGTTCAAAGCAGTATCGATCAAATAGTAAATCCATTTGTTCAACTCACAGTTT - + - !''*((((***+))%%%++)(%%%%).1***-+*''))**55CCF>>>>>>CCCCCCC65 - FAKE_FQ - gzip -n ${prefix}_outs/outs/fastq_path/sample/files/fake_file.fastq - # data for reports output channel mkdir -p "${prefix}_outs/outs/fastq_path/Reports" - # data for stats output channel mkdir -p "${prefix}_outs/outs/fastq_path/Stats" - # data for interops output channel mkdir -p "${prefix}_outs/outs/interop_path/" touch "${prefix}_outs/outs/interop_path/IndexMetricsOut.bin" + # data for fastq channels + mkdir -p "${prefix}_outs/outs/fastq_path/sample/files/" + touch "${prefix}_outs/outs/fastq_path/Undetermined_fake_file.fastq" + touch "${prefix}_outs/outs/fastq_path/sample/files/fake_file.fastq" + gzip "${prefix}_outs/outs/fastq_path/Undetermined_fake_file.fastq" + gzip "${prefix}_outs/outs/fastq_path/sample/files/fake_file.fastq" cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/cellranger/mkfastq/tests/main.nf.test.snap b/modules/nf-core/cellranger/mkfastq/tests/main.nf.test.snap index b12246889dd..f2c1b325862 100644 --- a/modules/nf-core/cellranger/mkfastq/tests/main.nf.test.snap +++ b/modules/nf-core/cellranger/mkfastq/tests/main.nf.test.snap @@ -54,11 +54,17 @@ "id": "test", "lane": 1 }, - "fake_file.fastq.gz:md5,8e16c1f4a441cbb3b7de374d2c924da7" + "fake_file.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], [ - + [ + { + "id": "test", + "lane": 1 + }, + "Undetermined_fake_file.fastq.gz:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], [ [ @@ -75,9 +81,9 @@ ], "meta": { "nf-test": "0.9.0", - "nextflow": "24.04.3" + "nextflow": "24.04.4" }, - "timestamp": "2024-08-15T20:01:48.013075" + "timestamp": "2024-09-26T18:47:18.168033527" }, "cellranger - tiny - illumina": { "content": [