From cb31b886e153b81215d71a529fa8fb56a01e8dc5 Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Fri, 12 Jul 2024 14:02:22 +0000 Subject: [PATCH 01/10] Apply a blanket renaming in multiqc to catch inconsistent naming paired and unpaired --- modules.json | 4 ++-- modules/nf-core/multiqc/main.nf | 3 +++ modules/nf-core/multiqc/meta.yml | 7 +++++++ modules/nf-core/multiqc/tests/main.nf.test | 3 +++ workflows/rnaseq/main.nf | 12 +++++++++++- 5 files changed, 26 insertions(+), 3 deletions(-) diff --git a/modules.json b/modules.json index 320ddb009..34c17a2eb 100644 --- a/modules.json +++ b/modules.json @@ -91,8 +91,8 @@ "installed_by": ["modules", "quantify_pseudo_alignment"] }, "multiqc": { - "branch": "master", - "git_sha": "45d482f013f7da8a3e050e855f04ada418d91c3c", + "branch": "multiqc_rename_input", + "git_sha": "6123a870eda29fb2c8d6034cf396e576d41e2b70", "installed_by": ["modules"] }, "picard/markduplicates": { diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index ac624e869..80a5efaa6 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -11,6 +11,7 @@ process MULTIQC { path(multiqc_config) path(extra_multiqc_config) path(multiqc_logo) + path(replace_names) output: path "*multiqc_report.html", emit: report @@ -26,6 +27,7 @@ process MULTIQC { def config = multiqc_config ? "--config $multiqc_config" : '' def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' def logo = multiqc_logo ? /--cl-config 'custom_logo: "${multiqc_logo}"'/ : '' + def replace = replace_names ? "--replace-names ${replace_names}" : '' """ multiqc \\ --force \\ @@ -33,6 +35,7 @@ process MULTIQC { $config \\ $extra_config \\ $logo \\ + $replace \\ . cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml index 45a9bc35e..6de07593c 100644 --- a/modules/nf-core/multiqc/meta.yml +++ b/modules/nf-core/multiqc/meta.yml @@ -29,6 +29,13 @@ input: type: file description: Optional logo file for MultiQC pattern: "*.{png}" + - replace_names: + type: file + description: | + Optional two-column sample renaming file. First column a set of + patterns, second column a set of corresponding replacements. Passed via + MultiQC's `--replace-names` option. + pattern: "*.{tsv}" output: - report: type: file diff --git a/modules/nf-core/multiqc/tests/main.nf.test b/modules/nf-core/multiqc/tests/main.nf.test index f1c4242ef..c54f7df3f 100644 --- a/modules/nf-core/multiqc/tests/main.nf.test +++ b/modules/nf-core/multiqc/tests/main.nf.test @@ -17,6 +17,7 @@ nextflow_process { input[1] = [] input[2] = [] input[3] = [] + input[4] = [] """ } } @@ -41,6 +42,7 @@ nextflow_process { input[1] = Channel.of(file("https://github.com/nf-core/tools/raw/dev/nf_core/pipeline-template/assets/multiqc_config.yml", checkIfExists: true)) input[2] = [] input[3] = [] + input[4] = [] """ } } @@ -66,6 +68,7 @@ nextflow_process { input[1] = [] input[2] = [] input[3] = [] + input[4] = [] """ } } diff --git a/workflows/rnaseq/main.nf b/workflows/rnaseq/main.nf index 4eefce0ed..b0cc2ba1e 100755 --- a/workflows/rnaseq/main.nf +++ b/workflows/rnaseq/main.nf @@ -114,6 +114,9 @@ workflow RNASEQ { // Channel .fromSamplesheet("input") + .set{ ch_samples } + + ch_samples .map { meta, fastq_1, fastq_2 -> if (!fastq_2) { @@ -857,6 +860,7 @@ workflow RNASEQ { // MODULE: MultiQC // ch_multiqc_report = Channel.empty() + if (!params.skip_multiqc) { ch_multiqc_config = Channel.fromPath("$projectDir/workflows/rnaseq/assets/multiqc/multiqc_config.yml", checkIfExists: true) ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config) : Channel.empty() @@ -865,12 +869,18 @@ workflow RNASEQ { ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) + ch_name_replacements = ch_samples + .map{ + it[1].name.split('\\.')[0] + "\t" + it[0].id + } + .collectFile(name: 'name_replacement.txt', newLine: true) MULTIQC ( ch_multiqc_files.collect(), ch_multiqc_config.toList(), ch_multiqc_custom_config.toList(), - ch_multiqc_logo.toList() + ch_multiqc_logo.toList(), + ch_name_replacements ) ch_multiqc_report = MULTIQC.out.report } From 6dcf3fd0b09e8431414bdeae3512e58e158db342 Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Fri, 12 Jul 2024 14:09:35 +0000 Subject: [PATCH 02/10] Switch renaming to exact matches --- workflows/rnaseq/assets/multiqc/multiqc_config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/workflows/rnaseq/assets/multiqc/multiqc_config.yml b/workflows/rnaseq/assets/multiqc/multiqc_config.yml index 6af9a02e4..2bf39b16c 100644 --- a/workflows/rnaseq/assets/multiqc/multiqc_config.yml +++ b/workflows/rnaseq/assets/multiqc/multiqc_config.yml @@ -10,6 +10,7 @@ report_section_order: export_plots: true disable_version_detection: true +sample_names_replace_exact: true # Run only these modules run_modules: From e12ea636c6f90ea8d17b2d0d03df751245449050 Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Fri, 12 Jul 2024 17:01:40 +0000 Subject: [PATCH 03/10] Bump multiqc module --- modules.json | 4 ++-- modules/nf-core/multiqc/main.nf | 3 +++ modules/nf-core/multiqc/meta.yml | 6 ++++++ modules/nf-core/multiqc/tests/main.nf.test | 3 +++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/modules.json b/modules.json index 34c17a2eb..b0125259a 100644 --- a/modules.json +++ b/modules.json @@ -91,8 +91,8 @@ "installed_by": ["modules", "quantify_pseudo_alignment"] }, "multiqc": { - "branch": "multiqc_rename_input", - "git_sha": "6123a870eda29fb2c8d6034cf396e576d41e2b70", + "branch": "master", + "git_sha": "b80f5fd12ff7c43938f424dd76392a2704fa2396", "installed_by": ["modules"] }, "picard/markduplicates": { diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index 80a5efaa6..459dfea5b 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -12,6 +12,7 @@ process MULTIQC { path(extra_multiqc_config) path(multiqc_logo) path(replace_names) + path(sample_names) output: path "*multiqc_report.html", emit: report @@ -28,6 +29,7 @@ process MULTIQC { def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' def logo = multiqc_logo ? /--cl-config 'custom_logo: "${multiqc_logo}"'/ : '' def replace = replace_names ? "--replace-names ${replace_names}" : '' + def samples = sample_names ? "--sample-names ${sample_names}" : '' """ multiqc \\ --force \\ @@ -36,6 +38,7 @@ process MULTIQC { $extra_config \\ $logo \\ $replace \\ + $samples \\ . cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml index 6de07593c..382c08cbc 100644 --- a/modules/nf-core/multiqc/meta.yml +++ b/modules/nf-core/multiqc/meta.yml @@ -36,6 +36,12 @@ input: patterns, second column a set of corresponding replacements. Passed via MultiQC's `--replace-names` option. pattern: "*.{tsv}" + - sample_names: + type: file + description: | + Optional TSV file with headers, passed to the MultiQC --sample_names + argument. + pattern: "*.{tsv}" output: - report: type: file diff --git a/modules/nf-core/multiqc/tests/main.nf.test b/modules/nf-core/multiqc/tests/main.nf.test index c54f7df3f..6aa27f4c9 100644 --- a/modules/nf-core/multiqc/tests/main.nf.test +++ b/modules/nf-core/multiqc/tests/main.nf.test @@ -18,6 +18,7 @@ nextflow_process { input[2] = [] input[3] = [] input[4] = [] + input[5] = [] """ } } @@ -43,6 +44,7 @@ nextflow_process { input[2] = [] input[3] = [] input[4] = [] + input[5] = [] """ } } @@ -69,6 +71,7 @@ nextflow_process { input[2] = [] input[3] = [] input[4] = [] + input[5] = [] """ } } From 47484f617e33dee70a9e730a1836edb44afecd53 Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Fri, 12 Jul 2024 17:02:46 +0000 Subject: [PATCH 04/10] Separate rename patterns for forward and reverse --- workflows/rnaseq/main.nf | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/workflows/rnaseq/main.nf b/workflows/rnaseq/main.nf index b0cc2ba1e..667df95c6 100755 --- a/workflows/rnaseq/main.nf +++ b/workflows/rnaseq/main.nf @@ -869,10 +869,23 @@ workflow RNASEQ { ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) + + // Derive a set of rename patterns such that any sample names derived + // from input fastqs in MultiQC are replaced with specified sample name + // strings + ch_name_replacements = ch_samples .map{ - it[1].name.split('\\.')[0] + "\t" + it[0].id + def read1 = it[1].name.split('\\.')[0] + "\t" + it[0].id + '_1' + def read2 = '' + if (it[2] ){ + read2 = it[2].name.split('\\.')[0] + "\t" + it[0].id + '_2' + } + return [[read1, read2]] } + .transpose() + .filter { it[0] != ''} + .map{it[0]} .collectFile(name: 'name_replacement.txt', newLine: true) MULTIQC ( @@ -880,7 +893,8 @@ workflow RNASEQ { ch_multiqc_config.toList(), ch_multiqc_custom_config.toList(), ch_multiqc_logo.toList(), - ch_name_replacements + ch_name_replacements, + [] ) ch_multiqc_report = MULTIQC.out.report } From 6fccd133155431b5c4d79b21f20219e5b1bb3036 Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Fri, 12 Jul 2024 17:13:07 +0000 Subject: [PATCH 05/10] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 750a30fbb..560d60f00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,7 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements - [PR #1330](https://github.com/nf-core/rnaseq/pull/1330) - Update all nf-core/modules and subworkflows - [PR #1331](https://github.com/nf-core/rnaseq/pull/1331) - Adding stubs for local modules - [PR #1334](https://github.com/nf-core/rnaseq/pull/1334) - Update all nf-core/modules and subworkflows with stubs +- [PR #1341](https://github.com/nf-core/rnaseq/pull/1341) - Add wholesale rename in the MultiQC report ### Parameters From 39071b9032a298eeb593fa545750171b5bc35201 Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Mon, 15 Jul 2024 08:41:10 +0000 Subject: [PATCH 06/10] simplify logic to generate renaming table --- workflows/rnaseq/main.nf | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/workflows/rnaseq/main.nf b/workflows/rnaseq/main.nf index 667df95c6..4161ac6a1 100755 --- a/workflows/rnaseq/main.nf +++ b/workflows/rnaseq/main.nf @@ -875,17 +875,16 @@ workflow RNASEQ { // strings ch_name_replacements = ch_samples - .map{ - def read1 = it[1].name.split('\\.')[0] + "\t" + it[0].id + '_1' - def read2 = '' - if (it[2] ){ - read2 = it[2].name.split('\\.')[0] + "\t" + it[0].id + '_2' + .map{ meta, reads1, reads2 -> + def name1 = file(reads1).simpleName + "\t" + meta.id + '_1' + if (reads2 ){ + def name2 = file(reads2).simpleName + "\t" + meta.id + '_2' + return [ name1, name2 ] + } else{ + return name1 } - return [[read1, read2]] } - .transpose() - .filter { it[0] != ''} - .map{it[0]} + .flatten() .collectFile(name: 'name_replacement.txt', newLine: true) MULTIQC ( From a09053085d0650a3c35760bd7700759b2aedba8f Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Mon, 15 Jul 2024 14:32:17 +0000 Subject: [PATCH 07/10] Make name replacement specific to samples without tech reps --- workflows/rnaseq/main.nf | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/workflows/rnaseq/main.nf b/workflows/rnaseq/main.nf index 4161ac6a1..2b1cb004c 100755 --- a/workflows/rnaseq/main.nf +++ b/workflows/rnaseq/main.nf @@ -114,9 +114,6 @@ workflow RNASEQ { // Channel .fromSamplesheet("input") - .set{ ch_samples } - - ch_samples .map { meta, fastq_1, fastq_2 -> if (!fastq_2) { @@ -870,15 +867,15 @@ workflow RNASEQ { ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) - // Derive a set of rename patterns such that any sample names derived - // from input fastqs in MultiQC are replaced with specified sample name - // strings + // Generate replacements based on sample names to ensure MultiQC + // renames sequence files correctly if they didn't go through + // CAT_FASTQ. - ch_name_replacements = ch_samples - .map{ meta, reads1, reads2 -> - def name1 = file(reads1).simpleName + "\t" + meta.id + '_1' - if (reads2 ){ - def name2 = file(reads2).simpleName + "\t" + meta.id + '_2' + ch_name_replacements = ch_fastq.single + .map{ meta, reads -> + def name1 = file(reads[0]).simpleName + "\t" + meta.id + '_1' + if (reads[1] ){ + def name2 = file(reads[1]).simpleName + "\t" + meta.id + '_2' return [ name1, name2 ] } else{ return name1 From 182f5232f240b2882d84ed010eea66651a045f6c Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Mon, 15 Jul 2024 16:01:08 +0100 Subject: [PATCH 08/10] Clarify comment --- workflows/rnaseq/main.nf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/workflows/rnaseq/main.nf b/workflows/rnaseq/main.nf index 2b1cb004c..589e779ce 100755 --- a/workflows/rnaseq/main.nf +++ b/workflows/rnaseq/main.nf @@ -866,10 +866,9 @@ workflow RNASEQ { ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) - - // Generate replacements based on sample names to ensure MultiQC - // renames sequence files correctly if they didn't go through - // CAT_FASTQ. + + // Provide MultiQC with rename patterns to ensure it uses sample names + // for single-techrep samples not processed by CAT_FASTQ. ch_name_replacements = ch_fastq.single .map{ meta, reads -> From d7ed3530729d7851cbd2ba1e3bebb0abbd0a4c85 Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Mon, 15 Jul 2024 16:03:19 +0100 Subject: [PATCH 09/10] Update main.nf --- workflows/rnaseq/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/rnaseq/main.nf b/workflows/rnaseq/main.nf index 589e779ce..30eac1986 100755 --- a/workflows/rnaseq/main.nf +++ b/workflows/rnaseq/main.nf @@ -866,7 +866,7 @@ workflow RNASEQ { ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) - + // Provide MultiQC with rename patterns to ensure it uses sample names // for single-techrep samples not processed by CAT_FASTQ. From f35f51b686f01d9dcd61517d15dd65dd0673d867 Mon Sep 17 00:00:00 2001 From: Jonathan Manning Date: Mon, 15 Jul 2024 16:10:17 +0100 Subject: [PATCH 10/10] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e244a8f41..0a114c42a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,7 +100,7 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements - [PR #1331](https://github.com/nf-core/rnaseq/pull/1331) - Adding stubs for local modules - [PR #1334](https://github.com/nf-core/rnaseq/pull/1334) - Update all nf-core/modules and subworkflows with stubs - [PR #1340](https://github.com/nf-core/rnaseq/pull/1340) - Remove out-of-date Azure specific guidance -- [PR #1341](https://github.com/nf-core/rnaseq/pull/1341) - Add wholesale rename in the MultiQC report +- [PR #1341](https://github.com/nf-core/rnaseq/pull/1341) - Add rename in the MultiQC report for samples without techreps ### Parameters