From 65034b59d4e25980cdba117c06913a2d4506a155 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Mon, 20 Nov 2023 22:25:04 +0000 Subject: [PATCH 01/49] Added ability to auto-create samplesheet for phageannotator --- conf/modules.config | 15 ++ conf/test.config | 14 ++ modules.json | 5 + modules/local/mag_merge_samplesheet.nf | 27 ++++ modules/local/mag_to_samplesheet.nf | 40 +++++ modules/nf-core/cat/cat/environment.yml | 7 + modules/nf-core/cat/cat/main.nf | 62 +++++++ modules/nf-core/cat/cat/meta.yml | 36 +++++ modules/nf-core/cat/cat/tests/main.nf.test | 153 ++++++++++++++++++ .../nf-core/cat/cat/tests/main.nf.test.snap | 121 ++++++++++++++ .../cat/tests/nextflow_unzipped_zipped.config | 6 + .../cat/tests/nextflow_zipped_unzipped.config | 8 + modules/nf-core/cat/cat/tests/tags.yml | 2 + nextflow.config | 1 + subworkflows/local/samplesheet_creation.nf | 115 +++++++++++++ workflows/mag.nf | 7 + 16 files changed, 619 insertions(+) create mode 100644 modules/local/mag_merge_samplesheet.nf create mode 100644 modules/local/mag_to_samplesheet.nf create mode 100644 modules/nf-core/cat/cat/environment.yml create mode 100644 modules/nf-core/cat/cat/main.nf create mode 100644 modules/nf-core/cat/cat/meta.yml create mode 100644 modules/nf-core/cat/cat/tests/main.nf.test create mode 100644 modules/nf-core/cat/cat/tests/main.nf.test.snap create mode 100644 modules/nf-core/cat/cat/tests/nextflow_unzipped_zipped.config create mode 100644 modules/nf-core/cat/cat/tests/nextflow_zipped_unzipped.config create mode 100644 modules/nf-core/cat/cat/tests/tags.yml create mode 100644 subworkflows/local/samplesheet_creation.nf diff --git a/conf/modules.config b/conf/modules.config index cbfa51fb..28d5afc5 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -741,6 +741,21 @@ process { ] } + withName: MAG_TO_SAMPLESHEET { + publishDir = [ + path: { "${params.outdir}/samplesheet" }, + enabled: false + ] + } + + withName: 'MAG_MERGE_SAMPLESHEET' { + publishDir = [ + path: { "${params.outdir}/samplesheet" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + withName: CUSTOM_DUMPSOFTWAREVERSIONS { publishDir = [ path: { "${params.outdir}/pipeline_info" }, diff --git a/conf/test.config b/conf/test.config index 9c93278f..3a5ea4f4 100644 --- a/conf/test.config +++ b/conf/test.config @@ -30,4 +30,18 @@ params { busco_clean = true skip_gtdbtk = true skip_concoct = true + + // For computational efficiency + nf_core_pipeline = 'phageannotator' + coassemble_group = false + skip_binning = true + skip_prokka = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + skip_krona = true + skip_adapter_trimming = true + skip_metabat2 = true + skip_maxbin2 = true + skip_busco = true } diff --git a/modules.json b/modules.json index e9162243..cbe0a14f 100644 --- a/modules.json +++ b/modules.json @@ -36,6 +36,11 @@ "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, + "cat/cat": { + "branch": "master", + "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", + "installed_by": ["modules"] + }, "cat/fastq": { "branch": "master", "git_sha": "5c460c5a4736974abde2843294f35307ee2b0e5e", diff --git a/modules/local/mag_merge_samplesheet.nf b/modules/local/mag_merge_samplesheet.nf new file mode 100644 index 00000000..5ad7d01a --- /dev/null +++ b/modules/local/mag_merge_samplesheet.nf @@ -0,0 +1,27 @@ +process MAG_MERGE_SAMPLESHEET { + + conda "conda-forge::sed=4.7" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : + 'nf-core/ubuntu:20.04' }" + + input: + path ('samplesheets/*') + + output: + path "samplesheet.csv", emit: samplesheet + path "versions.yml" , emit: versions + + script: + """ + head -n 1 `ls ./samplesheets/* | head -n 1` > samplesheet.csv + for fileid in `ls ./samplesheets/*`; do + awk 'NR>1' \$fileid >> samplesheet.csv + done + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + sed: \$(echo \$(sed --version 2>&1) | sed 's/^.*GNU sed) //; s/ .*\$//') + END_VERSIONS + """ +} diff --git a/modules/local/mag_to_samplesheet.nf b/modules/local/mag_to_samplesheet.nf new file mode 100644 index 00000000..a454bb44 --- /dev/null +++ b/modules/local/mag_to_samplesheet.nf @@ -0,0 +1,40 @@ +process MAG_TO_SAMPLESHEET { + tag "$meta.id" + + executor 'local' + memory 100.MB + + input: + val meta + val pipeline + + output: + tuple val(meta), path("*samplesheet.csv"), emit: samplesheet + + exec: + // + // Create samplesheet containing metadata + // + + // Add nf-core pipeline specific entries + if (pipeline) { + if (pipeline == 'phageannotator') { + pipeline_map = [ + sample : "${meta.id}", + group : "${meta.group}", + fastq_1 : meta.fastq_1, + fastq_2 : meta.fastq_2, + fasta : meta.fasta + ] + } + } + + // Create a samplesheet + samplesheet = pipeline_map.keySet().collect{ '"' + it + '"'}.join(",") + '\n' + samplesheet += pipeline_map.values().collect{ '"' + it + '"'}.join(",") + + // Write samplesheet to file + def samplesheet_file = task.workDir.resolve("${meta.id}.samplesheet.csv") + samplesheet_file.text = samplesheet + +} diff --git a/modules/nf-core/cat/cat/environment.yml b/modules/nf-core/cat/cat/environment.yml new file mode 100644 index 00000000..17a04ef2 --- /dev/null +++ b/modules/nf-core/cat/cat/environment.yml @@ -0,0 +1,7 @@ +name: cat_cat +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - conda-forge::pigz=2.3.4 diff --git a/modules/nf-core/cat/cat/main.nf b/modules/nf-core/cat/cat/main.nf new file mode 100644 index 00000000..4264a92c --- /dev/null +++ b/modules/nf-core/cat/cat/main.nf @@ -0,0 +1,62 @@ +process CAT_CAT { + tag "$meta.id" + label 'process_low' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/pigz:2.3.4' : + 'biocontainers/pigz:2.3.4' }" + + input: + tuple val(meta), path(files_in) + + output: + tuple val(meta), path("${prefix}"), emit: file_out + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def args2 = task.ext.args2 ?: '' + def file_list = files_in.collect { it.toString() } + + // | input | output | command1 | command2 | + // |-----------|------------|----------|----------| + // | gzipped | gzipped | cat | | + // | ungzipped | ungzipped | cat | | + // | gzipped | ungzipped | zcat | | + // | ungzipped | gzipped | cat | pigz | + + // Use input file ending as default + prefix = task.ext.prefix ?: "${meta.id}${file_list[0].substring(file_list[0].lastIndexOf('.'))}" + out_zip = prefix.endsWith('.gz') + in_zip = file_list[0].endsWith('.gz') + command1 = (in_zip && !out_zip) ? 'zcat' : 'cat' + command2 = (!in_zip && out_zip) ? "| pigz -c -p $task.cpus $args2" : '' + """ + $command1 \\ + $args \\ + ${file_list.join(' ')} \\ + $command2 \\ + > ${prefix} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) + END_VERSIONS + """ + + stub: + def file_list = files_in.collect { it.toString() } + prefix = task.ext.prefix ?: "${meta.id}${file_list[0].substring(file_list[0].lastIndexOf('.'))}" + """ + touch $prefix + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) + END_VERSIONS + """ +} diff --git a/modules/nf-core/cat/cat/meta.yml b/modules/nf-core/cat/cat/meta.yml new file mode 100644 index 00000000..00a8db0b --- /dev/null +++ b/modules/nf-core/cat/cat/meta.yml @@ -0,0 +1,36 @@ +name: cat_cat +description: A module for concatenation of gzipped or uncompressed files +keywords: + - concatenate + - gzip + - cat +tools: + - cat: + description: Just concatenation + documentation: https://man7.org/linux/man-pages/man1/cat.1.html + licence: ["GPL-3.0-or-later"] +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - files_in: + type: file + description: List of compressed / uncompressed files + pattern: "*" +output: + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + - file_out: + type: file + description: Concatenated file. Will be gzipped if file_out ends with ".gz" + pattern: "${file_out}" +authors: + - "@erikrikarddaniel" + - "@FriederikeHanssen" +maintainers: + - "@erikrikarddaniel" + - "@FriederikeHanssen" diff --git a/modules/nf-core/cat/cat/tests/main.nf.test b/modules/nf-core/cat/cat/tests/main.nf.test new file mode 100644 index 00000000..5766daaf --- /dev/null +++ b/modules/nf-core/cat/cat/tests/main.nf.test @@ -0,0 +1,153 @@ +nextflow_process { + + name "Test Process CAT_CAT" + script "../main.nf" + process "CAT_CAT" + tag "modules" + tag "modules_nfcore" + tag "cat" + tag "cat/cat" + + test("test_cat_unzipped_unzipped") { + when { + params { + outdir = "${outputDir}" + } + process { + """ + input[0] = + [ + [ id:'test', single_end:true ], + [ + file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), + file(params.test_data['sarscov2']['genome']['genome_sizes'], checkIfExists: true) + ] + ] + """ + } + } + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + + test("test_cat_zipped_zipped") { + when { + params { + outdir = "${outputDir}" + } + process { + """ + input[0] = + [ + [ id:'test', single_end:true ], + [ + file(params.test_data['sarscov2']['genome']['genome_gff3_gz'], checkIfExists: true), + file(params.test_data['sarscov2']['genome']['contigs_genome_maf_gz'], checkIfExists: true) + ] + ] + """ + } + } + then { + def lines = path(process.out.file_out.get(0).get(1)).linesGzip + assertAll( + { assert process.success }, + { assert snapshot(lines[0..5]).match("test_cat_zipped_zipped_lines") }, + { assert snapshot(lines.size()).match("test_cat_zipped_zipped_size")} + ) + } + } + + test("test_cat_zipped_unzipped") { + config './nextflow_zipped_unzipped.config' + + when { + params { + outdir = "${outputDir}" + } + process { + """ + input[0] = + [ + [ id:'test', single_end:true ], + [ + file(params.test_data['sarscov2']['genome']['genome_gff3_gz'], checkIfExists: true), + file(params.test_data['sarscov2']['genome']['contigs_genome_maf_gz'], checkIfExists: true) + ] + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("test_cat_unzipped_zipped") { + config './nextflow_unzipped_zipped.config' + when { + params { + outdir = "${outputDir}" + } + process { + """ + input[0] = + [ + [ id:'test', single_end:true ], + [ + file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), + file(params.test_data['sarscov2']['genome']['genome_sizes'], checkIfExists: true) + ] + ] + """ + } + } + then { + def lines = path(process.out.file_out.get(0).get(1)).linesGzip + assertAll( + { assert process.success }, + { assert snapshot(lines[0..5]).match("test_cat_unzipped_zipped_lines") }, + { assert snapshot(lines.size()).match("test_cat_unzipped_zipped_size")} + ) + } + } + + test("test_cat_one_file_unzipped_zipped") { + config './nextflow_unzipped_zipped.config' + when { + params { + outdir = "${outputDir}" + } + process { + """ + input[0] = + [ + [ id:'test', single_end:true ], + [ + file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) + ] + ] + """ + } + } + then { + def lines = path(process.out.file_out.get(0).get(1)).linesGzip + assertAll( + { assert process.success }, + { assert snapshot(lines[0..5]).match("test_cat_one_file_unzipped_zipped_lines") }, + { assert snapshot(lines.size()).match("test_cat_one_file_unzipped_zipped_size")} + ) + } + } +} + diff --git a/modules/nf-core/cat/cat/tests/main.nf.test.snap b/modules/nf-core/cat/cat/tests/main.nf.test.snap new file mode 100644 index 00000000..423571ba --- /dev/null +++ b/modules/nf-core/cat/cat/tests/main.nf.test.snap @@ -0,0 +1,121 @@ +{ + "test_cat_unzipped_zipped_size": { + "content": [ + 375 + ], + "timestamp": "2023-10-16T14:33:08.049445686" + }, + "test_cat_unzipped_unzipped": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": true + }, + "test.fasta:md5,f44b33a0e441ad58b2d3700270e2dbe2" + ] + ], + "1": [ + "versions.yml:md5,115ed6177ebcff24eb99d503fa5ef894" + ], + "file_out": [ + [ + { + "id": "test", + "single_end": true + }, + "test.fasta:md5,f44b33a0e441ad58b2d3700270e2dbe2" + ] + ], + "versions": [ + "versions.yml:md5,115ed6177ebcff24eb99d503fa5ef894" + ] + } + ], + "timestamp": "2023-10-16T14:32:18.500464399" + }, + "test_cat_zipped_unzipped": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": true + }, + "cat.txt:md5,c439d3b60e7bc03e8802a451a0d9a5d9" + ] + ], + "1": [ + "versions.yml:md5,115ed6177ebcff24eb99d503fa5ef894" + ], + "file_out": [ + [ + { + "id": "test", + "single_end": true + }, + "cat.txt:md5,c439d3b60e7bc03e8802a451a0d9a5d9" + ] + ], + "versions": [ + "versions.yml:md5,115ed6177ebcff24eb99d503fa5ef894" + ] + } + ], + "timestamp": "2023-10-16T14:32:49.642741302" + }, + "test_cat_zipped_zipped_lines": { + "content": [ + [ + "MT192765.1\tGenbank\ttranscript\t259\t29667\t.\t+\t.\tID=unknown_transcript_1;geneID=orf1ab;gene_name=orf1ab", + "MT192765.1\tGenbank\tgene\t259\t21548\t.\t+\t.\tParent=unknown_transcript_1", + "MT192765.1\tGenbank\tCDS\t259\t13461\t.\t+\t0\tParent=unknown_transcript_1;exception=\"ribosomal slippage\";gbkey=CDS;gene=orf1ab;note=\"pp1ab;translated=by -1 ribosomal frameshift\";product=\"orf1ab polyprotein\";protein_id=QIK50426.1", + "MT192765.1\tGenbank\tCDS\t13461\t21548\t.\t+\t0\tParent=unknown_transcript_1;exception=\"ribosomal slippage\";gbkey=CDS;gene=orf1ab;note=\"pp1ab;translated=by -1 ribosomal frameshift\";product=\"orf1ab polyprotein\";protein_id=QIK50426.1", + "MT192765.1\tGenbank\tCDS\t21556\t25377\t.\t+\t0\tParent=unknown_transcript_1;gbkey=CDS;gene=S;note=\"structural protein\";product=\"surface glycoprotein\";protein_id=QIK50427.1", + "MT192765.1\tGenbank\tgene\t21556\t25377\t.\t+\t.\tParent=unknown_transcript_1" + ] + ], + "timestamp": "2023-10-16T14:32:33.629048645" + }, + "test_cat_unzipped_zipped_lines": { + "content": [ + [ + ">MT192765.1 Severe acute respiratory syndrome coronavirus 2 isolate SARS-CoV-2/human/USA/PC00101P/2020, complete genome", + "GTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGT", + "GTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAG", + "TAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTTTGTCCGG", + "GTGTGACCGAAAGGTAAGATGGAGAGCCTTGTCCCTGGTTTCAACGAGAAAACACACGTCCAACTCAGTTTGCCTGTTTT", + "ACAGGTTCGCGACGTGCTCGTACGTGGCTTTGGAGACTCCGTGGAGGAGGTCTTATCAGAGGCACGTCAACATCTTAAAG" + ] + ], + "timestamp": "2023-10-16T14:33:08.038830506" + }, + "test_cat_one_file_unzipped_zipped_lines": { + "content": [ + [ + ">MT192765.1 Severe acute respiratory syndrome coronavirus 2 isolate SARS-CoV-2/human/USA/PC00101P/2020, complete genome", + "GTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGT", + "GTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAG", + "TAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTTTGTCCGG", + "GTGTGACCGAAAGGTAAGATGGAGAGCCTTGTCCCTGGTTTCAACGAGAAAACACACGTCCAACTCAGTTTGCCTGTTTT", + "ACAGGTTCGCGACGTGCTCGTACGTGGCTTTGGAGACTCCGTGGAGGAGGTCTTATCAGAGGCACGTCAACATCTTAAAG" + ] + ], + "timestamp": "2023-10-16T14:33:21.39642399" + }, + "test_cat_zipped_zipped_size": { + "content": [ + 78 + ], + "timestamp": "2023-10-16T14:32:33.641869244" + }, + "test_cat_one_file_unzipped_zipped_size": { + "content": [ + 374 + ], + "timestamp": "2023-10-16T14:33:21.4094373" + } +} \ No newline at end of file diff --git a/modules/nf-core/cat/cat/tests/nextflow_unzipped_zipped.config b/modules/nf-core/cat/cat/tests/nextflow_unzipped_zipped.config new file mode 100644 index 00000000..ec26b0fd --- /dev/null +++ b/modules/nf-core/cat/cat/tests/nextflow_unzipped_zipped.config @@ -0,0 +1,6 @@ + +process { + withName: CAT_CAT { + ext.prefix = 'cat.txt.gz' + } +} diff --git a/modules/nf-core/cat/cat/tests/nextflow_zipped_unzipped.config b/modules/nf-core/cat/cat/tests/nextflow_zipped_unzipped.config new file mode 100644 index 00000000..fbc79783 --- /dev/null +++ b/modules/nf-core/cat/cat/tests/nextflow_zipped_unzipped.config @@ -0,0 +1,8 @@ + +process { + + withName: CAT_CAT { + ext.prefix = 'cat.txt' + } + +} diff --git a/modules/nf-core/cat/cat/tests/tags.yml b/modules/nf-core/cat/cat/tests/tags.yml new file mode 100644 index 00000000..37b578f5 --- /dev/null +++ b/modules/nf-core/cat/cat/tests/tags.yml @@ -0,0 +1,2 @@ +cat/cat: + - modules/nf-core/cat/cat/** diff --git a/nextflow.config b/nextflow.config index 66314e9a..90fde95a 100644 --- a/nextflow.config +++ b/nextflow.config @@ -13,6 +13,7 @@ params { input = null single_end = false assembly_input = null + nf_core_pipeline = null // short read preprocessing options skip_clipping = false diff --git a/subworkflows/local/samplesheet_creation.nf b/subworkflows/local/samplesheet_creation.nf new file mode 100644 index 00000000..d9edb182 --- /dev/null +++ b/subworkflows/local/samplesheet_creation.nf @@ -0,0 +1,115 @@ +include { CAT_CAT } from '../../modules/nf-core/cat/cat/main' +include { MAG_TO_SAMPLESHEET } from '../../modules/local/mag_to_samplesheet' +include { MAG_MERGE_SAMPLESHEET } from '../../modules/local/mag_merge_samplesheet' + +workflow SAMPLESHEET_CREATION { + take: + short_reads //channel: [val(meta), path(fastq_1), path(fastq_2)] + assemblies //channel: [val(meta), path(fasta)] + main: + ch_versions = Channel.empty() + + // combine assemblies by sample/group if multiple assembly methods were used + ch_assemblies = assemblies + .map { + meta, fasta -> + def meta_new = meta.subMap('id') + [ meta_new, fasta ] + } + .groupTuple() + + // + // MODULE: Combine all assemblies from a sample into one FastA file + // + ch_combined_assemblies = CAT_CAT ( ch_assemblies ).file_out + ch_versions = ch_versions.mix( CAT_CAT.out.versions ) + + // if no coassembly, join FastQ and FastA by ID + if ( !params.coassemble_group ){ + ch_combined_assemblies_remap = ch_combined_assemblies + .map { + meta, fasta -> + def id = meta.id + + return [ id, fasta ] + } + short_reads + .map { + meta, fastq -> + def id = meta.id + def group = meta.group + def single_end = meta.single_end + + return [ id, group, single_end, fastq ] + }.join ( ch_combined_assemblies_remap ) + .map { + id, group, single_end, fastq, fasta -> + def reads = fastq instanceof List ? fastq.flatten() : [ fastq ] + def meta = [:] + + meta.id = id + meta.group = group + meta.single_end = single_end + meta.fastq_1 = reads[0] ? reads[0] : '' + meta.fastq_2 = reads[1] && !meta.single_end ? reads[1] : '' + meta.fasta = fasta ? fasta : '' + + return meta + } + .set { ch_mag_metadata } + } else { + // if coassembly was used, join FastQ and FastA by group + ch_combined_assemblies_remap = ch_combined_assemblies + .map { + meta, fasta -> + def group = meta.id.split('-') + + return [ group[1], fasta ] + } + short_reads + .map { + meta, fastq -> + def id = meta.id + def group = meta.group + def single_end = meta.single_end + + return [ group, id, single_end, fastq ] + } + .join ( ch_combined_assemblies_remap ) + .map { + id, group, single_end, fastq, fasta -> + def reads = fastq instanceof List ? fastq.flatten() : [ fastq ] + def meta = [:] + + meta.id = id + meta.group = group + meta.single_end = single_end + meta.fastq_1 = reads[0] ? reads[0] : '' + meta.fastq_2 = reads[1] && !meta.single_end ? reads[1] : '' + meta.fasta = fasta ? fasta : '' + + return meta + } + .set { ch_mag_metadata } + } + + // + // MODULE: Stage FastQ/FastA files generated by nf-core/mag together and auto-create a samplesheet + // + MAG_TO_SAMPLESHEET ( + ch_mag_metadata, + params.nf_core_pipeline ?: '' + ) + + // + // MODULE: Create a merged samplesheet across all samples for the pipeline + // + MAG_MERGE_SAMPLESHEET ( + MAG_TO_SAMPLESHEET.out.samplesheet.collect{it[1]} + ) + ch_versions = ch_versions.mix( MAG_MERGE_SAMPLESHEET.out.versions ) + + emit: + samplesheet = ch_assemblies + versions = ch_versions // channel: [ versions.yml ] +} diff --git a/workflows/mag.nf b/workflows/mag.nf index 160928d2..ec69017d 100644 --- a/workflows/mag.nf +++ b/workflows/mag.nf @@ -98,6 +98,7 @@ include { GTDBTK } from '../subworkflows/local/gtdbtk' include { ANCIENT_DNA_ASSEMBLY_VALIDATION } from '../subworkflows/local/ancient_dna' include { DOMAIN_CLASSIFICATION } from '../subworkflows/local/domain_classification' include { DEPTHS } from '../subworkflows/local/depths' +include { SAMPLESHEET_CREATION } from '../subworkflows/local/samplesheet_creation' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1013,6 +1014,12 @@ workflow MAG { } } + // + // SUBWORKFLOW: Auto-create samplesheets for downstream nf-core pipelines + // + ch_samplesheet = SAMPLESHEET_CREATION ( ch_short_reads_assembly, ch_assemblies ).samplesheet + ch_versions = ch_versions.mix( SAMPLESHEET_CREATION.out.versions ) + CUSTOM_DUMPSOFTWAREVERSIONS ( ch_versions.unique().collectFile(name: 'collated_versions.yml') ) From 43491d863d8f30787b1139ccd7806afbcce4933d Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Mon, 20 Nov 2023 22:30:21 +0000 Subject: [PATCH 02/49] Created tests for samplesheet creation --- .github/workflows/ci.yml | 1 + conf/test.config | 14 ----------- conf/test_samplesheet.config | 47 ++++++++++++++++++++++++++++++++++++ nextflow.config | 1 + 4 files changed, 49 insertions(+), 14 deletions(-) create mode 100644 conf/test_samplesheet.config diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3aaa6f3e..f6c12464 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,7 @@ jobs: test_adapterremoval, test_binrefinement, test_virus_identification, + test_samplesheet, ] steps: - name: Free some space diff --git a/conf/test.config b/conf/test.config index 3a5ea4f4..9c93278f 100644 --- a/conf/test.config +++ b/conf/test.config @@ -30,18 +30,4 @@ params { busco_clean = true skip_gtdbtk = true skip_concoct = true - - // For computational efficiency - nf_core_pipeline = 'phageannotator' - coassemble_group = false - skip_binning = true - skip_prokka = true - skip_spadeshybrid = true - skip_quast = true - skip_prodigal = true - skip_krona = true - skip_adapter_trimming = true - skip_metabat2 = true - skip_maxbin2 = true - skip_busco = true } diff --git a/conf/test_samplesheet.config b/conf/test_samplesheet.config new file mode 100644 index 00000000..3a5ea4f4 --- /dev/null +++ b/conf/test_samplesheet.config @@ -0,0 +1,47 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for running minimal tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Defines input files and everything required to run a fast and simple pipeline test. + + Use as follows: + nextflow run nf-core/mag -profile test, --outdir + +---------------------------------------------------------------------------------------- +*/ + +params { + config_profile_name = 'Test profile' + config_profile_description = 'Minimal test dataset to check pipeline function' + + // Limit resources so that this can run on GitHub Actions + max_cpus = 2 + max_memory = '6.GB' + max_time = '6.h' + + // Input data + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' + centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" + kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" + skip_krona = true + min_length_unbinned_contigs = 1 + max_unbinned_contigs = 2 + busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz" + busco_clean = true + skip_gtdbtk = true + skip_concoct = true + + // For computational efficiency + nf_core_pipeline = 'phageannotator' + coassemble_group = false + skip_binning = true + skip_prokka = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + skip_krona = true + skip_adapter_trimming = true + skip_metabat2 = true + skip_maxbin2 = true + skip_busco = true +} diff --git a/nextflow.config b/nextflow.config index 90fde95a..67611d98 100644 --- a/nextflow.config +++ b/nextflow.config @@ -314,6 +314,7 @@ profiles { test_bbnorm { includeConfig 'conf/test_bbnorm.config' } test_nothing { includeConfig 'conf/test_nothing.config' } test_virus_identification { includeConfig 'conf/test_virus_identification.config' } + test_samplesheet { includeConfig 'conf/test_samplesheet.config' } } // Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile From e39111e9d4adc0c85c840cf6a21e6c09fa6fa6eb Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Mon, 20 Nov 2023 22:41:31 +0000 Subject: [PATCH 03/49] Added nf-core-pipeline parameter to schema --- nextflow_schema.json | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 13f7e6bc..a747c7f7 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -52,6 +52,10 @@ "type": "string", "description": "MultiQC report title. Printed as page header, used for filename if not otherwise specified.", "fa_icon": "fas fa-file-signature" + }, + "nf_core_pipeline": { + "type": "string", + "description": "Create a samplesheet for the specified nf-core pipeline" } } }, @@ -530,7 +534,7 @@ }, "gtdbtk_min_completeness": { "type": "number", - "default": 50.0, + "default": 50, "description": "Min. bin completeness (in %) required to apply GTDB-tk classification.", "help_text": "Completeness assessed with BUSCO analysis (100% - %Missing). Must be greater than 0 (min. 0.01) to avoid GTDB-tk errors. If too low, GTDB-tk classification results can be impaired due to not enough marker genes!", "minimum": 0.01, @@ -538,7 +542,7 @@ }, "gtdbtk_max_contamination": { "type": "number", - "default": 10.0, + "default": 10, "description": "Max. bin contamination (in %) allowed to apply GTDB-tk classification.", "help_text": "Contamination approximated based on BUSCO analysis (%Complete and duplicated). If too high, GTDB-tk classification results can be impaired due to contamination!", "minimum": 0, @@ -546,7 +550,7 @@ }, "gtdbtk_min_perc_aa": { "type": "number", - "default": 10.0, + "default": 10, "description": "Min. fraction of AA (in %) in the MSA for bins to be kept.", "minimum": 0, "maximum": 100 @@ -560,7 +564,7 @@ }, "gtdbtk_pplacer_cpus": { "type": "number", - "default": 1.0, + "default": 1, "description": "Number of CPUs used for the by GTDB-Tk run tool pplacer.", "help_text": "A low number of CPUs helps to reduce the memory required/reported by GTDB-Tk. See also the [GTDB-Tk documentation](https://ecogenomics.github.io/GTDBTk/faq.html#gtdb-tk-reaches-the-memory-limit-pplacer-crashes)." }, From 69c684e77355a233505d7fb02f97ce9f06363296 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Mon, 20 Nov 2023 22:44:37 +0000 Subject: [PATCH 04/49] Changed input to samplesheet creation --- workflows/mag.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/mag.nf b/workflows/mag.nf index ec69017d..6cd17d01 100644 --- a/workflows/mag.nf +++ b/workflows/mag.nf @@ -1017,7 +1017,7 @@ workflow MAG { // // SUBWORKFLOW: Auto-create samplesheets for downstream nf-core pipelines // - ch_samplesheet = SAMPLESHEET_CREATION ( ch_short_reads_assembly, ch_assemblies ).samplesheet + ch_samplesheet = SAMPLESHEET_CREATION ( ch_short_reads, ch_assemblies ).samplesheet ch_versions = ch_versions.mix( SAMPLESHEET_CREATION.out.versions ) CUSTOM_DUMPSOFTWAREVERSIONS ( From bd065320dc61f2da5873186498ddd0872db6d633 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Mon, 20 Nov 2023 22:59:48 +0000 Subject: [PATCH 05/49] Conditional running of samplesheet creation --- workflows/mag.nf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/workflows/mag.nf b/workflows/mag.nf index 6cd17d01..d1b9eb0e 100644 --- a/workflows/mag.nf +++ b/workflows/mag.nf @@ -1017,8 +1017,10 @@ workflow MAG { // // SUBWORKFLOW: Auto-create samplesheets for downstream nf-core pipelines // - ch_samplesheet = SAMPLESHEET_CREATION ( ch_short_reads, ch_assemblies ).samplesheet - ch_versions = ch_versions.mix( SAMPLESHEET_CREATION.out.versions ) + if ( params.nf_core_pipeline ) { + ch_samplesheet = SAMPLESHEET_CREATION ( ch_short_reads, ch_assemblies ).samplesheet + ch_versions = ch_versions.mix( SAMPLESHEET_CREATION.out.versions ) + } CUSTOM_DUMPSOFTWAREVERSIONS ( ch_versions.unique().collectFile(name: 'collated_versions.yml') From a564425b5ec04d763b2ea37edcc04d2e432115d1 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Wed, 29 Nov 2023 18:25:51 +0000 Subject: [PATCH 06/49] Updated docs and added pipeline to output file name --- CHANGELOG.md | 2 ++ README.md | 1 + conf/modules.config | 1 + docs/output.md | 15 +++++++++++++++ modules/local/mag_merge_samplesheet.nf | 7 ++++--- nextflow_schema.json | 12 +++++++----- 6 files changed, 30 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc6eed0d..65a36b27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Added` +- [#543](https://github.com/nf-core/mag/pull/543) - Automatic samplesheet generation for nf-core/phageannotator (@CarsonJM) + ### `Changed` ### `Fixed` diff --git a/README.md b/README.md index 8a4ba8d2..76ea9508 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ The pipeline then: - Performs ancient DNA validation and repair with [pyDamage](https://github.com/maxibor/pydamage) and [freebayes](https://github.com/freebayes/freebayes) - optionally refines bins with [DAS Tool](https://github.com/cmks/DAS_Tool) - assigns taxonomy to bins using [GTDB-Tk](https://github.com/Ecogenomics/GTDBTk) and/or [CAT](https://github.com/dutilh/CAT) and optionally identifies viruses in assemblies using [geNomad](https://github.com/apcamargo/genomad), or Eukaryotes with [Tiara](https://github.com/ibe-uw/tiara) +- generates a samplesheet that can be used as input for other nf-core pipelines. Currently, [phageannotator](https://github.com/nf-core/phageannotator) is supported. Furthermore, the pipeline creates various reports in the results directory specified, including a [MultiQC](https://multiqc.info/) report summarizing some of the findings and software versions. diff --git a/conf/modules.config b/conf/modules.config index 28d5afc5..e98280f5 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -749,6 +749,7 @@ process { } withName: 'MAG_MERGE_SAMPLESHEET' { + ext.prefix = "${params.nf_core_pipeline}" publishDir = [ path: { "${params.outdir}/samplesheet" }, mode: params.publish_dir_mode, diff --git a/docs/output.md b/docs/output.md index 88aba227..681dc371 100644 --- a/docs/output.md +++ b/docs/output.md @@ -21,6 +21,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d - [Genome annotation of binned genomes](#genome-annotation-of-binned-genomes) - [Additional summary for binned genomes](#additional-summary-for-binned-genomes) - [Ancient DNA](#ancient-dna) +- [Samplesheet generation](#sampleseet-generation) - [MultiQC](#multiqc) - aggregate report, describing results of the whole pipeline - [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution @@ -706,6 +707,20 @@ Because of aDNA damage, _de novo_ assemblers sometimes struggle to call a correc +### Samplesheet generation + +
+Output files + +- `samplesheet/` + - `[nf_core_pipeline].csv`: a samplesheet in CSV format that can be directly used as input for the specified nf-core pipeline + +
+ +Currently, samplesheets for the following nf-core pipelines can be automatically generated: + +- [phageannotator](https://github.com/nf-core/phageannotator): a pipeline for identifying, annotation, and quantifying phage sequences in (meta)-genomic sequences. + ### MultiQC
diff --git a/modules/local/mag_merge_samplesheet.nf b/modules/local/mag_merge_samplesheet.nf index 5ad7d01a..34641de6 100644 --- a/modules/local/mag_merge_samplesheet.nf +++ b/modules/local/mag_merge_samplesheet.nf @@ -9,14 +9,15 @@ process MAG_MERGE_SAMPLESHEET { path ('samplesheets/*') output: - path "samplesheet.csv", emit: samplesheet + path "*_samplesheet.csv", emit: samplesheet path "versions.yml" , emit: versions script: + def prefix = task.ext.prefix ?: "${meta.id}" """ - head -n 1 `ls ./samplesheets/* | head -n 1` > samplesheet.csv + head -n 1 `ls ./samplesheets/* | head -n 1` > ${prefix}_samplesheet.csv for fileid in `ls ./samplesheets/*`; do - awk 'NR>1' \$fileid >> samplesheet.csv + awk 'NR>1' \$fileid >> ${prefix}_samplesheet.csv done cat <<-END_VERSIONS > versions.yml diff --git a/nextflow_schema.json b/nextflow_schema.json index a747c7f7..a50c9ae3 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -55,7 +55,9 @@ }, "nf_core_pipeline": { "type": "string", - "description": "Create a samplesheet for the specified nf-core pipeline" + "description": "Create a samplesheet for the specified nf-core pipeline", + "help_text": "Automatically generate a samplesheet in CSV format that can be directly used as input for the specified nf-core pipeline.", + "enum": ["phageannotator"] } } }, @@ -534,7 +536,7 @@ }, "gtdbtk_min_completeness": { "type": "number", - "default": 50, + "default": 50.0, "description": "Min. bin completeness (in %) required to apply GTDB-tk classification.", "help_text": "Completeness assessed with BUSCO analysis (100% - %Missing). Must be greater than 0 (min. 0.01) to avoid GTDB-tk errors. If too low, GTDB-tk classification results can be impaired due to not enough marker genes!", "minimum": 0.01, @@ -542,7 +544,7 @@ }, "gtdbtk_max_contamination": { "type": "number", - "default": 10, + "default": 10.0, "description": "Max. bin contamination (in %) allowed to apply GTDB-tk classification.", "help_text": "Contamination approximated based on BUSCO analysis (%Complete and duplicated). If too high, GTDB-tk classification results can be impaired due to contamination!", "minimum": 0, @@ -550,7 +552,7 @@ }, "gtdbtk_min_perc_aa": { "type": "number", - "default": 10, + "default": 10.0, "description": "Min. fraction of AA (in %) in the MSA for bins to be kept.", "minimum": 0, "maximum": 100 @@ -564,7 +566,7 @@ }, "gtdbtk_pplacer_cpus": { "type": "number", - "default": 1, + "default": 1.0, "description": "Number of CPUs used for the by GTDB-Tk run tool pplacer.", "help_text": "A low number of CPUs helps to reduce the memory required/reported by GTDB-Tk. See also the [GTDB-Tk documentation](https://ecogenomics.github.io/GTDBTk/faq.html#gtdb-tk-reaches-the-memory-limit-pplacer-crashes)." }, From 11bc7219cc1642bb9f69ebdd5bd1ebcd05404f66 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Fri, 8 Dec 2023 12:10:53 -0800 Subject: [PATCH 07/49] Started structuring mag for nf-tests --- .gitignore | 2 + conf/test.config | 10 ++-- conf/test_adapterremoval.config | 10 ++-- conf/test_nothing.config | 7 +-- conf/test_virus_identification.config | 11 ++-- lib/NftestUtils.groovy | 38 ++++++++++++++ modules/nf-core/genomad/download/main.nf | 2 + nf-test.config | 16 ++++++ tests/main.nf.test | 54 ++++++++++++++++++++ tests/test_adapterremoval.nf.test | 45 ++++++++++++++++ tests/test_nothing.nf.test | 30 +++++++++++ tests/test_virus_identification.nf.test | 32 ++++++++++++ tests/test_virus_identification.nf.test.snap | 34 ++++++++++++ 13 files changed, 279 insertions(+), 12 deletions(-) create mode 100644 lib/NftestUtils.groovy create mode 100644 nf-test.config create mode 100644 tests/main.nf.test create mode 100644 tests/test_adapterremoval.nf.test create mode 100644 tests/test_nothing.nf.test create mode 100644 tests/test_virus_identification.nf.test create mode 100644 tests/test_virus_identification.nf.test.snap diff --git a/.gitignore b/.gitignore index 5124c9ac..8d38c7c0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ results/ testing/ testing* *.pyc +.nf-tests/ +.nf-test.log diff --git a/conf/test.config b/conf/test.config index 9c93278f..c2389ce3 100644 --- a/conf/test.config +++ b/conf/test.config @@ -15,9 +15,9 @@ params { config_profile_description = 'Minimal test dataset to check pipeline function' // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' + //max_cpus = 2 + //max_memory = '6.GB' + //max_time = '6.h' // Input data input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' @@ -30,4 +30,8 @@ params { busco_clean = true skip_gtdbtk = true skip_concoct = true + megahit_fix_cpu_1 = true + spades_fix_cpus = 4 + spadeshybrid_fix_cpus = 4 + metabat_rng_seed = 1 } diff --git a/conf/test_adapterremoval.config b/conf/test_adapterremoval.config index d8bd581a..f9f759cd 100644 --- a/conf/test_adapterremoval.config +++ b/conf/test_adapterremoval.config @@ -15,9 +15,9 @@ params { config_profile_description = 'Minimal test dataset to check pipeline function with AdapterRemoval data and domain classification.' // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' + //max_cpus = 2 + //max_memory = '6.GB' + //max_time = '6.h' // Input data input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.euk.csv' @@ -32,4 +32,8 @@ params { clip_tool = 'adapterremoval' skip_concoct = true bin_domain_classification = true + megahit_fix_cpu_1 = true + spades_fix_cpus = 4 + spadeshybrid_fix_cpus = 4 + metabat_rng_seed = 1 } diff --git a/conf/test_nothing.config b/conf/test_nothing.config index 53df219f..eb7201db 100644 --- a/conf/test_nothing.config +++ b/conf/test_nothing.config @@ -16,14 +16,15 @@ params { config_profile_description = 'Minimal test dataset to check pipeline function' // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' + //max_cpus = 2 + //max_memory = '6.GB' + //max_time = '6.h' // Input data input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' centrifuge_db = null kraken2_db = null + keep_phix = true skip_krona = true skip_clipping = true skip_adapter_trimming = true diff --git a/conf/test_virus_identification.config b/conf/test_virus_identification.config index e15fab7d..0b80c9a3 100644 --- a/conf/test_virus_identification.config +++ b/conf/test_virus_identification.config @@ -15,9 +15,9 @@ params { config_profile_description = 'Minimal test dataset to check pipeline function virus identification' // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' + //max_cpus = 2 + //max_memory = '6.GB' + //max_time = '6.h' // Input data input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' @@ -26,6 +26,11 @@ params { // For computational efficiency reads_minlength = 150 + megahit_fix_cpu_1 = true + centrifuge_db = null + kraken2_db = null + keep_phix = true + skip_clipping = true coassemble_group = true skip_gtdbtk = true skip_binning = true diff --git a/lib/NftestUtils.groovy b/lib/NftestUtils.groovy new file mode 100644 index 00000000..ce6fc8b4 --- /dev/null +++ b/lib/NftestUtils.groovy @@ -0,0 +1,38 @@ +// Helper functions for pipeline tests + +class NftestUtils { + + // Function to remove Nextflow version from software_versions.yml + public static String removeNextflowVersion(outputDir) { + def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml + if (softwareVersions.containsKey("Workflow")) { + softwareVersions.Workflow.remove("Nextflow") + } + return softwareVersions + } + + // Function to filter lines from a file and return a new file + public static File filterLines(String inFilePath, int linesToSkip) { + if (linesToSkip >= 0) { + File inputFile = new File(inFilePath) + File outputFile = new File(inFilePath + ".filtered") + def lineCount = 0 + inputFile.eachLine { line -> + lineCount++ + if (lineCount > linesToSkip) { + outputFile.append(line + '\n') + } + } + return outputFile + } else { + File inputFile = new File(inFilePath) + File outputFile = new File(inFilePath + ".filtered") + def lines = inputFile.readLines() + def totalLines = lines.size() + lines.take(totalLines + linesToSkip).each { line -> + outputFile.append(line + '\n') + } + return outputFile + } + } +} diff --git a/modules/nf-core/genomad/download/main.nf b/modules/nf-core/genomad/download/main.nf index a2ac6ecb..632d494e 100644 --- a/modules/nf-core/genomad/download/main.nf +++ b/modules/nf-core/genomad/download/main.nf @@ -16,6 +16,8 @@ process GENOMAD_DOWNLOAD { script: def args = task.ext.args ?: '' """ + https_proxy=http://klone-dip1-A-ib:3128 + export https_proxy genomad \\ download-database . diff --git a/nf-test.config b/nf-test.config new file mode 100644 index 00000000..2f9c518d --- /dev/null +++ b/nf-test.config @@ -0,0 +1,16 @@ +config { + // location for all nf-tests + testsDir "." + + // nf-test directory including temporary files for each test + workDir ".nf-tests" + + // location of library folder that is added automatically to the classpath + libDir "lib/" + + // location of an optional nextflow.config file specific for executing tests + configFile "nextflow.config" + + // run all test with the defined docker profile from the main nextflow.config + profile "" +} diff --git a/tests/main.nf.test b/tests/main.nf.test new file mode 100644 index 00000000..405cc4a5 --- /dev/null +++ b/tests/main.nf.test @@ -0,0 +1,54 @@ +nextflow_pipeline { + + name "Test pipeline: NFCORE_MAG" + script "main.nf" + tag "pipeline" + tag "nfcore_mag" + tag "nfcore_mag_test" + + test("Parameters: default test") { + when { + params { + outdir = "$outputDir" + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + path("${params.outdir}/Annotation/Prodigal/"), + path("${params.outdir}/Annotation/Prokka/MEGAHIT/MEGAHIT-MaxBin2-test_minigut.001/MEGAHIT-MaxBin2-test_minigut.001.tsv"), + path("${params.outdir}/Annotation/Prokka/MEGAHIT/MEGAHIT-MetaBAT2-test_minigut_sample2.unbinned.1/MEGAHIT-MetaBAT2-test_minigut_sample2.unbinned.1.tsv"), + path("${params.outdir}/Annotation/Prokka/MEGAHIT/MEGAHIT-MetaBAT2-test_minigut.1/MEGAHIT-MetaBAT2-test_minigut.1.tsv"), + path("${params.outdir}/Annotation/Prokka/MEGAHIT/MEGAHIT-MetaBAT2-test_minigut.unbinned.1/MEGAHIT-MetaBAT2-test_minigut.unbinned.1.tsv"), + path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MaxBin2-test_minigut_sample2.001/SPAdes-MaxBin2-test_minigut_sample2.001.tsv"), + path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MaxBin2-test_minigut_sample2.noclass.1/SPAdes-MaxBin2-test_minigut_sample2.noclass.1.tsv"), + path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MaxBin2-test_minigut.001/SPAdes-MaxBin2-test_minigut.001.tsv"), + path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MetaBAT2-test_minigut_sample2.unbinned.1/SPAdes-MetaBAT2-test_minigut_sample2.unbinned.1.tsv"), + path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MetaBAT2-test_minigut.1/SPAdes-MetaBAT2-test_minigut.1.tsv"), + path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MetaBAT2-test_minigut.unbinned.1/SPAdes-MetaBAT2-test_minigut.unbinned.1.tsv"), + path("${params.outdir}/Assembly/MEGAHIT/MEGAHIT-test_minigut_sample2.contigs.fa.gz"), + path("${params.outdir}/Assembly/MEGAHIT/MEGAHIT-test_minigut.contigs.fa.gz"), + path("${params.outdir}/Assembly/SPAdes/SPAdes-test_minigut_sample2_contigs.fasta.gz"), + path("${params.outdir}/Assembly/SPAdes/SPAdes-test_minigut_contigs.fasta.gz"), + //path("${params.outdir}/GenomeBinning/depths/bins/bin_depths_summary.tsv"), + path("${params.outdir}/GenomeBinning/depths/contigs/"), + path("${params.outdir}/GenomeBinning/MaxBin2/"), + path("${params.outdir}/GenomeBinning/MetaBAT2/"), + //path("${params.outdir}/GenomeBinning/QC/busco_summary.tsv"), + //path("${params.outdir}/GenomeBinning/QC/quast_summary.tsv"), + //path("${params.outdir}/GenomeBinning/bin_summary.tsv"), + path("${params.outdir}/QC_shortreads/fastp/test_minigut/test_minigut_run1_fastp.fastp.json"), + path("${params.outdir}/QC_shortreads/fastp/test_minigut/test_minigut_run2_fastp.fastp.json"), + path("${params.outdir}/QC_shortreads/fastp/test_minigut_sample2/test_minigut_sample2_run0_fastp.fastp.json"), + path("${params.outdir}/QC_shortreads/remove_phix/test_minigut_run1_phix_removed.bowtie2.log"), + path("${params.outdir}/QC_shortreads/remove_phix/test_minigut_run2_phix_removed.bowtie2.log"), + path("${params.outdir}/QC_shortreads/remove_phix/test_minigut_sample2_run0_phix_removed.bowtie2.log"), + path("${params.outdir}/Taxonomy/") + ).match() }, + { assert new File("$outputDir/multiqc/multiqc_report.html").exists() } + ) + } + } +} diff --git a/tests/test_adapterremoval.nf.test b/tests/test_adapterremoval.nf.test new file mode 100644 index 00000000..69b6ade0 --- /dev/null +++ b/tests/test_adapterremoval.nf.test @@ -0,0 +1,45 @@ +nextflow_pipeline { + + name "Test pipeline: NFCORE_MAG" + script "main.nf" + tag "pipeline" + tag "nfcore_mag" + tag "nfcore_mag_test_adapterremoval" + + test("Parameters: clip_tool = 'adapterremoval'") { + when { + params { + outdir = "$outputDir" + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + path("${params.outdir}/Annotation/MetaEuk/"), + path("${params.outdir}/Annotation/Prodigal/"), + path("${params.outdir}/Annotation/Prokka/MEGAHIT/MEGAHIT-MaxBin2-test_minigut_euk.001/MEGAHIT-MaxBin2-test_minigut_euk.001.tsv"), + path("${params.outdir}/Annotation/Prokka/MEGAHIT/MEGAHIT-MetaBAT2-test_minigut_euk.2/MEGAHIT-MetaBAT2-test_minigut_euk.2.tsv"), + path("${params.outdir}/Annotation/Prokka/MEGAHIT/MEGAHIT-MetaBAT2-test_minigut_euk.unbinned.1/MEGAHIT-MetaBAT2-test_minigut_euk.unbinned.1.tsv"), + path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MaxBin2-test_minigut_euk.002/SPAdes-MaxBin2-test_minigut_euk.002.tsv"), + path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MetaBAT2-test_minigut_euk.2/SPAdes-MetaBAT2-test_minigut_euk.2.tsv"), + path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MetaBAT2-test_minigut_euk.unbinned.1/SPAdes-MetaBAT2-test_minigut_euk.unbinned.1.tsv"), + path("${params.outdir}/Assembly/MEGAHIT/MEGAHIT-test_minigut_euk.contigs.fa.gz"), + path("${params.outdir}/Assembly/MEGAHIT/MEGAHIT-test_minigut_sample2_euk.contigs.fa.gz"), + path("${params.outdir}/Assembly/SPAdes/SPAdes-test_minigut_euk_contigs.fasta.gz"), + path("${params.outdir}/Assembly/SPAdes/SPAdes-test_minigut_sample2_euk_contigs.fasta.gz"), + path("${params.outdir}/GenomeBinning/depths/contigs/"), + path("${params.outdir}/GenomeBinning/MaxBin2/"), + path("${params.outdir}/GenomeBinning/MetaBAT2/"), + path("${params.outdir}/QC_shortreads/adapterremoval/test_minigut_euk/test_minigut_euk_run0_ar2.settings"), + path("${params.outdir}/QC_shortreads/adapterremoval/test_minigut_sample2_euk/test_minigut_sample2_euk_run0_ar2.settings"), + path("${params.outdir}/QC_shortreads/remove_phix/test_minigut_euk_run0_phix_removed.bowtie2.log"), + path("${params.outdir}/QC_shortreads/remove_phix/test_minigut_sample2_euk_run0_phix_removed.bowtie2.log"), + path("${params.outdir}/Taxonomy/") + ).match() }, + { assert new File("$outputDir/multiqc/multiqc_report.html").exists() } + ) + } + } +} diff --git a/tests/test_nothing.nf.test b/tests/test_nothing.nf.test new file mode 100644 index 00000000..88fd806e --- /dev/null +++ b/tests/test_nothing.nf.test @@ -0,0 +1,30 @@ +nextflow_pipeline { + + name "Test pipeline: NFCORE_MAG" + script "main.nf" + tag "pipeline" + tag "nfcore_mag" + tag "nfcore_mag_test_nothing" + + test("Parameters: test_nothing") { + when { + params { + outdir = "$outputDir" + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + path("${params.outdir}/multiqc/multiqc_data/multiqc_fastqc.yaml"), + ).match() }, + { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_run0_raw_1_fastqc.html").exists() }, + { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_run0_raw_2_fastqc.html").exists() }, + { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_sample2_run0_raw_1_fastqc.html").exists() }, + { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_sample2_run0_raw_1_fastqc.html").exists() }, + { assert new File("$outputDir/multiqc/multiqc_report.html").exists() } + ) + } + } +} diff --git a/tests/test_virus_identification.nf.test b/tests/test_virus_identification.nf.test new file mode 100644 index 00000000..0204c116 --- /dev/null +++ b/tests/test_virus_identification.nf.test @@ -0,0 +1,32 @@ +nextflow_pipeline { + + name "Test pipeline: NFCORE_MAG" + script "main.nf" + tag "pipeline" + tag "nfcore_mag" + tag "nfcore_mag_test_virus_identification" + + test("Parameters: run_virus_identification = true") { + when { + params { + outdir = "$outputDir" + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + path("${params.outdir}/Assembly/MEGAHIT/MEGAHIT-group-0.contigs.fa.gz"), + path("${params.outdir}/multiqc/multiqc_data/multiqc_fastqc.yaml"), + path("${params.outdir}/VirusIdentification/"), + ).match() }, + { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_run0_raw_1_fastqc.html").exists() }, + { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_run0_raw_2_fastqc.html").exists() }, + { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_sample2_run0_raw_1_fastqc.html").exists() }, + { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_sample2_run0_raw_1_fastqc.html").exists() }, + { assert new File("$outputDir/multiqc/multiqc_report.html").exists() } + ) + } + } +} diff --git a/tests/test_virus_identification.nf.test.snap b/tests/test_virus_identification.nf.test.snap new file mode 100644 index 00000000..61cf6f2a --- /dev/null +++ b/tests/test_virus_identification.nf.test.snap @@ -0,0 +1,34 @@ +{ + "Parameters: run_virus_identification = true": { + "content": [ + "MEGAHIT-group-0.contigs.fa.gz:md5,da40ccf271166ccfc233e70adb0a23ae", + "multiqc_fastqc.yaml:md5,2ef2c5154a509b13d9bcfe8cc879d71b", + [ + [ + [ + [ + "MEGAHIT-group-0.contigs_aggregated_classification.tsv:md5,afcdcac7a8f02544c4888533e870c4e8" + ], + [ + "MEGAHIT-group-0.contigs_taxonomy.tsv:md5,08148a4884b69088a8955c3fd8860027" + ], + [ + "MEGAHIT-group-0.contigs_provirus.tsv:md5,93a6bca59b0bf7f57c0b9b60d2e57082" + ], + [ + "MEGAHIT-group-0.contigs_plasmid.fna:md5,b1105afa94c08cd95e643e1534b57cd1", + "MEGAHIT-group-0.contigs_plasmid_genes.tsv:md5,8af86c89056190cbbe326e50b19f6bea", + "MEGAHIT-group-0.contigs_plasmid_proteins.faa:md5,bdd43fd82f22633079ee71dfc47ae0a1", + "MEGAHIT-group-0.contigs_plasmid_summary.tsv:md5,493372e1e756edb8c63318de474dacd4", + "MEGAHIT-group-0.contigs_virus.fna:md5,228873166621bd0c304938dd868757e2", + "MEGAHIT-group-0.contigs_virus_genes.tsv:md5,5bddb3bfe0a241796b26326478f06255", + "MEGAHIT-group-0.contigs_virus_proteins.faa:md5,4617c2217be860dbda882c6ae66a264e", + "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,4812eb99f0e3b162ae1b8a747d54e9fe" + ] + ] + ] + ] + ], + "timestamp": "2023-12-07T11:20:44.005938987" + } +} \ No newline at end of file From f97ef32658d0edb5245a9219ff43822e36a7afc5 Mon Sep 17 00:00:00 2001 From: Carson J Miller <68351153+CarsonJM@users.noreply.github.com> Date: Tue, 12 Dec 2023 19:52:15 +0000 Subject: [PATCH 08/49] Moving config file params to nf-test files --- conf/nf_test.config | 23 +++++++++++++ modules/nf-core/genomad/download/main.nf | 2 -- nextflow.config | 1 + tests/test_nothing.nf.test | 20 ++++++++++++ tests/test_nothing.nf.test.snap | 8 +++++ tests/test_virus_identification.nf.test | 26 +++++++++++++-- tests/test_virus_identification.nf.test.snap | 34 -------------------- 7 files changed, 76 insertions(+), 38 deletions(-) create mode 100644 conf/nf_test.config create mode 100644 tests/test_nothing.nf.test.snap delete mode 100644 tests/test_virus_identification.nf.test.snap diff --git a/conf/nf_test.config b/conf/nf_test.config new file mode 100644 index 00000000..c2271d18 --- /dev/null +++ b/conf/nf_test.config @@ -0,0 +1,23 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for running minimal tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Runs input data but skipping all possible steps to allow for a fast testing + profile for input checks etc. + + Use as follows: + nextflow run nf-core/mag -profile test_nothing, --outdir + +---------------------------------------------------------------------------------------- +*/ + +params { + config_profile_name = 'Test profile' + config_profile_description = 'Minimal test dataset to check pipeline function' + + // Limit resources so that this can run on GitHub Actions + max_cpus = 2 + max_memory = '6.GB' + max_time = '6.h' + +} diff --git a/modules/nf-core/genomad/download/main.nf b/modules/nf-core/genomad/download/main.nf index 632d494e..a2ac6ecb 100644 --- a/modules/nf-core/genomad/download/main.nf +++ b/modules/nf-core/genomad/download/main.nf @@ -16,8 +16,6 @@ process GENOMAD_DOWNLOAD { script: def args = task.ext.args ?: '' """ - https_proxy=http://klone-dip1-A-ib:3128 - export https_proxy genomad \\ download-database . diff --git a/nextflow.config b/nextflow.config index 66314e9a..229bb803 100644 --- a/nextflow.config +++ b/nextflow.config @@ -299,6 +299,7 @@ profiles { executor.cpus = 4 executor.memory = 8.GB } + nf_test { includeConfig 'conf/nf_test.config' } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } test_host_rm { includeConfig 'conf/test_host_rm.config' } diff --git a/tests/test_nothing.nf.test b/tests/test_nothing.nf.test index 88fd806e..b24f3885 100644 --- a/tests/test_nothing.nf.test +++ b/tests/test_nothing.nf.test @@ -10,6 +10,26 @@ nextflow_pipeline { when { params { outdir = "$outputDir" + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + centrifuge_db = null + kraken2_db = null + keep_phix = true + skip_krona = true + skip_clipping = true + skip_adapter_trimming = true + skip_spades = true + skip_spadeshybrid = true + skip_megahit = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_metabat2 = true + skip_maxbin2 = true + skip_concoct = true + skip_prokka = true + skip_binqc = true + skip_gtdbtk = true + skip_concoct = true } } diff --git a/tests/test_nothing.nf.test.snap b/tests/test_nothing.nf.test.snap new file mode 100644 index 00000000..a2c985d1 --- /dev/null +++ b/tests/test_nothing.nf.test.snap @@ -0,0 +1,8 @@ +{ + "Parameters: test_nothing": { + "content": [ + "multiqc_fastqc.yaml:md5,2ef2c5154a509b13d9bcfe8cc879d71b" + ], + "timestamp": "2023-12-12T19:28:46.793245086" + } +} \ No newline at end of file diff --git a/tests/test_virus_identification.nf.test b/tests/test_virus_identification.nf.test index 0204c116..20693816 100644 --- a/tests/test_virus_identification.nf.test +++ b/tests/test_virus_identification.nf.test @@ -9,7 +9,29 @@ nextflow_pipeline { test("Parameters: run_virus_identification = true") { when { params { - outdir = "$outputDir" + outdir = "$outputDir" + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + run_virus_identification = true + genomad_splits = 7 + reads_minlength = 150 + megahit_fix_cpu_1 = true + centrifuge_db = null + kraken2_db = null + keep_phix = true + skip_clipping = true + coassemble_group = true + skip_gtdbtk = true + skip_binning = true + skip_prokka = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + skip_krona = true + skip_adapter_trimming = true + skip_metabat2 = true + skip_maxbin2 = true + skip_busco = true } } @@ -19,7 +41,7 @@ nextflow_pipeline { { assert snapshot( path("${params.outdir}/Assembly/MEGAHIT/MEGAHIT-group-0.contigs.fa.gz"), path("${params.outdir}/multiqc/multiqc_data/multiqc_fastqc.yaml"), - path("${params.outdir}/VirusIdentification/"), + path("${params.outdir}/VirusIdentification/") ).match() }, { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_run0_raw_1_fastqc.html").exists() }, { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_run0_raw_2_fastqc.html").exists() }, diff --git a/tests/test_virus_identification.nf.test.snap b/tests/test_virus_identification.nf.test.snap deleted file mode 100644 index 61cf6f2a..00000000 --- a/tests/test_virus_identification.nf.test.snap +++ /dev/null @@ -1,34 +0,0 @@ -{ - "Parameters: run_virus_identification = true": { - "content": [ - "MEGAHIT-group-0.contigs.fa.gz:md5,da40ccf271166ccfc233e70adb0a23ae", - "multiqc_fastqc.yaml:md5,2ef2c5154a509b13d9bcfe8cc879d71b", - [ - [ - [ - [ - "MEGAHIT-group-0.contigs_aggregated_classification.tsv:md5,afcdcac7a8f02544c4888533e870c4e8" - ], - [ - "MEGAHIT-group-0.contigs_taxonomy.tsv:md5,08148a4884b69088a8955c3fd8860027" - ], - [ - "MEGAHIT-group-0.contigs_provirus.tsv:md5,93a6bca59b0bf7f57c0b9b60d2e57082" - ], - [ - "MEGAHIT-group-0.contigs_plasmid.fna:md5,b1105afa94c08cd95e643e1534b57cd1", - "MEGAHIT-group-0.contigs_plasmid_genes.tsv:md5,8af86c89056190cbbe326e50b19f6bea", - "MEGAHIT-group-0.contigs_plasmid_proteins.faa:md5,bdd43fd82f22633079ee71dfc47ae0a1", - "MEGAHIT-group-0.contigs_plasmid_summary.tsv:md5,493372e1e756edb8c63318de474dacd4", - "MEGAHIT-group-0.contigs_virus.fna:md5,228873166621bd0c304938dd868757e2", - "MEGAHIT-group-0.contigs_virus_genes.tsv:md5,5bddb3bfe0a241796b26326478f06255", - "MEGAHIT-group-0.contigs_virus_proteins.faa:md5,4617c2217be860dbda882c6ae66a264e", - "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,4812eb99f0e3b162ae1b8a747d54e9fe" - ] - ] - ] - ] - ], - "timestamp": "2023-12-07T11:20:44.005938987" - } -} \ No newline at end of file From 53e58e1930d73e16f5f4a6892d93faaad0170fec Mon Sep 17 00:00:00 2001 From: Carson J Miller <68351153+CarsonJM@users.noreply.github.com> Date: Wed, 13 Dec 2023 00:51:24 +0000 Subject: [PATCH 09/49] virus identification test complete --- conf/test_virus_identification.config | 6 ++-- tests/test_virus_identification.nf.test | 6 +--- tests/test_virus_identification.nf.test.snap | 34 ++++++++++++++++++++ 3 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 tests/test_virus_identification.nf.test.snap diff --git a/conf/test_virus_identification.config b/conf/test_virus_identification.config index 0b80c9a3..743c27f6 100644 --- a/conf/test_virus_identification.config +++ b/conf/test_virus_identification.config @@ -15,9 +15,9 @@ params { config_profile_description = 'Minimal test dataset to check pipeline function virus identification' // Limit resources so that this can run on GitHub Actions - //max_cpus = 2 - //max_memory = '6.GB' - //max_time = '6.h' + max_cpus = 2 + max_memory = '6.GB' + max_time = '6.h' // Input data input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' diff --git a/tests/test_virus_identification.nf.test b/tests/test_virus_identification.nf.test index 20693816..3d7f48a6 100644 --- a/tests/test_virus_identification.nf.test +++ b/tests/test_virus_identification.nf.test @@ -10,7 +10,7 @@ nextflow_pipeline { when { params { outdir = "$outputDir" - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' run_virus_identification = true genomad_splits = 7 reads_minlength = 150 @@ -43,10 +43,6 @@ nextflow_pipeline { path("${params.outdir}/multiqc/multiqc_data/multiqc_fastqc.yaml"), path("${params.outdir}/VirusIdentification/") ).match() }, - { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_run0_raw_1_fastqc.html").exists() }, - { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_run0_raw_2_fastqc.html").exists() }, - { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_sample2_run0_raw_1_fastqc.html").exists() }, - { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_sample2_run0_raw_1_fastqc.html").exists() }, { assert new File("$outputDir/multiqc/multiqc_report.html").exists() } ) } diff --git a/tests/test_virus_identification.nf.test.snap b/tests/test_virus_identification.nf.test.snap new file mode 100644 index 00000000..dc480813 --- /dev/null +++ b/tests/test_virus_identification.nf.test.snap @@ -0,0 +1,34 @@ +{ + "Parameters: run_virus_identification = true": { + "content": [ + "MEGAHIT-group-0.contigs.fa.gz:md5,8419d4d6c72ac1006324a9fee1394646", + "multiqc_fastqc.yaml:md5,42cf8e13e8ef14ac9192896be672997c", + [ + [ + [ + [ + "MEGAHIT-group-0.contigs_aggregated_classification.tsv:md5,f153c7a73fcfdd50b984d66f028ee1f5" + ], + [ + "MEGAHIT-group-0.contigs_taxonomy.tsv:md5,0d7f92268635ade939854ab3c9d25d8a" + ], + [ + "MEGAHIT-group-0.contigs_provirus.tsv:md5,93a6bca59b0bf7f57c0b9b60d2e57082" + ], + [ + "MEGAHIT-group-0.contigs_plasmid.fna:md5,de02fb2a6118aeaad11ba53bea4b1915", + "MEGAHIT-group-0.contigs_plasmid_genes.tsv:md5,95d4f272d73ce65b77caa0ff02d8f478", + "MEGAHIT-group-0.contigs_plasmid_proteins.faa:md5,1d9cecab547cb22c5554e5c4d194d96a", + "MEGAHIT-group-0.contigs_plasmid_summary.tsv:md5,9320622f9a70dbe40b519b7659971ee2", + "MEGAHIT-group-0.contigs_virus.fna:md5,382caf28272d1e263e0a13221314562e", + "MEGAHIT-group-0.contigs_virus_genes.tsv:md5,b8c98e49ded642f7f1f18ed5f14ec617", + "MEGAHIT-group-0.contigs_virus_proteins.faa:md5,8bf26b6fa9ebf1c3931a7b92311f4e35", + "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,95db91fff85f23a19e64ef7ea0f8d371" + ] + ] + ] + ] + ], + "timestamp": "2023-12-12T21:17:39.225353455" + } +} \ No newline at end of file From 4dc19db2d20e3decbdce45e834d7e2651e2e5fcb Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Wed, 13 Dec 2023 00:53:05 +0000 Subject: [PATCH 10/49] Added test_adapterremoval tests --- conf/test.config | 10 +++--- tests/main.nf.test | 16 ++++++++- tests/test_adapterremoval.nf.test | 46 ++++++++++++++------------ tests/test_adapterremoval.nf.test.snap | 18 ++++++++++ tests/test_nothing.nf.test | 20 ----------- 5 files changed, 63 insertions(+), 47 deletions(-) create mode 100644 tests/test_adapterremoval.nf.test.snap diff --git a/conf/test.config b/conf/test.config index c2389ce3..4168c871 100644 --- a/conf/test.config +++ b/conf/test.config @@ -15,9 +15,9 @@ params { config_profile_description = 'Minimal test dataset to check pipeline function' // Limit resources so that this can run on GitHub Actions - //max_cpus = 2 - //max_memory = '6.GB' - //max_time = '6.h' + max_cpus = 2 + max_memory = '6.GB' + max_time = '6.h' // Input data input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' @@ -31,7 +31,7 @@ params { skip_gtdbtk = true skip_concoct = true megahit_fix_cpu_1 = true - spades_fix_cpus = 4 - spadeshybrid_fix_cpus = 4 + spades_fix_cpus = 2 + spadeshybrid_fix_cpus = 2 metabat_rng_seed = 1 } diff --git a/tests/main.nf.test b/tests/main.nf.test index 405cc4a5..9e6f6990 100644 --- a/tests/main.nf.test +++ b/tests/main.nf.test @@ -9,7 +9,21 @@ nextflow_pipeline { test("Parameters: default test") { when { params { - outdir = "$outputDir" + outdir = "$outputDir" + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' + centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" + kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" + skip_krona = true + min_length_unbinned_contigs = 1 + max_unbinned_contigs = 2 + busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz" + busco_clean = true + skip_gtdbtk = true + skip_concoct = true + megahit_fix_cpu_1 = true + spades_fix_cpus = 2 + spadeshybrid_fix_cpus = 2 + metabat_rng_seed = 1 } } diff --git a/tests/test_adapterremoval.nf.test b/tests/test_adapterremoval.nf.test index 69b6ade0..4354a242 100644 --- a/tests/test_adapterremoval.nf.test +++ b/tests/test_adapterremoval.nf.test @@ -9,7 +9,28 @@ nextflow_pipeline { test("Parameters: clip_tool = 'adapterremoval'") { when { params { + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' outdir = "$outputDir" + clip_tool = 'adapterremoval' + centrifuge_db = null + kraken2_db = null + keep_phix = true + coassemble_group = true + skip_spades = true + skip_spadeshybrid = true + skip_krona = true + skip_adapter_trimming = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_metabat2 = true + skip_maxbin2 = true + skip_concoct = true + skip_prokka = true + skip_binqc = true + skip_gtdbtk = true + skip_concoct = true + megahit_fix_cpu_1 = true } } @@ -17,27 +38,10 @@ nextflow_pipeline { assertAll( { assert workflow.success }, { assert snapshot( - path("${params.outdir}/Annotation/MetaEuk/"), - path("${params.outdir}/Annotation/Prodigal/"), - path("${params.outdir}/Annotation/Prokka/MEGAHIT/MEGAHIT-MaxBin2-test_minigut_euk.001/MEGAHIT-MaxBin2-test_minigut_euk.001.tsv"), - path("${params.outdir}/Annotation/Prokka/MEGAHIT/MEGAHIT-MetaBAT2-test_minigut_euk.2/MEGAHIT-MetaBAT2-test_minigut_euk.2.tsv"), - path("${params.outdir}/Annotation/Prokka/MEGAHIT/MEGAHIT-MetaBAT2-test_minigut_euk.unbinned.1/MEGAHIT-MetaBAT2-test_minigut_euk.unbinned.1.tsv"), - path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MaxBin2-test_minigut_euk.002/SPAdes-MaxBin2-test_minigut_euk.002.tsv"), - path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MetaBAT2-test_minigut_euk.2/SPAdes-MetaBAT2-test_minigut_euk.2.tsv"), - path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MetaBAT2-test_minigut_euk.unbinned.1/SPAdes-MetaBAT2-test_minigut_euk.unbinned.1.tsv"), - path("${params.outdir}/Assembly/MEGAHIT/MEGAHIT-test_minigut_euk.contigs.fa.gz"), - path("${params.outdir}/Assembly/MEGAHIT/MEGAHIT-test_minigut_sample2_euk.contigs.fa.gz"), - path("${params.outdir}/Assembly/SPAdes/SPAdes-test_minigut_euk_contigs.fasta.gz"), - path("${params.outdir}/Assembly/SPAdes/SPAdes-test_minigut_sample2_euk_contigs.fasta.gz"), - path("${params.outdir}/GenomeBinning/depths/contigs/"), - path("${params.outdir}/GenomeBinning/MaxBin2/"), - path("${params.outdir}/GenomeBinning/MetaBAT2/"), - path("${params.outdir}/QC_shortreads/adapterremoval/test_minigut_euk/test_minigut_euk_run0_ar2.settings"), - path("${params.outdir}/QC_shortreads/adapterremoval/test_minigut_sample2_euk/test_minigut_sample2_euk_run0_ar2.settings"), - path("${params.outdir}/QC_shortreads/remove_phix/test_minigut_euk_run0_phix_removed.bowtie2.log"), - path("${params.outdir}/QC_shortreads/remove_phix/test_minigut_sample2_euk_run0_phix_removed.bowtie2.log"), - path("${params.outdir}/Taxonomy/") - ).match() }, + "${outputDir}/Assembly/MEGAHIT/MEGAHIT-group-0.contigs.fa.gz", + path("${outputDir}/QC_shortreads/adapterremoval/"), + "${params.outdir}/multiqc/multiqc_data/multiqc_fastqc.yaml", + ).match()}, { assert new File("$outputDir/multiqc/multiqc_report.html").exists() } ) } diff --git a/tests/test_adapterremoval.nf.test.snap b/tests/test_adapterremoval.nf.test.snap new file mode 100644 index 00000000..cf1cc5dd --- /dev/null +++ b/tests/test_adapterremoval.nf.test.snap @@ -0,0 +1,18 @@ +{ + "Parameters: clip_tool = 'adapterremoval'": { + "content": [ + "MEGAHIT-group-0.contigs.fa.gz:md5,766fe3571e8774489f9b96ae913386a7", + [ + [ + "test_minigut_run1_ar2.settings:md5,eb194399448c3bfe5eced0b1d546a541", + "test_minigut_run2_ar2.settings:md5,eb194399448c3bfe5eced0b1d546a541" + ], + [ + "test_minigut_sample2_run0_ar2.settings:md5,8c175a320fab230961c2886e096a019e" + ] + ], + "multiqc_fastqc.yaml:md5,42cf8e13e8ef14ac9192896be672997c" + ], + "timestamp": "2023-12-13T00:48:34.419699875" + } +} \ No newline at end of file diff --git a/tests/test_nothing.nf.test b/tests/test_nothing.nf.test index b24f3885..88fd806e 100644 --- a/tests/test_nothing.nf.test +++ b/tests/test_nothing.nf.test @@ -10,26 +10,6 @@ nextflow_pipeline { when { params { outdir = "$outputDir" - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - centrifuge_db = null - kraken2_db = null - keep_phix = true - skip_krona = true - skip_clipping = true - skip_adapter_trimming = true - skip_spades = true - skip_spadeshybrid = true - skip_megahit = true - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_metabat2 = true - skip_maxbin2 = true - skip_concoct = true - skip_prokka = true - skip_binqc = true - skip_gtdbtk = true - skip_concoct = true } } From 8ecb7edb79e711310ada951c025c720210b30ced Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Wed, 13 Dec 2023 01:02:39 +0000 Subject: [PATCH 11/49] Updated CI to run nf-tests --- .github/workflows/ci.yml | 145 +++++++++++++++++++-------------------- tests/main.nf.test | 68 ------------------ tests/tags.yml | 2 + 3 files changed, 71 insertions(+), 144 deletions(-) delete mode 100644 tests/main.nf.test create mode 100644 tests/tags.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3aaa6f3e..325faee4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,112 +1,105 @@ name: nf-core CI -# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors on: - push: - branches: - - dev pull_request: release: types: [published] + merge_group: + types: + - checks_requested + branches: + - master + - dev env: NXF_ANSI_LOG: false + NFTEST_VER: "0.8.1" concurrency: group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" cancel-in-progress: true jobs: - test: - name: Run pipeline with test data - # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/mag') }}" + changes: + name: Check for changes runs-on: ubuntu-latest - strategy: - matrix: - NXF_VER: - - "23.04.0" - - "latest-everything" + outputs: + # Expose matched filters as job 'tags' output variable + tags: ${{ steps.filter.outputs.changes }} steps: - - name: Free some space - run: | - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - - name: Check out pipeline code - uses: actions/checkout@v3 - - - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + - uses: actions/checkout@v3 + - name: Combine all tags.yml files + id: get_username + run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml + - name: debug + run: cat .github/tags.yml + - uses: dorny/paths-filter@v2 + id: filter with: - version: "${{ matrix.NXF_VER }}" + filters: ".github/tags.yml" - - name: Run pipeline with test data + define_nxf_versions: + name: Choose nextflow versions to test against depending on target branch + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.nxf_versions.outputs.matrix }} + steps: + - id: nxf_versions run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results + if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then + echo matrix='["latest-everything"]' | tee -a $GITHUB_OUTPUT + else + echo matrix='["latest-everything", "23.04.0"]' | tee -a $GITHUB_OUTPUT + fi - profiles: - name: Run workflow profile - # Only run on push if this is the nf-core dev branch (merged PRs) - if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/mag') }} + test: + name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }} + needs: [changes, define_nxf_versions] + if: needs.changes.outputs.tags != '[]' runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - # Run remaining test profiles with minimum nextflow version + NXF_VER: ${{ fromJson(needs.define_nxf_versions.outputs.matrix) }} + tags: ["${{ fromJson(needs.changes.outputs.tags) }}"] profile: - [ - test_host_rm, - test_hybrid, - test_hybrid_host_rm, - test_busco_auto, - test_ancient_dna, - test_adapterremoval, - test_binrefinement, - test_virus_identification, - ] - steps: - - name: Free some space - run: | - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" + - "docker" + steps: - name: Check out pipeline code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Nextflow - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Run pipeline with ${{ matrix.profile }} test profile - run: | - nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.profile }},docker --outdir ./results + uses: nf-core/setup-nextflow@v1 + with: + version: "23.10.0" - checkm: - name: Run single test to checkm due to database download - # Only run on push if this is the nf-core dev branch (merged PRs) - if: ${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/mag') }} - runs-on: ubuntu-latest + - name: Cache nf-test installation + id: cache-software + uses: actions/cache@v3 + with: + path: | + /usr/local/bin/nf-test + /home/runner/.nf-test/nf-test.jar + key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest - steps: - - name: Free some space + - name: Install nf-test + if: steps.cache-software.outputs.cache-hit != 'true' run: | - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - - name: Check out pipeline code - uses: actions/checkout@v2 + wget -qO- https://code.askimed.com/install/nf-test | bash + sudo mv nf-test /usr/local/bin/ - - name: Install Nextflow + - name: Run nf-test run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + nf-test test --tag ${{ matrix.tags }} --profile "nf_test,${{ matrix.profile }}" --junitxml=test.xml - - name: Download and prepare CheckM database + - name: Output log on failure + if: failure() run: | - mkdir -p databases/checkm - wget https://data.ace.uq.edu.au/public/CheckM_databases/checkm_data_2015_01_16.tar.gz -P databases/checkm - tar xzvf databases/checkm/checkm_data_2015_01_16.tar.gz -C databases/checkm/ + sudo apt install bat > /dev/null + batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log - - name: Run pipeline with ${{ matrix.profile }} test profile - run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results --binqc_tool checkm --checkm_db databases/checkm + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + if: always() # always run even if the previous step fails + with: + report_paths: test.xml diff --git a/tests/main.nf.test b/tests/main.nf.test deleted file mode 100644 index 9e6f6990..00000000 --- a/tests/main.nf.test +++ /dev/null @@ -1,68 +0,0 @@ -nextflow_pipeline { - - name "Test pipeline: NFCORE_MAG" - script "main.nf" - tag "pipeline" - tag "nfcore_mag" - tag "nfcore_mag_test" - - test("Parameters: default test") { - when { - params { - outdir = "$outputDir" - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' - centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" - kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" - skip_krona = true - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz" - busco_clean = true - skip_gtdbtk = true - skip_concoct = true - megahit_fix_cpu_1 = true - spades_fix_cpus = 2 - spadeshybrid_fix_cpus = 2 - metabat_rng_seed = 1 - } - } - - then { - assertAll( - { assert workflow.success }, - { assert snapshot( - path("${params.outdir}/Annotation/Prodigal/"), - path("${params.outdir}/Annotation/Prokka/MEGAHIT/MEGAHIT-MaxBin2-test_minigut.001/MEGAHIT-MaxBin2-test_minigut.001.tsv"), - path("${params.outdir}/Annotation/Prokka/MEGAHIT/MEGAHIT-MetaBAT2-test_minigut_sample2.unbinned.1/MEGAHIT-MetaBAT2-test_minigut_sample2.unbinned.1.tsv"), - path("${params.outdir}/Annotation/Prokka/MEGAHIT/MEGAHIT-MetaBAT2-test_minigut.1/MEGAHIT-MetaBAT2-test_minigut.1.tsv"), - path("${params.outdir}/Annotation/Prokka/MEGAHIT/MEGAHIT-MetaBAT2-test_minigut.unbinned.1/MEGAHIT-MetaBAT2-test_minigut.unbinned.1.tsv"), - path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MaxBin2-test_minigut_sample2.001/SPAdes-MaxBin2-test_minigut_sample2.001.tsv"), - path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MaxBin2-test_minigut_sample2.noclass.1/SPAdes-MaxBin2-test_minigut_sample2.noclass.1.tsv"), - path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MaxBin2-test_minigut.001/SPAdes-MaxBin2-test_minigut.001.tsv"), - path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MetaBAT2-test_minigut_sample2.unbinned.1/SPAdes-MetaBAT2-test_minigut_sample2.unbinned.1.tsv"), - path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MetaBAT2-test_minigut.1/SPAdes-MetaBAT2-test_minigut.1.tsv"), - path("${params.outdir}/Annotation/Prokka/SPAdes/SPAdes-MetaBAT2-test_minigut.unbinned.1/SPAdes-MetaBAT2-test_minigut.unbinned.1.tsv"), - path("${params.outdir}/Assembly/MEGAHIT/MEGAHIT-test_minigut_sample2.contigs.fa.gz"), - path("${params.outdir}/Assembly/MEGAHIT/MEGAHIT-test_minigut.contigs.fa.gz"), - path("${params.outdir}/Assembly/SPAdes/SPAdes-test_minigut_sample2_contigs.fasta.gz"), - path("${params.outdir}/Assembly/SPAdes/SPAdes-test_minigut_contigs.fasta.gz"), - //path("${params.outdir}/GenomeBinning/depths/bins/bin_depths_summary.tsv"), - path("${params.outdir}/GenomeBinning/depths/contigs/"), - path("${params.outdir}/GenomeBinning/MaxBin2/"), - path("${params.outdir}/GenomeBinning/MetaBAT2/"), - //path("${params.outdir}/GenomeBinning/QC/busco_summary.tsv"), - //path("${params.outdir}/GenomeBinning/QC/quast_summary.tsv"), - //path("${params.outdir}/GenomeBinning/bin_summary.tsv"), - path("${params.outdir}/QC_shortreads/fastp/test_minigut/test_minigut_run1_fastp.fastp.json"), - path("${params.outdir}/QC_shortreads/fastp/test_minigut/test_minigut_run2_fastp.fastp.json"), - path("${params.outdir}/QC_shortreads/fastp/test_minigut_sample2/test_minigut_sample2_run0_fastp.fastp.json"), - path("${params.outdir}/QC_shortreads/remove_phix/test_minigut_run1_phix_removed.bowtie2.log"), - path("${params.outdir}/QC_shortreads/remove_phix/test_minigut_run2_phix_removed.bowtie2.log"), - path("${params.outdir}/QC_shortreads/remove_phix/test_minigut_sample2_run0_phix_removed.bowtie2.log"), - path("${params.outdir}/Taxonomy/") - ).match() }, - { assert new File("$outputDir/multiqc/multiqc_report.html").exists() } - ) - } - } -} diff --git a/tests/tags.yml b/tests/tags.yml new file mode 100644 index 00000000..88447a89 --- /dev/null +++ b/tests/tags.yml @@ -0,0 +1,2 @@ +nfcore_mag: + - ./** From 490260bceeb5e46a00447a91fea11b681440cead Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Wed, 13 Dec 2023 03:10:18 +0000 Subject: [PATCH 12/49] Fixed test_nothing nf-test --- tests/test_nothing.nf.test | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_nothing.nf.test b/tests/test_nothing.nf.test index 88fd806e..b24f3885 100644 --- a/tests/test_nothing.nf.test +++ b/tests/test_nothing.nf.test @@ -10,6 +10,26 @@ nextflow_pipeline { when { params { outdir = "$outputDir" + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + centrifuge_db = null + kraken2_db = null + keep_phix = true + skip_krona = true + skip_clipping = true + skip_adapter_trimming = true + skip_spades = true + skip_spadeshybrid = true + skip_megahit = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_metabat2 = true + skip_maxbin2 = true + skip_concoct = true + skip_prokka = true + skip_binqc = true + skip_gtdbtk = true + skip_concoct = true } } From 647a197e464823e650c0c6484f39a87e41cf28c4 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Thu, 14 Dec 2023 21:50:50 +0000 Subject: [PATCH 13/49] Modified names and used collectFile() --- conf/modules.config | 16 ------- conf/test_samplesheet.config | 42 ++++++++----------- docs/output.md | 2 +- nextflow.config | 2 +- nextflow_schema.json | 2 +- ...f => create_phageannotator_samplesheet.nf} | 37 +++++++--------- workflows/mag.nf | 32 +++++++------- 7 files changed, 51 insertions(+), 82 deletions(-) rename subworkflows/local/{samplesheet_creation.nf => create_phageannotator_samplesheet.nf} (75%) diff --git a/conf/modules.config b/conf/modules.config index e98280f5..cbfa51fb 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -741,22 +741,6 @@ process { ] } - withName: MAG_TO_SAMPLESHEET { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - enabled: false - ] - } - - withName: 'MAG_MERGE_SAMPLESHEET' { - ext.prefix = "${params.nf_core_pipeline}" - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - withName: CUSTOM_DUMPSOFTWAREVERSIONS { publishDir = [ path: { "${params.outdir}/pipeline_info" }, diff --git a/conf/test_samplesheet.config b/conf/test_samplesheet.config index 3a5ea4f4..d36c87fe 100644 --- a/conf/test_samplesheet.config +++ b/conf/test_samplesheet.config @@ -20,28 +20,22 @@ params { max_time = '6.h' // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' - centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" - kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" - skip_krona = true - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz" - busco_clean = true - skip_gtdbtk = true - skip_concoct = true - - // For computational efficiency - nf_core_pipeline = 'phageannotator' - coassemble_group = false - skip_binning = true - skip_prokka = true - skip_spadeshybrid = true - skip_quast = true - skip_prodigal = true - skip_krona = true - skip_adapter_trimming = true - skip_metabat2 = true - skip_maxbin2 = true - skip_busco = true + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' + generate_downstream_samplesheet = 'phageannotator' + skip_clipping = true + skip_adapter_trimming = true + keep_phix = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + coassemble_group = true + megahit_fix_cpu_1 = true + skip_spadeshybrid = true + skip_spades = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true } diff --git a/docs/output.md b/docs/output.md index 681dc371..a4141232 100644 --- a/docs/output.md +++ b/docs/output.md @@ -713,7 +713,7 @@ Because of aDNA damage, _de novo_ assemblers sometimes struggle to call a correc Output files - `samplesheet/` - - `[nf_core_pipeline].csv`: a samplesheet in CSV format that can be directly used as input for the specified nf-core pipeline + - `[generate_downstream_samplesheet].csv`: a samplesheet in CSV format that can be directly used as input for the specified nf-core pipeline
diff --git a/nextflow.config b/nextflow.config index 67611d98..bf879ac9 100644 --- a/nextflow.config +++ b/nextflow.config @@ -13,7 +13,7 @@ params { input = null single_end = false assembly_input = null - nf_core_pipeline = null + generate_downstream_samplesheet = null // short read preprocessing options skip_clipping = false diff --git a/nextflow_schema.json b/nextflow_schema.json index a50c9ae3..1d412462 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -53,7 +53,7 @@ "description": "MultiQC report title. Printed as page header, used for filename if not otherwise specified.", "fa_icon": "fas fa-file-signature" }, - "nf_core_pipeline": { + "generate_downstream_samplesheet": { "type": "string", "description": "Create a samplesheet for the specified nf-core pipeline", "help_text": "Automatically generate a samplesheet in CSV format that can be directly used as input for the specified nf-core pipeline.", diff --git a/subworkflows/local/samplesheet_creation.nf b/subworkflows/local/create_phageannotator_samplesheet.nf similarity index 75% rename from subworkflows/local/samplesheet_creation.nf rename to subworkflows/local/create_phageannotator_samplesheet.nf index d9edb182..6423486e 100644 --- a/subworkflows/local/samplesheet_creation.nf +++ b/subworkflows/local/create_phageannotator_samplesheet.nf @@ -2,7 +2,7 @@ include { CAT_CAT } from '../../modules/nf-core/cat/cat/main' include { MAG_TO_SAMPLESHEET } from '../../modules/local/mag_to_samplesheet' include { MAG_MERGE_SAMPLESHEET } from '../../modules/local/mag_merge_samplesheet' -workflow SAMPLESHEET_CREATION { +workflow CREATE_PHAGEANNOTATOR_SAMPLESHEET { take: short_reads //channel: [val(meta), path(fastq_1), path(fastq_2)] assemblies //channel: [val(meta), path(fasta)] @@ -50,8 +50,8 @@ workflow SAMPLESHEET_CREATION { meta.id = id meta.group = group meta.single_end = single_end - meta.fastq_1 = reads[0] ? reads[0] : '' - meta.fastq_2 = reads[1] && !meta.single_end ? reads[1] : '' + meta.fastq_1 = reads[0] + meta.fastq_2 = !meta.single_end ? reads[1] : '' meta.fasta = fasta ? fasta : '' return meta @@ -62,7 +62,7 @@ workflow SAMPLESHEET_CREATION { ch_combined_assemblies_remap = ch_combined_assemblies .map { meta, fasta -> - def group = meta.id.split('-') + def group = meta.id.split('group-') return [ group[1], fasta ] } @@ -75,17 +75,17 @@ workflow SAMPLESHEET_CREATION { return [ group, id, single_end, fastq ] } - .join ( ch_combined_assemblies_remap ) + .combine ( ch_combined_assemblies_remap, by:0 ) .map { - id, group, single_end, fastq, fasta -> + group, id, single_end, fastq, fasta -> def reads = fastq instanceof List ? fastq.flatten() : [ fastq ] def meta = [:] meta.id = id meta.group = group meta.single_end = single_end - meta.fastq_1 = reads[0] ? reads[0] : '' - meta.fastq_2 = reads[1] && !meta.single_end ? reads[1] : '' + meta.fastq_1 = reads[0] + meta.fastq_2 = !meta.single_end ? reads[1] : '' meta.fasta = fasta ? fasta : '' return meta @@ -93,23 +93,14 @@ workflow SAMPLESHEET_CREATION { .set { ch_mag_metadata } } - // - // MODULE: Stage FastQ/FastA files generated by nf-core/mag together and auto-create a samplesheet - // - MAG_TO_SAMPLESHEET ( - ch_mag_metadata, - params.nf_core_pipeline ?: '' - ) + // Create samplesheet for each sample using meta information + ch_mag_id_samplesheets = ch_mag_metadata.collectFile() { meta -> + [ "${meta.id}_phageannotator_samplesheet.csv", "sample,group,fastq_1,fastq_2,fasta" + '\n' + "${meta.id},${meta.group},${meta.fastq_1},${meta.fastq_2},${meta.fasta}" + '\n' ] + } - // - // MODULE: Create a merged samplesheet across all samples for the pipeline - // - MAG_MERGE_SAMPLESHEET ( - MAG_TO_SAMPLESHEET.out.samplesheet.collect{it[1]} - ) - ch_versions = ch_versions.mix( MAG_MERGE_SAMPLESHEET.out.versions ) + // Merge samplesheet across all samples for the pipeline + ch_mag_id_samplesheets.collectFile(name: "phageannotator_samplesheet.csv", keepHeader:true, skip:1, storeDir:"${params.outdir}/downstream_samplesheets/") emit: - samplesheet = ch_assemblies versions = ch_versions // channel: [ versions.yml ] } diff --git a/workflows/mag.nf b/workflows/mag.nf index d1b9eb0e..75e6111e 100644 --- a/workflows/mag.nf +++ b/workflows/mag.nf @@ -86,19 +86,19 @@ include { COMBINE_TSV as COMBINE_SUMMARY_TSV } from '../modules // // SUBWORKFLOW: Consisting of a mix of local and nf-core/modules // -include { INPUT_CHECK } from '../subworkflows/local/input_check' -include { BINNING_PREPARATION } from '../subworkflows/local/binning_preparation' -include { BINNING } from '../subworkflows/local/binning' -include { BINNING_REFINEMENT } from '../subworkflows/local/binning_refinement' -include { BUSCO_QC } from '../subworkflows/local/busco_qc' -include { VIRUS_IDENTIFICATION } from '../subworkflows/local/virus_identification' -include { CHECKM_QC } from '../subworkflows/local/checkm_qc' -include { GUNC_QC } from '../subworkflows/local/gunc_qc' -include { GTDBTK } from '../subworkflows/local/gtdbtk' -include { ANCIENT_DNA_ASSEMBLY_VALIDATION } from '../subworkflows/local/ancient_dna' -include { DOMAIN_CLASSIFICATION } from '../subworkflows/local/domain_classification' -include { DEPTHS } from '../subworkflows/local/depths' -include { SAMPLESHEET_CREATION } from '../subworkflows/local/samplesheet_creation' +include { INPUT_CHECK } from '../subworkflows/local/input_check' +include { BINNING_PREPARATION } from '../subworkflows/local/binning_preparation' +include { BINNING } from '../subworkflows/local/binning' +include { BINNING_REFINEMENT } from '../subworkflows/local/binning_refinement' +include { BUSCO_QC } from '../subworkflows/local/busco_qc' +include { VIRUS_IDENTIFICATION } from '../subworkflows/local/virus_identification' +include { CHECKM_QC } from '../subworkflows/local/checkm_qc' +include { GUNC_QC } from '../subworkflows/local/gunc_qc' +include { GTDBTK } from '../subworkflows/local/gtdbtk' +include { ANCIENT_DNA_ASSEMBLY_VALIDATION } from '../subworkflows/local/ancient_dna' +include { DOMAIN_CLASSIFICATION } from '../subworkflows/local/domain_classification' +include { DEPTHS } from '../subworkflows/local/depths' +include { CREATE_PHAGEANNOTATOR_SAMPLESHEET } from '../subworkflows/local/create_phageannotator_samplesheet' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1017,9 +1017,9 @@ workflow MAG { // // SUBWORKFLOW: Auto-create samplesheets for downstream nf-core pipelines // - if ( params.nf_core_pipeline ) { - ch_samplesheet = SAMPLESHEET_CREATION ( ch_short_reads, ch_assemblies ).samplesheet - ch_versions = ch_versions.mix( SAMPLESHEET_CREATION.out.versions ) + if ( params.generate_downstream_samplesheet == 'phageannotator' ) { + CREATE_PHAGEANNOTATOR_SAMPLESHEET ( ch_short_reads, ch_assemblies ) + ch_versions = ch_versions.mix( CREATE_PHAGEANNOTATOR_SAMPLESHEET.out.versions ) } CUSTOM_DUMPSOFTWAREVERSIONS ( From ff096dce2a11d57a600a10099978175ae24c0560 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Thu, 14 Dec 2023 15:03:00 -0800 Subject: [PATCH 14/49] Restructured workflows dir and added more nf-tests --- .github/workflows/ci.yml | 2 +- conf/base.config | 7 + conf/modules.config | 5 +- conf/nf_test.config | 23 --- conf/test_adapterremoval.config | 36 ++-- conf/test_bbnorm.config | 34 ++-- conf/test_host_rm.config | 22 +- conf/test_hybrid.config | 21 +- conf/test_hybrid_host_rm.config | 20 +- conf/test_nothing.config | 43 ++-- conf/test_virus_identification.config | 24 +-- main.nf | 2 +- nextflow.config | 7 +- subworkflows/local/busco_qc.nf | 9 +- subworkflows/local/gtdbtk.nf | 5 +- subworkflows/local/gunc_qc.nf | 3 +- subworkflows/local/virus_identification.nf | 5 +- tests/tags.yml | 2 - tests/test_adapterremoval.nf.test | 49 ----- tests/test_adapterremoval.nf.test.snap | 18 -- tests/test_nothing.nf.test | 50 ----- tests/test_nothing.nf.test.snap | 8 - tests/test_virus_identification.nf.test | 50 ----- tests/test_virus_identification.nf.test.snap | 34 ---- workflows/{mag.nf => mag/main.nf} | 183 +++++++++++------ workflows/mag/tests/tags.yml | 15 ++ .../mag/tests/test_adapterremoval.nf.test | 21 ++ .../tests/test_adapterremoval.nf.test.snap | 163 +++++++++++++++ workflows/mag/tests/test_bbnorm.nf.test | 21 ++ workflows/mag/tests/test_bbnorm.nf.test.snap | 167 ++++++++++++++++ workflows/mag/tests/test_host_rm.nf.test | 21 ++ workflows/mag/tests/test_host_rm.nf.test.snap | 159 +++++++++++++++ workflows/mag/tests/test_hybrid.nf.test | 21 ++ workflows/mag/tests/test_hybrid.nf.test.snap | 181 +++++++++++++++++ .../mag/tests/test_hybrid_host_rm.nf.test | 21 ++ .../tests/test_hybrid_host_rm.nf.test.snap | 161 +++++++++++++++ workflows/mag/tests/test_nothing.nf.test | 20 ++ workflows/mag/tests/test_nothing.nf.test.snap | 155 ++++++++++++++ .../tests/test_virus_identification.nf.test | 20 ++ .../test_virus_identification.nf.test.snap | 189 ++++++++++++++++++ 40 files changed, 1609 insertions(+), 388 deletions(-) delete mode 100644 conf/nf_test.config delete mode 100644 tests/tags.yml delete mode 100644 tests/test_adapterremoval.nf.test delete mode 100644 tests/test_adapterremoval.nf.test.snap delete mode 100644 tests/test_nothing.nf.test delete mode 100644 tests/test_nothing.nf.test.snap delete mode 100644 tests/test_virus_identification.nf.test delete mode 100644 tests/test_virus_identification.nf.test.snap rename workflows/{mag.nf => mag/main.nf} (85%) create mode 100644 workflows/mag/tests/tags.yml create mode 100644 workflows/mag/tests/test_adapterremoval.nf.test create mode 100644 workflows/mag/tests/test_adapterremoval.nf.test.snap create mode 100644 workflows/mag/tests/test_bbnorm.nf.test create mode 100644 workflows/mag/tests/test_bbnorm.nf.test.snap create mode 100644 workflows/mag/tests/test_host_rm.nf.test create mode 100644 workflows/mag/tests/test_host_rm.nf.test.snap create mode 100644 workflows/mag/tests/test_hybrid.nf.test create mode 100644 workflows/mag/tests/test_hybrid.nf.test.snap create mode 100644 workflows/mag/tests/test_hybrid_host_rm.nf.test create mode 100644 workflows/mag/tests/test_hybrid_host_rm.nf.test.snap create mode 100644 workflows/mag/tests/test_nothing.nf.test create mode 100644 workflows/mag/tests/test_nothing.nf.test.snap create mode 100644 workflows/mag/tests/test_virus_identification.nf.test create mode 100644 workflows/mag/tests/test_virus_identification.nf.test.snap diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 325faee4..5763fdd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,7 @@ jobs: - name: Run nf-test run: | - nf-test test --tag ${{ matrix.tags }} --profile "nf_test,${{ matrix.profile }}" --junitxml=test.xml + nf-test test --tag ${{ matrix.tags }} --profile mag_"${{ matrix.tags }},${{ matrix.profile }}" --junitxml=test.xml - name: Output log on failure if: failure() diff --git a/conf/base.config b/conf/base.config index 7dec9e28..4f4b3d44 100644 --- a/conf/base.config +++ b/conf/base.config @@ -122,6 +122,13 @@ process { memory = { check_max (128.GB * task.attempt, 'memory' ) } time = { check_max (12.h * task.attempt, 'time' ) } } + //bowtie2 returns exit code 250 when running out of memory + withName: BOWTIE2_HOST_REMOVAL_ALIGN { + cpus = { check_bowtie2_cpus (8, task.attempt ) } + memory = { check_max (40.GB * task.attempt, 'memory' ) } + time = { check_max (16.h * task.attempt, 'time' ) } + errorStrategy = { task.exitStatus in [143,137,104,134,139,250] ? 'retry' : 'finish' } + } //MEGAHIT returns exit code 250 when running out of memory withName: MEGAHIT { cpus = { check_megahit_cpus (8, task.attempt ) } diff --git a/conf/modules.config b/conf/modules.config index cbfa51fb..f896d70a 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -115,7 +115,10 @@ process { } withName: BOWTIE2_HOST_REMOVAL_ALIGN { - ext.args = params.host_removal_verysensitive ? "--very-sensitive" : "--sensitive" + ext.args = [ + params.host_removal_verysensitive ? "--very-sensitive" : "--sensitive", + "--seed ${params.bowtie2_seed}" + ].join(' ').trim() ext.args2 = params.host_removal_save_ids ? "--host_removal_save_ids" : '' ext.prefix = { "${meta.id}_run${meta.run}_host_removed" } publishDir = [ diff --git a/conf/nf_test.config b/conf/nf_test.config deleted file mode 100644 index c2271d18..00000000 --- a/conf/nf_test.config +++ /dev/null @@ -1,23 +0,0 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Nextflow config file for running minimal tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Runs input data but skipping all possible steps to allow for a fast testing - profile for input checks etc. - - Use as follows: - nextflow run nf-core/mag -profile test_nothing, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Test profile' - config_profile_description = 'Minimal test dataset to check pipeline function' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - -} diff --git a/conf/test_adapterremoval.config b/conf/test_adapterremoval.config index f9f759cd..54bb7959 100644 --- a/conf/test_adapterremoval.config +++ b/conf/test_adapterremoval.config @@ -15,25 +15,27 @@ params { config_profile_description = 'Minimal test dataset to check pipeline function with AdapterRemoval data and domain classification.' // Limit resources so that this can run on GitHub Actions - //max_cpus = 2 - //max_memory = '6.GB' - //max_time = '6.h' + max_cpus = 2 + max_memory = '6.GB' + max_time = '6.h' // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.euk.csv' - centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" - kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" - metaeuk_db = "https://github.com/nf-core/test-datasets/raw/modules/data/proteomics/database/yeast_UPS.fasta" + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' + clip_tool = 'adapterremoval' + keep_phix = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null skip_krona = true - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz" + skip_megahit = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true skip_gtdbtk = true - clip_tool = 'adapterremoval' - skip_concoct = true - bin_domain_classification = true - megahit_fix_cpu_1 = true - spades_fix_cpus = 4 - spadeshybrid_fix_cpus = 4 - metabat_rng_seed = 1 + skip_prokka = true + skip_metaeuk = true + } diff --git a/conf/test_bbnorm.config b/conf/test_bbnorm.config index 55d48a8b..8e7d6e94 100644 --- a/conf/test_bbnorm.config +++ b/conf/test_bbnorm.config @@ -20,21 +20,21 @@ params { max_time = '6.h' // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - keep_phix = true - skip_clipping = true - skip_prokka = true - skip_prodigal = true - skip_quast = true - skip_binning = true - centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" - kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" - skip_krona = true - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz" - busco_clean = true - skip_gtdbtk = true - bbnorm = true - coassemble_group = true + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' + bbnorm = true + keep_phix = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_megahit = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true } diff --git a/conf/test_host_rm.config b/conf/test_host_rm.config index 7af3bcd4..906c68a7 100644 --- a/conf/test_host_rm.config +++ b/conf/test_host_rm.config @@ -20,11 +20,23 @@ params { max_time = '6.h' // Input data - host_fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/host_reference/genome.hg38.chr21_10000bp_region.fa" input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.host_rm.csv' - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz" + keep_phix = true + host_fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/host_reference/genome.hg38.chr21_10000bp_region.fa" + bowtie2_fix_cpu_1 = true + skip_clipping = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_megahit = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true skip_gtdbtk = true - skip_concoct = true + skip_prokka = true + skip_metaeuk = true } diff --git a/conf/test_hybrid.config b/conf/test_hybrid.config index 0600c88c..d0fb0bcc 100644 --- a/conf/test_hybrid.config +++ b/conf/test_hybrid.config @@ -21,9 +21,22 @@ params { // Input data input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.hybrid.csv' - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz" + keep_phix = true + skip_clipping = false + skip_adapter_trimming = false + centrifuge_db = null + kraken2_db = null + skip_krona = true + coassemble_group = true + megahit_fix_cpu_1 = true + skip_spades = true + skip_spadeshybrid = false + spadeshybrid_fix_cpus = 2 + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true skip_gtdbtk = true - skip_concoct = true + skip_prokka = true + skip_metaeuk = true } diff --git a/conf/test_hybrid_host_rm.config b/conf/test_hybrid_host_rm.config index 7a0e4a15..05b2f8b4 100644 --- a/conf/test_hybrid_host_rm.config +++ b/conf/test_hybrid_host_rm.config @@ -20,11 +20,23 @@ params { max_time = '6.h' // Input data - host_fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/host_reference/genome.hg38.chr21_10000bp_region.fa" input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.hybrid_host_rm.csv' - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 + host_fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/host_reference/genome.hg38.chr21_10000bp_region.fa" + bowtie2_fix_cpu_1 = true + keep_phix = true + skip_clipping = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_megahit = true + skip_spades = true + skip_spadeshybrid = false + spadeshybrid_fix_cpus = 2 + skip_quast = true + skip_prodigal = true + skip_binning = true skip_binqc = true - skip_concoct = true skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true } diff --git a/conf/test_nothing.config b/conf/test_nothing.config index eb7201db..8904d2f6 100644 --- a/conf/test_nothing.config +++ b/conf/test_nothing.config @@ -16,29 +16,26 @@ params { config_profile_description = 'Minimal test dataset to check pipeline function' // Limit resources so that this can run on GitHub Actions - //max_cpus = 2 - //max_memory = '6.GB' - //max_time = '6.h' + max_cpus = 2 + max_memory = '6.GB' + max_time = '6.h' // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - centrifuge_db = null - kraken2_db = null - keep_phix = true - skip_krona = true - skip_clipping = true - skip_adapter_trimming = true - skip_spades = true - skip_spadeshybrid = true - skip_megahit = true - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_metabat2 = true - skip_maxbin2 = true - skip_concoct = true - skip_prokka = true - skip_binqc = true - skip_gtdbtk = true - skip_concoct = true + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' + keep_phix = true + skip_clipping = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_megahit = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true } diff --git a/conf/test_virus_identification.config b/conf/test_virus_identification.config index 743c27f6..635edb50 100644 --- a/conf/test_virus_identification.config +++ b/conf/test_virus_identification.config @@ -20,28 +20,26 @@ params { max_time = '6.h' // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' run_virus_identification = true genomad_splits = 7 // For computational efficiency - reads_minlength = 150 - megahit_fix_cpu_1 = true - centrifuge_db = null - kraken2_db = null keep_phix = true skip_clipping = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true coassemble_group = true - skip_gtdbtk = true - skip_binning = true - skip_prokka = true + megahit_fix_cpu_1 = true skip_spades = true skip_spadeshybrid = true skip_quast = true skip_prodigal = true - skip_krona = true - skip_adapter_trimming = true - skip_metabat2 = true - skip_maxbin2 = true - skip_busco = true + skip_binning = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true } diff --git a/main.nf b/main.nf index 71285a74..d8868661 100644 --- a/main.nf +++ b/main.nf @@ -52,7 +52,7 @@ WorkflowMain.initialise(workflow, params, log) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { MAG } from './workflows/mag' +include { MAG } from './workflows/mag/main' // // WORKFLOW: Run main nf-core/mag analysis pipeline diff --git a/nextflow.config b/nextflow.config index 229bb803..e5865f94 100644 --- a/nextflow.config +++ b/nextflow.config @@ -139,6 +139,8 @@ params { spades_fix_cpus = -1 spadeshybrid_fix_cpus = -1 metabat_rng_seed = 1 + bowtie2_seed = 1 + bowtie2_fix_cpu_1 = false // Annotation options skip_metaeuk = false @@ -299,7 +301,6 @@ profiles { executor.cpus = 4 executor.memory = 8.GB } - nf_test { includeConfig 'conf/nf_test.config' } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } test_host_rm { includeConfig 'conf/test_host_rm.config' } @@ -411,6 +412,10 @@ def check_max(obj, type) { // Functions to fix number of cpus to allow reproducibility for MEGAHIT and SPAdes // if corresponding parameters are specified, number of cpus is not increased with retries +def check_bowtie2_cpus (x, attempt ) { + if (params.bowtie2_fix_cpu_1) return 1 + else return check_max (x * attempt, 'cpus' ) +} def check_megahit_cpus (x, attempt ) { if (params.megahit_fix_cpu_1) return 1 else return check_max (x * attempt, 'cpus' ) diff --git a/subworkflows/local/busco_qc.nf b/subworkflows/local/busco_qc.nf index a5c3be8d..15ab8cfa 100644 --- a/subworkflows/local/busco_qc.nf +++ b/subworkflows/local/busco_qc.nf @@ -76,8 +76,9 @@ workflow BUSCO_QC { ) emit: - summary = BUSCO_SUMMARY.out.summary - failed_bin = BUSCO.out.failed_bin.map{it[1]} - multiqc = BUSCO.out.summary_domain.mix(BUSCO.out.summary_specific).map{it[1]} - versions = BUSCO.out.versions + summary_specific = BUSCO.out.summary_specific + summary = BUSCO_SUMMARY.out.summary + failed_bin = BUSCO.out.failed_bin.map{it[1]} + multiqc = BUSCO.out.summary_domain.mix(BUSCO.out.summary_specific).map{it[1]} + versions = BUSCO.out.versions } diff --git a/subworkflows/local/gtdbtk.nf b/subworkflows/local/gtdbtk.nf index 2f110a43..aaf7133c 100644 --- a/subworkflows/local/gtdbtk.nf +++ b/subworkflows/local/gtdbtk.nf @@ -92,6 +92,7 @@ workflow GTDBTK { ) emit: - summary = GTDBTK_SUMMARY.out.summary - versions = GTDBTK_CLASSIFYWF.out.versions + gtdbtk_summaries = GTDBTK_CLASSIFYWF.out.summary + summary = GTDBTK_SUMMARY.out.summary + versions = GTDBTK_CLASSIFYWF.out.versions } diff --git a/subworkflows/local/gunc_qc.nf b/subworkflows/local/gunc_qc.nf index 912b9425..0d52d666 100644 --- a/subworkflows/local/gunc_qc.nf +++ b/subworkflows/local/gunc_qc.nf @@ -46,6 +46,7 @@ workflow GUNC_QC { } emit: - versions = ch_versions + gunc_maxcss_level_tsv = GUNC_RUN.out.maxcss_level_tsv + versions = ch_versions } diff --git a/subworkflows/local/virus_identification.nf b/subworkflows/local/virus_identification.nf index 4a3a2dac..1cecc7cb 100644 --- a/subworkflows/local/virus_identification.nf +++ b/subworkflows/local/virus_identification.nf @@ -24,7 +24,8 @@ workflow VIRUS_IDENTIFICATION { ch_versions.mix( GENOMAD_ENDTOEND.out.versions ) emit: - identified_viruses = ch_identified_viruses - versions = ch_versions + identified_viruses = ch_identified_viruses + virus_summary = GENOMAD_ENDTOEND.out.virus_summary + versions = ch_versions } diff --git a/tests/tags.yml b/tests/tags.yml deleted file mode 100644 index 88447a89..00000000 --- a/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -nfcore_mag: - - ./** diff --git a/tests/test_adapterremoval.nf.test b/tests/test_adapterremoval.nf.test deleted file mode 100644 index 4354a242..00000000 --- a/tests/test_adapterremoval.nf.test +++ /dev/null @@ -1,49 +0,0 @@ -nextflow_pipeline { - - name "Test pipeline: NFCORE_MAG" - script "main.nf" - tag "pipeline" - tag "nfcore_mag" - tag "nfcore_mag_test_adapterremoval" - - test("Parameters: clip_tool = 'adapterremoval'") { - when { - params { - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' - outdir = "$outputDir" - clip_tool = 'adapterremoval' - centrifuge_db = null - kraken2_db = null - keep_phix = true - coassemble_group = true - skip_spades = true - skip_spadeshybrid = true - skip_krona = true - skip_adapter_trimming = true - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_metabat2 = true - skip_maxbin2 = true - skip_concoct = true - skip_prokka = true - skip_binqc = true - skip_gtdbtk = true - skip_concoct = true - megahit_fix_cpu_1 = true - } - } - - then { - assertAll( - { assert workflow.success }, - { assert snapshot( - "${outputDir}/Assembly/MEGAHIT/MEGAHIT-group-0.contigs.fa.gz", - path("${outputDir}/QC_shortreads/adapterremoval/"), - "${params.outdir}/multiqc/multiqc_data/multiqc_fastqc.yaml", - ).match()}, - { assert new File("$outputDir/multiqc/multiqc_report.html").exists() } - ) - } - } -} diff --git a/tests/test_adapterremoval.nf.test.snap b/tests/test_adapterremoval.nf.test.snap deleted file mode 100644 index cf1cc5dd..00000000 --- a/tests/test_adapterremoval.nf.test.snap +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Parameters: clip_tool = 'adapterremoval'": { - "content": [ - "MEGAHIT-group-0.contigs.fa.gz:md5,766fe3571e8774489f9b96ae913386a7", - [ - [ - "test_minigut_run1_ar2.settings:md5,eb194399448c3bfe5eced0b1d546a541", - "test_minigut_run2_ar2.settings:md5,eb194399448c3bfe5eced0b1d546a541" - ], - [ - "test_minigut_sample2_run0_ar2.settings:md5,8c175a320fab230961c2886e096a019e" - ] - ], - "multiqc_fastqc.yaml:md5,42cf8e13e8ef14ac9192896be672997c" - ], - "timestamp": "2023-12-13T00:48:34.419699875" - } -} \ No newline at end of file diff --git a/tests/test_nothing.nf.test b/tests/test_nothing.nf.test deleted file mode 100644 index b24f3885..00000000 --- a/tests/test_nothing.nf.test +++ /dev/null @@ -1,50 +0,0 @@ -nextflow_pipeline { - - name "Test pipeline: NFCORE_MAG" - script "main.nf" - tag "pipeline" - tag "nfcore_mag" - tag "nfcore_mag_test_nothing" - - test("Parameters: test_nothing") { - when { - params { - outdir = "$outputDir" - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - centrifuge_db = null - kraken2_db = null - keep_phix = true - skip_krona = true - skip_clipping = true - skip_adapter_trimming = true - skip_spades = true - skip_spadeshybrid = true - skip_megahit = true - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_metabat2 = true - skip_maxbin2 = true - skip_concoct = true - skip_prokka = true - skip_binqc = true - skip_gtdbtk = true - skip_concoct = true - } - } - - then { - assertAll( - { assert workflow.success }, - { assert snapshot( - path("${params.outdir}/multiqc/multiqc_data/multiqc_fastqc.yaml"), - ).match() }, - { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_run0_raw_1_fastqc.html").exists() }, - { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_run0_raw_2_fastqc.html").exists() }, - { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_sample2_run0_raw_1_fastqc.html").exists() }, - { assert new File("$outputDir/QC_shortreads/fastqc/test_minigut_sample2_run0_raw_1_fastqc.html").exists() }, - { assert new File("$outputDir/multiqc/multiqc_report.html").exists() } - ) - } - } -} diff --git a/tests/test_nothing.nf.test.snap b/tests/test_nothing.nf.test.snap deleted file mode 100644 index a2c985d1..00000000 --- a/tests/test_nothing.nf.test.snap +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Parameters: test_nothing": { - "content": [ - "multiqc_fastqc.yaml:md5,2ef2c5154a509b13d9bcfe8cc879d71b" - ], - "timestamp": "2023-12-12T19:28:46.793245086" - } -} \ No newline at end of file diff --git a/tests/test_virus_identification.nf.test b/tests/test_virus_identification.nf.test deleted file mode 100644 index 3d7f48a6..00000000 --- a/tests/test_virus_identification.nf.test +++ /dev/null @@ -1,50 +0,0 @@ -nextflow_pipeline { - - name "Test pipeline: NFCORE_MAG" - script "main.nf" - tag "pipeline" - tag "nfcore_mag" - tag "nfcore_mag_test_virus_identification" - - test("Parameters: run_virus_identification = true") { - when { - params { - outdir = "$outputDir" - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' - run_virus_identification = true - genomad_splits = 7 - reads_minlength = 150 - megahit_fix_cpu_1 = true - centrifuge_db = null - kraken2_db = null - keep_phix = true - skip_clipping = true - coassemble_group = true - skip_gtdbtk = true - skip_binning = true - skip_prokka = true - skip_spades = true - skip_spadeshybrid = true - skip_quast = true - skip_prodigal = true - skip_krona = true - skip_adapter_trimming = true - skip_metabat2 = true - skip_maxbin2 = true - skip_busco = true - } - } - - then { - assertAll( - { assert workflow.success }, - { assert snapshot( - path("${params.outdir}/Assembly/MEGAHIT/MEGAHIT-group-0.contigs.fa.gz"), - path("${params.outdir}/multiqc/multiqc_data/multiqc_fastqc.yaml"), - path("${params.outdir}/VirusIdentification/") - ).match() }, - { assert new File("$outputDir/multiqc/multiqc_report.html").exists() } - ) - } - } -} diff --git a/tests/test_virus_identification.nf.test.snap b/tests/test_virus_identification.nf.test.snap deleted file mode 100644 index dc480813..00000000 --- a/tests/test_virus_identification.nf.test.snap +++ /dev/null @@ -1,34 +0,0 @@ -{ - "Parameters: run_virus_identification = true": { - "content": [ - "MEGAHIT-group-0.contigs.fa.gz:md5,8419d4d6c72ac1006324a9fee1394646", - "multiqc_fastqc.yaml:md5,42cf8e13e8ef14ac9192896be672997c", - [ - [ - [ - [ - "MEGAHIT-group-0.contigs_aggregated_classification.tsv:md5,f153c7a73fcfdd50b984d66f028ee1f5" - ], - [ - "MEGAHIT-group-0.contigs_taxonomy.tsv:md5,0d7f92268635ade939854ab3c9d25d8a" - ], - [ - "MEGAHIT-group-0.contigs_provirus.tsv:md5,93a6bca59b0bf7f57c0b9b60d2e57082" - ], - [ - "MEGAHIT-group-0.contigs_plasmid.fna:md5,de02fb2a6118aeaad11ba53bea4b1915", - "MEGAHIT-group-0.contigs_plasmid_genes.tsv:md5,95d4f272d73ce65b77caa0ff02d8f478", - "MEGAHIT-group-0.contigs_plasmid_proteins.faa:md5,1d9cecab547cb22c5554e5c4d194d96a", - "MEGAHIT-group-0.contigs_plasmid_summary.tsv:md5,9320622f9a70dbe40b519b7659971ee2", - "MEGAHIT-group-0.contigs_virus.fna:md5,382caf28272d1e263e0a13221314562e", - "MEGAHIT-group-0.contigs_virus_genes.tsv:md5,b8c98e49ded642f7f1f18ed5f14ec617", - "MEGAHIT-group-0.contigs_virus_proteins.faa:md5,8bf26b6fa9ebf1c3931a7b92311f4e35", - "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,95db91fff85f23a19e64ef7ea0f8d371" - ] - ] - ] - ] - ], - "timestamp": "2023-12-12T21:17:39.225353455" - } -} \ No newline at end of file diff --git a/workflows/mag.nf b/workflows/mag/main.nf similarity index 85% rename from workflows/mag.nf rename to workflows/mag/main.nf index 160928d2..7ad1d9a5 100644 --- a/workflows/mag.nf +++ b/workflows/mag/main.nf @@ -51,53 +51,53 @@ ch_multiqc_custom_methods_description = params.multiqc_methods_description ? fil // // MODULE: Local to the pipeline // -include { BOWTIE2_REMOVAL_BUILD as BOWTIE2_HOST_REMOVAL_BUILD } from '../modules/local/bowtie2_removal_build' -include { BOWTIE2_REMOVAL_ALIGN as BOWTIE2_HOST_REMOVAL_ALIGN } from '../modules/local/bowtie2_removal_align' -include { BOWTIE2_REMOVAL_BUILD as BOWTIE2_PHIX_REMOVAL_BUILD } from '../modules/local/bowtie2_removal_build' -include { BOWTIE2_REMOVAL_ALIGN as BOWTIE2_PHIX_REMOVAL_ALIGN } from '../modules/local/bowtie2_removal_align' -include { PORECHOP } from '../modules/local/porechop' -include { NANOLYSE } from '../modules/local/nanolyse' -include { FILTLONG } from '../modules/local/filtlong' -include { NANOPLOT as NANOPLOT_RAW } from '../modules/local/nanoplot' -include { NANOPLOT as NANOPLOT_FILTERED } from '../modules/local/nanoplot' -include { CENTRIFUGE_DB_PREPARATION } from '../modules/local/centrifuge_db_preparation' -include { CENTRIFUGE } from '../modules/local/centrifuge' -include { KRAKEN2_DB_PREPARATION } from '../modules/local/kraken2_db_preparation' -include { KRAKEN2 } from '../modules/local/kraken2' -include { KRONA_DB } from '../modules/local/krona_db' -include { KRONA } from '../modules/local/krona' -include { POOL_SINGLE_READS as POOL_SHORT_SINGLE_READS } from '../modules/local/pool_single_reads' -include { POOL_PAIRED_READS } from '../modules/local/pool_paired_reads' -include { POOL_SINGLE_READS as POOL_LONG_READS } from '../modules/local/pool_single_reads' -include { MEGAHIT } from '../modules/local/megahit' -include { SPADES } from '../modules/local/spades' -include { SPADESHYBRID } from '../modules/local/spadeshybrid' -include { GUNZIP as GUNZIP_ASSEMBLIES } from '../modules/nf-core/gunzip' -include { QUAST } from '../modules/local/quast' -include { QUAST_BINS } from '../modules/local/quast_bins' -include { QUAST_BINS_SUMMARY } from '../modules/local/quast_bins_summary' -include { CAT_DB } from '../modules/local/cat_db' -include { CAT_DB_GENERATE } from '../modules/local/cat_db_generate' -include { CAT } from '../modules/local/cat' -include { CAT_SUMMARY } from "../modules/local/cat_summary" -include { BIN_SUMMARY } from '../modules/local/bin_summary' -include { COMBINE_TSV as COMBINE_SUMMARY_TSV } from '../modules/local/combine_tsv' +include { BOWTIE2_REMOVAL_BUILD as BOWTIE2_HOST_REMOVAL_BUILD } from '../../modules/local/bowtie2_removal_build' +include { BOWTIE2_REMOVAL_ALIGN as BOWTIE2_HOST_REMOVAL_ALIGN } from '../../modules/local/bowtie2_removal_align' +include { BOWTIE2_REMOVAL_BUILD as BOWTIE2_PHIX_REMOVAL_BUILD } from '../../modules/local/bowtie2_removal_build' +include { BOWTIE2_REMOVAL_ALIGN as BOWTIE2_PHIX_REMOVAL_ALIGN } from '../../modules/local/bowtie2_removal_align' +include { PORECHOP } from '../../modules/local/porechop' +include { NANOLYSE } from '../../modules/local/nanolyse' +include { FILTLONG } from '../../modules/local/filtlong' +include { NANOPLOT as NANOPLOT_RAW } from '../../modules/local/nanoplot' +include { NANOPLOT as NANOPLOT_FILTERED } from '../../modules/local/nanoplot' +include { CENTRIFUGE_DB_PREPARATION } from '../../modules/local/centrifuge_db_preparation' +include { CENTRIFUGE } from '../../modules/local/centrifuge' +include { KRAKEN2_DB_PREPARATION } from '../../modules/local/kraken2_db_preparation' +include { KRAKEN2 } from '../../modules/local/kraken2' +include { KRONA_DB } from '../../modules/local/krona_db' +include { KRONA } from '../../modules/local/krona' +include { POOL_SINGLE_READS as POOL_SHORT_SINGLE_READS } from '../../modules/local/pool_single_reads' +include { POOL_PAIRED_READS } from '../../modules/local/pool_paired_reads' +include { POOL_SINGLE_READS as POOL_LONG_READS } from '../../modules/local/pool_single_reads' +include { MEGAHIT } from '../../modules/local/megahit' +include { SPADES } from '../../modules/local/spades' +include { SPADESHYBRID } from '../../modules/local/spadeshybrid' +include { GUNZIP as GUNZIP_ASSEMBLIES } from '../../modules/nf-core/gunzip' +include { QUAST } from '../../modules/local/quast' +include { QUAST_BINS } from '../../modules/local/quast_bins' +include { QUAST_BINS_SUMMARY } from '../../modules/local/quast_bins_summary' +include { CAT_DB } from '../../modules/local/cat_db' +include { CAT_DB_GENERATE } from '../../modules/local/cat_db_generate' +include { CAT } from '../../modules/local/cat' +include { CAT_SUMMARY } from "../../modules/local/cat_summary" +include { BIN_SUMMARY } from '../../modules/local/bin_summary' +include { COMBINE_TSV as COMBINE_SUMMARY_TSV } from '../../modules/local/combine_tsv' // // SUBWORKFLOW: Consisting of a mix of local and nf-core/modules // -include { INPUT_CHECK } from '../subworkflows/local/input_check' -include { BINNING_PREPARATION } from '../subworkflows/local/binning_preparation' -include { BINNING } from '../subworkflows/local/binning' -include { BINNING_REFINEMENT } from '../subworkflows/local/binning_refinement' -include { BUSCO_QC } from '../subworkflows/local/busco_qc' -include { VIRUS_IDENTIFICATION } from '../subworkflows/local/virus_identification' -include { CHECKM_QC } from '../subworkflows/local/checkm_qc' -include { GUNC_QC } from '../subworkflows/local/gunc_qc' -include { GTDBTK } from '../subworkflows/local/gtdbtk' -include { ANCIENT_DNA_ASSEMBLY_VALIDATION } from '../subworkflows/local/ancient_dna' -include { DOMAIN_CLASSIFICATION } from '../subworkflows/local/domain_classification' -include { DEPTHS } from '../subworkflows/local/depths' +include { INPUT_CHECK } from '../../subworkflows/local/input_check' +include { BINNING_PREPARATION } from '../../subworkflows/local/binning_preparation' +include { BINNING } from '../../subworkflows/local/binning' +include { BINNING_REFINEMENT } from '../../subworkflows/local/binning_refinement' +include { BUSCO_QC } from '../../subworkflows/local/busco_qc' +include { VIRUS_IDENTIFICATION } from '../../subworkflows/local/virus_identification' +include { CHECKM_QC } from '../../subworkflows/local/checkm_qc' +include { GUNC_QC } from '../../subworkflows/local/gunc_qc' +include { GTDBTK } from '../../subworkflows/local/gtdbtk' +include { ANCIENT_DNA_ASSEMBLY_VALIDATION } from '../../subworkflows/local/ancient_dna' +include { DOMAIN_CLASSIFICATION } from '../../subworkflows/local/domain_classification' +include { DEPTHS } from '../../subworkflows/local/depths' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -108,21 +108,21 @@ include { DEPTHS } from '../subworkflows/local/depths' // // MODULE: Installed directly from nf-core/modules // -include { ARIA2 as ARIA2_UNTAR } from '../modules/nf-core/aria2/main' -include { FASTQC as FASTQC_RAW } from '../modules/nf-core/fastqc/main' -include { FASTQC as FASTQC_TRIMMED } from '../modules/nf-core/fastqc/main' -include { SEQTK_MERGEPE } from '../modules/nf-core/seqtk/mergepe/main' -include { BBMAP_BBNORM } from '../modules/nf-core/bbmap/bbnorm/main' -include { FASTP } from '../modules/nf-core/fastp/main' -include { ADAPTERREMOVAL as ADAPTERREMOVAL_PE } from '../modules/nf-core/adapterremoval/main' -include { ADAPTERREMOVAL as ADAPTERREMOVAL_SE } from '../modules/nf-core/adapterremoval/main' -include { CAT_FASTQ } from '../modules/nf-core/cat/fastq/main' -include { PRODIGAL } from '../modules/nf-core/prodigal/main' -include { PROKKA } from '../modules/nf-core/prokka/main' -include { MMSEQS_DATABASES } from '../modules/nf-core/mmseqs/databases/main' -include { METAEUK_EASYPREDICT } from '../modules/nf-core/metaeuk/easypredict/main' -include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main' -include { MULTIQC } from '../modules/nf-core/multiqc/main' +include { ARIA2 as ARIA2_UNTAR } from '../../modules/nf-core/aria2/main' +include { FASTQC as FASTQC_RAW } from '../../modules/nf-core/fastqc/main' +include { FASTQC as FASTQC_TRIMMED } from '../../modules/nf-core/fastqc/main' +include { SEQTK_MERGEPE } from '../../modules/nf-core/seqtk/mergepe/main' +include { BBMAP_BBNORM } from '../../modules/nf-core/bbmap/bbnorm/main' +include { FASTP } from '../../modules/nf-core/fastp/main' +include { ADAPTERREMOVAL as ADAPTERREMOVAL_PE } from '../../modules/nf-core/adapterremoval/main' +include { ADAPTERREMOVAL as ADAPTERREMOVAL_SE } from '../../modules/nf-core/adapterremoval/main' +include { CAT_FASTQ } from '../../modules/nf-core/cat/fastq/main' +include { PRODIGAL } from '../../modules/nf-core/prodigal/main' +include { PROKKA } from '../../modules/nf-core/prokka/main' +include { MMSEQS_DATABASES } from '../../modules/nf-core/mmseqs/databases/main' +include { METAEUK_EASYPREDICT } from '../../modules/nf-core/metaeuk/easypredict/main' +include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../../modules/nf-core/custom/dumpsoftwareversions/main' +include { MULTIQC } from '../../modules/nf-core/multiqc/main' //////////////////////////////////////////////////// /* -- Create channel for reference databases -- */ @@ -216,6 +216,24 @@ if(params.metaeuk_db && !params.skip_metaeuk) { ch_metaeuk_db = Channel.empty() } +// Create empty channels for all emitted outputs + ch_short_reads = Channel.empty() + ch_assemblies = Channel.empty() + ch_prodigal_gene_annotations = Channel.empty() + ch_genomad_virus_summary = Channel.empty() + ch_binning_results_bins = Channel.empty() + ch_binning_results_unbins = Channel.empty() + ch_refined_bins = Channel.empty() + ch_refined_unbins = Channel.empty() + ch_busco_summary_specific = Channel.empty() + ch_checkm_tsv = Channel.empty() + ch_gunc_maxcss_level_tsv = Channel.empty() + ch_quast_bin_summaries = Channel.empty() + ch_cat_tax_classification_names = Channel.empty() + ch_gtdbtk_summaries = Channel.empty() + ch_prokka_faa = Channel.empty() + ch_metaeuk_easypredict_faa = Channel.empty() + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RUN MAIN WORKFLOW @@ -681,6 +699,7 @@ workflow MAG { ch_assemblies, 'gff' ) + ch_prodigal_gene_annotations = PRODIGAL.out.gene_annotations ch_versions = ch_versions.mix(PRODIGAL.out.versions.first()) } @@ -692,6 +711,7 @@ workflow MAG { if (params.run_virus_identification){ VIRUS_IDENTIFICATION(ch_assemblies, ch_genomad_db) + ch_genomad_virus_summary = VIRUS_IDENTIFICATION.out.virus_summary ch_versions = ch_versions.mix(VIRUS_IDENTIFICATION.out.versions.first()) } @@ -861,6 +881,7 @@ workflow MAG { ch_input_bins_for_qc ) ch_busco_summary = BUSCO_QC.out.summary + ch_busco_summary_specific = BUSCO_QC.out.summary_specific ch_versions = ch_versions.mix(BUSCO_QC.out.versions.first()) // process information if BUSCO analysis failed for individual bins due to no matching genes BUSCO_QC.out @@ -884,6 +905,7 @@ workflow MAG { ch_checkm_db ) ch_checkm_summary = CHECKM_QC.out.summary + ch_checkm_tsv = CHECKM_QC.out.checkm_tsv ch_versions = ch_versions.mix(CHECKM_QC.out.versions) @@ -898,6 +920,7 @@ workflow MAG { meta.domain != "eukarya" } GUNC_QC ( ch_input_bins_for_qc, ch_gunc_db, [] ) + ch_gunc_maxcss_level_tsv = GUNC_QC.out.gunc_maxcss_level_tsv ch_versions = ch_versions.mix( GUNC_QC.out.versions ) } @@ -913,6 +936,7 @@ workflow MAG { QUAST_BINS ( ch_input_for_quast_bins ) ch_versions = ch_versions.mix(QUAST_BINS.out.versions.first()) + ch_quast_bin_summaries = QUAST_BINS.out.quast_bin_summaries QUAST_BINS_SUMMARY ( QUAST_BINS.out.quast_bin_summaries.collect() ) ch_quast_bins_summary = QUAST_BINS_SUMMARY.out.summary } @@ -935,6 +959,7 @@ workflow MAG { CAT_SUMMARY( CAT.out.tax_classification_names.collect() ) + ch_cat_tax_classification_names = CAT.out.tax_classification_names ch_versions = ch_versions.mix(CAT.out.versions.first()) ch_versions = ch_versions.mix(CAT_SUMMARY.out.versions) @@ -960,6 +985,7 @@ workflow MAG { gtdb_mash ) ch_versions = ch_versions.mix(GTDBTK.out.versions.first()) + ch_gtdbtk_summaries = GTDBTK.out.gtdbtk_summaries ch_gtdbtk_summary = GTDBTK.out.summary } } else { @@ -995,6 +1021,7 @@ workflow MAG { [], [] ) + ch_prokka_faa = PROKKA.out.faa ch_versions = ch_versions.mix(PROKKA.out.versions.first()) } @@ -1009,6 +1036,7 @@ workflow MAG { } METAEUK_EASYPREDICT (ch_bins_for_metaeuk, ch_metaeuk_db) + ch_metaeuk_easypredict_faa = METAEUK_EASYPREDICT.out.faa ch_versions = ch_versions.mix(METAEUK_EASYPREDICT.out.versions) } } @@ -1089,6 +1117,45 @@ workflow MAG { ) multiqc_report = MULTIQC.out.report.toList() + + emit: + short_reads = ch_short_reads + // fastqc_raw = FASTQC_RAW.out.json + // adapterremoval_se = ADAPTERREMOVAL_SE.out.singles_truncated + // adapterremoval_pe = ADAPTERREMOVAL_PE.out.paired_truncated + // host_rm = BOWTIE2_HOST_REMOVAL_ALIGN.out.reads + // phix_rm = BOWTIE2_PHIX_REMOVAL_ALIGN.out.reads + // fastqc_trimmed = FASTQC_TRIMMED.out.json + // cat_fastq = CAT_FASTQ.out.reads + // seqtk = SEQTK_MERGEPE.out.reads + // bbmap = BBMAP_BBNORM.out.fastq + // nanoplot_raw = NANOPLOT_RAW.out.txt + // porechop = PORECHOP.out.reads + // nanolyse = NANOLYSE.out.reads + // filtlong = FILTLONG.out.reads + // nanoplot_filtered = NANOPLOT_FILTERED.out.txt + // kraken2 = KRAKEN2.out.report + // centrifuge = CENTRIFUGE.out.report + assemblies = ch_assemblies + // megahit = ch_megahit_assemblies + // spades = ch_spades_assemblies + // spadeshybrid = ch_spadeshybrid_assemblies + // quast_contigs = QUAST.out.report + prodigal = ch_prodigal_gene_annotations + genomad = ch_genomad_virus_summary + bins = ch_binning_results_bins + unbins = ch_binning_results_unbins + refined_bins = ch_refined_bins + refined_unbins = ch_refined_unbins + busco = ch_busco_summary_specific + checkm = ch_checkm_tsv + gunc = ch_gunc_maxcss_level_tsv + quast_bins = ch_quast_bin_summaries + cat = ch_cat_tax_classification_names + gtdbtk = ch_gtdbtk_summaries + prokka = ch_prokka_faa + metaeuk = ch_metaeuk_easypredict_faa + versions = ch_versions } /* diff --git a/workflows/mag/tests/tags.yml b/workflows/mag/tests/tags.yml new file mode 100644 index 00000000..3d9e9033 --- /dev/null +++ b/workflows/mag/tests/tags.yml @@ -0,0 +1,15 @@ +test_adapterremoval: + - ./** +test_bbnorm: + - ./** +test_host_rm: + - ./** +test_hybrid_host_rm: + - ./** +test_hybrid: + - ./** +test_nothing: + - ./** +test_virus_identification: + - ./** + diff --git a/workflows/mag/tests/test_adapterremoval.nf.test b/workflows/mag/tests/test_adapterremoval.nf.test new file mode 100644 index 00000000..9d05d884 --- /dev/null +++ b/workflows/mag/tests/test_adapterremoval.nf.test @@ -0,0 +1,21 @@ +nextflow_workflow { + + name "Test workflow: MAG" + script "../main.nf" + workflow "MAG" + tag "workflows" + tag "mag" + tag "mag_test_adapterremoval" + + test("Parameters: clip_tool = 'adapterremoval'") { + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out, + ).match()} + ) + } + } +} diff --git a/workflows/mag/tests/test_adapterremoval.nf.test.snap b/workflows/mag/tests/test_adapterremoval.nf.test.snap new file mode 100644 index 00000000..b5b418b1 --- /dev/null +++ b/workflows/mag/tests/test_adapterremoval.nf.test.snap @@ -0,0 +1,163 @@ +{ + "Parameters: clip_tool = 'adapterremoval'": { + "content": [ + { + "0": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "test_minigut_1.merged.fastq.gz:md5,3ba44a474c09f2bc8770299fc6d327fb", + "test_minigut_2.merged.fastq.gz:md5,587c61c5f7bf3f0377f152ce9835bb88" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "test_minigut_sample2_run0_ar2.pair1.truncated.fastq.gz:md5,6ec5c5d6a273c2145096cd8a1488e159", + "test_minigut_sample2_run0_ar2.pair2.truncated.fastq.gz:md5,2c037d5d633998bf4b7006d30dc95610" + ] + ] + ], + "1": [ + + ], + "10": [ + + ], + "11": [ + + ], + "12": [ + + ], + "13": [ + + ], + "14": [ + + ], + "15": [ + + ], + "16": [ + "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", + "versions.yml:md5,eca9f4c6e135848b1247916040e52b99" + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + + ], + "5": [ + + ], + "6": [ + + ], + "7": [ + + ], + "8": [ + + ], + "9": [ + + ], + "assemblies": [ + + ], + "bins": [ + + ], + "busco": [ + + ], + "cat": [ + + ], + "checkm": [ + + ], + "genomad": [ + + ], + "gtdbtk": [ + + ], + "gunc": [ + + ], + "metaeuk": [ + + ], + "prodigal": [ + + ], + "prokka": [ + + ], + "quast_bins": [ + + ], + "refined_bins": [ + + ], + "refined_unbins": [ + + ], + "short_reads": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "test_minigut_1.merged.fastq.gz:md5,3ba44a474c09f2bc8770299fc6d327fb", + "test_minigut_2.merged.fastq.gz:md5,587c61c5f7bf3f0377f152ce9835bb88" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "test_minigut_sample2_run0_ar2.pair1.truncated.fastq.gz:md5,6ec5c5d6a273c2145096cd8a1488e159", + "test_minigut_sample2_run0_ar2.pair2.truncated.fastq.gz:md5,2c037d5d633998bf4b7006d30dc95610" + ] + ] + ], + "unbins": [ + + ], + "versions": [ + "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", + "versions.yml:md5,eca9f4c6e135848b1247916040e52b99" + ] + } + ], + "timestamp": "2023-12-14T11:34:38.395383683" + } +} \ No newline at end of file diff --git a/workflows/mag/tests/test_bbnorm.nf.test b/workflows/mag/tests/test_bbnorm.nf.test new file mode 100644 index 00000000..5a7dd6ba --- /dev/null +++ b/workflows/mag/tests/test_bbnorm.nf.test @@ -0,0 +1,21 @@ +nextflow_workflow { + + name "Test workflow: MAG" + script "../main.nf" + workflow "MAG" + tag "workflows" + tag "mag" + tag "mag_test_bbnorm" + + test("Parameters: bbnorm = true") { + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out, + ).match() } + ) + } + } +} diff --git a/workflows/mag/tests/test_bbnorm.nf.test.snap b/workflows/mag/tests/test_bbnorm.nf.test.snap new file mode 100644 index 00000000..fb848b61 --- /dev/null +++ b/workflows/mag/tests/test_bbnorm.nf.test.snap @@ -0,0 +1,167 @@ +{ + "Parameters: bbnorm = true": { + "content": [ + { + "0": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "test_minigut_1.merged.fastq.gz:md5,fe81462f19e6acc0bbcec55528be7ad2", + "test_minigut_2.merged.fastq.gz:md5,88ebc2ac01082507f7be1761049e5eff" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "test_minigut_sample2_run0_fastp_1.fastp.fastq.gz:md5,25580ab85c8288a6a7a0b2100d93f1d0", + "test_minigut_sample2_run0_fastp_2.fastp.fastq.gz:md5,a91bde6d5c85dd6b47f5bc5cfab65572" + ] + ] + ], + "1": [ + + ], + "10": [ + + ], + "11": [ + + ], + "12": [ + + ], + "13": [ + + ], + "14": [ + + ], + "15": [ + + ], + "16": [ + "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", + "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", + "versions.yml:md5,67cbab915f2801633ea61219086a7735", + "versions.yml:md5,67cbab915f2801633ea61219086a7735", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + + ], + "5": [ + + ], + "6": [ + + ], + "7": [ + + ], + "8": [ + + ], + "9": [ + + ], + "assemblies": [ + + ], + "bins": [ + + ], + "busco": [ + + ], + "cat": [ + + ], + "checkm": [ + + ], + "genomad": [ + + ], + "gtdbtk": [ + + ], + "gunc": [ + + ], + "metaeuk": [ + + ], + "prodigal": [ + + ], + "prokka": [ + + ], + "quast_bins": [ + + ], + "refined_bins": [ + + ], + "refined_unbins": [ + + ], + "short_reads": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "test_minigut_1.merged.fastq.gz:md5,fe81462f19e6acc0bbcec55528be7ad2", + "test_minigut_2.merged.fastq.gz:md5,88ebc2ac01082507f7be1761049e5eff" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "test_minigut_sample2_run0_fastp_1.fastp.fastq.gz:md5,25580ab85c8288a6a7a0b2100d93f1d0", + "test_minigut_sample2_run0_fastp_2.fastp.fastq.gz:md5,a91bde6d5c85dd6b47f5bc5cfab65572" + ] + ] + ], + "unbins": [ + + ], + "versions": [ + "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", + "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", + "versions.yml:md5,67cbab915f2801633ea61219086a7735", + "versions.yml:md5,67cbab915f2801633ea61219086a7735", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + ] + } + ], + "timestamp": "2023-12-14T12:01:36.938067062" + } +} \ No newline at end of file diff --git a/workflows/mag/tests/test_host_rm.nf.test b/workflows/mag/tests/test_host_rm.nf.test new file mode 100644 index 00000000..83ff9862 --- /dev/null +++ b/workflows/mag/tests/test_host_rm.nf.test @@ -0,0 +1,21 @@ +nextflow_workflow { + + name "Test workflow: MAG" + script "../main.nf" + workflow "MAG" + tag "workflows" + tag "mag" + tag "mag_test_host_rm" + + test("Parameters: host_fasta != null") { + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out, + ).match()} + ) + } + } +} diff --git a/workflows/mag/tests/test_host_rm.nf.test.snap b/workflows/mag/tests/test_host_rm.nf.test.snap new file mode 100644 index 00000000..9a90030d --- /dev/null +++ b/workflows/mag/tests/test_host_rm.nf.test.snap @@ -0,0 +1,159 @@ +{ + "Parameters: host_fasta != null": { + "content": [ + { + "0": [ + [ + { + "id": "test_minigut_hg38host", + "group": "0", + "single_end": false + }, + [ + "test_minigut_hg38host_run0_host_removed.unmapped_1.fastq.gz:md5,48ab5cdbf8567cf6023bd304fcf896a9", + "test_minigut_hg38host_run0_host_removed.unmapped_2.fastq.gz:md5,b4246225aa4a15496991ac7c6b58563f" + ] + ], + [ + { + "id": "test_minigut_sample2_hg38host", + "group": "0", + "single_end": false + }, + [ + "test_minigut_sample2_hg38host_run0_host_removed.unmapped_1.fastq.gz:md5,92e68dfac8156b01c547a26cb45e42b1", + "test_minigut_sample2_hg38host_run0_host_removed.unmapped_2.fastq.gz:md5,6ded2b5f426aca4e9b29877fac4e9bff" + ] + ] + ], + "1": [ + + ], + "10": [ + + ], + "11": [ + + ], + "12": [ + + ], + "13": [ + + ], + "14": [ + + ], + "15": [ + + ], + "16": [ + "versions.yml:md5,59050473c9f749b269ad7079fd4693df", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + + ], + "5": [ + + ], + "6": [ + + ], + "7": [ + + ], + "8": [ + + ], + "9": [ + + ], + "assemblies": [ + + ], + "bins": [ + + ], + "busco": [ + + ], + "cat": [ + + ], + "checkm": [ + + ], + "genomad": [ + + ], + "gtdbtk": [ + + ], + "gunc": [ + + ], + "metaeuk": [ + + ], + "prodigal": [ + + ], + "prokka": [ + + ], + "quast_bins": [ + + ], + "refined_bins": [ + + ], + "refined_unbins": [ + + ], + "short_reads": [ + [ + { + "id": "test_minigut_hg38host", + "group": "0", + "single_end": false + }, + [ + "test_minigut_hg38host_run0_host_removed.unmapped_1.fastq.gz:md5,48ab5cdbf8567cf6023bd304fcf896a9", + "test_minigut_hg38host_run0_host_removed.unmapped_2.fastq.gz:md5,b4246225aa4a15496991ac7c6b58563f" + ] + ], + [ + { + "id": "test_minigut_sample2_hg38host", + "group": "0", + "single_end": false + }, + [ + "test_minigut_sample2_hg38host_run0_host_removed.unmapped_1.fastq.gz:md5,92e68dfac8156b01c547a26cb45e42b1", + "test_minigut_sample2_hg38host_run0_host_removed.unmapped_2.fastq.gz:md5,6ded2b5f426aca4e9b29877fac4e9bff" + ] + ] + ], + "unbins": [ + + ], + "versions": [ + "versions.yml:md5,59050473c9f749b269ad7079fd4693df", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + ] + } + ], + "timestamp": "2023-12-14T14:23:51.5082671" + } +} \ No newline at end of file diff --git a/workflows/mag/tests/test_hybrid.nf.test b/workflows/mag/tests/test_hybrid.nf.test new file mode 100644 index 00000000..424bd4af --- /dev/null +++ b/workflows/mag/tests/test_hybrid.nf.test @@ -0,0 +1,21 @@ +nextflow_workflow { + + name "Test workflow: MAG" + script "../main.nf" + workflow "MAG" + tag "workflows" + tag "mag" + tag "mag_test_hybrid" + + test("Parameters: long reads included") { + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out, + ).match()} + ) + } + } +} diff --git a/workflows/mag/tests/test_hybrid.nf.test.snap b/workflows/mag/tests/test_hybrid.nf.test.snap new file mode 100644 index 00000000..1b9556cd --- /dev/null +++ b/workflows/mag/tests/test_hybrid.nf.test.snap @@ -0,0 +1,181 @@ +{ + "Parameters: long reads included": { + "content": [ + { + "0": [ + [ + { + "id": "minigut", + "group": "0", + "single_end": false + }, + [ + "minigut_run0_fastp_1.fastp.fastq.gz:md5,bac85d89ef44638200c247bc941a66fb", + "minigut_run0_fastp_2.fastp.fastq.gz:md5,be6b3eb0d336c333e97d925b0e3a0ff6" + ] + ] + ], + "1": [ + [ + { + "id": "group-0", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-group-0.contigs.fa:md5,c78fa53feb4b35380998bc0c339cdb0e" + ], + [ + { + "id": "group-0", + "group": "0", + "single_end": false, + "assembler": "SPAdesHybrid" + }, + "SPAdesHybrid-group-0_scaffolds.fasta:md5,84c68c6ba721ad5858588a1487e05b6f" + ] + ], + "10": [ + + ], + "11": [ + + ], + "12": [ + + ], + "13": [ + + ], + "14": [ + + ], + "15": [ + + ], + "16": [ + "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", + "versions.yml:md5,1bd6bbec156534880e04e0320dbce063", + "versions.yml:md5,5de8983775d3ef64cdbec6f382852619", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,b94aaba1559ac9deaf7339181c398405", + "versions.yml:md5,cc143ba1b806a335a9963a434c887eba", + "versions.yml:md5,d13508b80e270ac6f1543725b431a682", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + + ], + "5": [ + + ], + "6": [ + + ], + "7": [ + + ], + "8": [ + + ], + "9": [ + + ], + "assemblies": [ + [ + { + "id": "group-0", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-group-0.contigs.fa:md5,c78fa53feb4b35380998bc0c339cdb0e" + ], + [ + { + "id": "group-0", + "group": "0", + "single_end": false, + "assembler": "SPAdesHybrid" + }, + "SPAdesHybrid-group-0_scaffolds.fasta:md5,84c68c6ba721ad5858588a1487e05b6f" + ] + ], + "bins": [ + + ], + "busco": [ + + ], + "cat": [ + + ], + "checkm": [ + + ], + "genomad": [ + + ], + "gtdbtk": [ + + ], + "gunc": [ + + ], + "metaeuk": [ + + ], + "prodigal": [ + + ], + "prokka": [ + + ], + "quast_bins": [ + + ], + "refined_bins": [ + + ], + "refined_unbins": [ + + ], + "short_reads": [ + [ + { + "id": "minigut", + "group": "0", + "single_end": false + }, + [ + "minigut_run0_fastp_1.fastp.fastq.gz:md5,bac85d89ef44638200c247bc941a66fb", + "minigut_run0_fastp_2.fastp.fastq.gz:md5,be6b3eb0d336c333e97d925b0e3a0ff6" + ] + ] + ], + "unbins": [ + + ], + "versions": [ + "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", + "versions.yml:md5,1bd6bbec156534880e04e0320dbce063", + "versions.yml:md5,5de8983775d3ef64cdbec6f382852619", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,b94aaba1559ac9deaf7339181c398405", + "versions.yml:md5,cc143ba1b806a335a9963a434c887eba", + "versions.yml:md5,d13508b80e270ac6f1543725b431a682", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + ] + } + ], + "timestamp": "2023-12-14T14:09:37.526745496" + } +} \ No newline at end of file diff --git a/workflows/mag/tests/test_hybrid_host_rm.nf.test b/workflows/mag/tests/test_hybrid_host_rm.nf.test new file mode 100644 index 00000000..ffa38654 --- /dev/null +++ b/workflows/mag/tests/test_hybrid_host_rm.nf.test @@ -0,0 +1,21 @@ +nextflow_workflow { + + name "Test workflow: MAG" + script "../main.nf" + workflow "MAG" + tag "workflows" + tag "mag" + tag "mag_test_hybrid_host_rm" + + test("Parameters: long reads & host removal included") { + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out, + ).match()} + ) + } + } +} diff --git a/workflows/mag/tests/test_hybrid_host_rm.nf.test.snap b/workflows/mag/tests/test_hybrid_host_rm.nf.test.snap new file mode 100644 index 00000000..b4c23e10 --- /dev/null +++ b/workflows/mag/tests/test_hybrid_host_rm.nf.test.snap @@ -0,0 +1,161 @@ +{ + "Parameters: long reads & host removal included": { + "content": [ + { + "0": [ + [ + { + "id": "minigut", + "group": "0", + "single_end": false + }, + [ + "minigut_run0_host_removed.unmapped_1.fastq.gz:md5,48ab5cdbf8567cf6023bd304fcf896a9", + "minigut_run0_host_removed.unmapped_2.fastq.gz:md5,b4246225aa4a15496991ac7c6b58563f" + ] + ] + ], + "1": [ + [ + { + "id": "minigut", + "group": "0", + "single_end": false, + "assembler": "SPAdesHybrid" + }, + "SPAdesHybrid-minigut_scaffolds.fasta:md5,d3c5e27f011a08daea597f9d2650436d" + ] + ], + "10": [ + + ], + "11": [ + + ], + "12": [ + + ], + "13": [ + + ], + "14": [ + + ], + "15": [ + + ], + "16": [ + "versions.yml:md5,1bd6bbec156534880e04e0320dbce063", + "versions.yml:md5,59050473c9f749b269ad7079fd4693df", + "versions.yml:md5,5de8983775d3ef64cdbec6f382852619", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,b94aaba1559ac9deaf7339181c398405", + "versions.yml:md5,cc143ba1b806a335a9963a434c887eba", + "versions.yml:md5,d13508b80e270ac6f1543725b431a682", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + + ], + "5": [ + + ], + "6": [ + + ], + "7": [ + + ], + "8": [ + + ], + "9": [ + + ], + "assemblies": [ + [ + { + "id": "minigut", + "group": "0", + "single_end": false, + "assembler": "SPAdesHybrid" + }, + "SPAdesHybrid-minigut_scaffolds.fasta:md5,d3c5e27f011a08daea597f9d2650436d" + ] + ], + "bins": [ + + ], + "busco": [ + + ], + "cat": [ + + ], + "checkm": [ + + ], + "genomad": [ + + ], + "gtdbtk": [ + + ], + "gunc": [ + + ], + "metaeuk": [ + + ], + "prodigal": [ + + ], + "prokka": [ + + ], + "quast_bins": [ + + ], + "refined_bins": [ + + ], + "refined_unbins": [ + + ], + "short_reads": [ + [ + { + "id": "minigut", + "group": "0", + "single_end": false + }, + [ + "minigut_run0_host_removed.unmapped_1.fastq.gz:md5,48ab5cdbf8567cf6023bd304fcf896a9", + "minigut_run0_host_removed.unmapped_2.fastq.gz:md5,b4246225aa4a15496991ac7c6b58563f" + ] + ] + ], + "unbins": [ + + ], + "versions": [ + "versions.yml:md5,1bd6bbec156534880e04e0320dbce063", + "versions.yml:md5,59050473c9f749b269ad7079fd4693df", + "versions.yml:md5,5de8983775d3ef64cdbec6f382852619", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,b94aaba1559ac9deaf7339181c398405", + "versions.yml:md5,cc143ba1b806a335a9963a434c887eba", + "versions.yml:md5,d13508b80e270ac6f1543725b431a682", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + ] + } + ], + "timestamp": "2023-12-14T14:52:24.237611909" + } +} \ No newline at end of file diff --git a/workflows/mag/tests/test_nothing.nf.test b/workflows/mag/tests/test_nothing.nf.test new file mode 100644 index 00000000..210cf184 --- /dev/null +++ b/workflows/mag/tests/test_nothing.nf.test @@ -0,0 +1,20 @@ +nextflow_workflow { + + name "Test workflow: MAG" + script "../main.nf" + workflow "MAG" + tag "workflows" + tag "mag" + tag "mag_test_nothing" + + test("Parameters: test_nothing") { + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out + ).match() } + ) + } + } +} diff --git a/workflows/mag/tests/test_nothing.nf.test.snap b/workflows/mag/tests/test_nothing.nf.test.snap new file mode 100644 index 00000000..a1a58ed6 --- /dev/null +++ b/workflows/mag/tests/test_nothing.nf.test.snap @@ -0,0 +1,155 @@ +{ + "Parameters: test_nothing": { + "content": [ + { + "0": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "test_minigut_1.merged.fastq.gz:md5,b7eaf06531000c4d32a48d050c2df34f", + "test_minigut_2.merged.fastq.gz:md5,a1629ec2f2931892a2a0c43f0f677056" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R1.fastq.gz", + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R2.fastq.gz" + ] + ] + ], + "1": [ + + ], + "10": [ + + ], + "11": [ + + ], + "12": [ + + ], + "13": [ + + ], + "14": [ + + ], + "15": [ + + ], + "16": [ + "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + + ], + "5": [ + + ], + "6": [ + + ], + "7": [ + + ], + "8": [ + + ], + "9": [ + + ], + "assemblies": [ + + ], + "bins": [ + + ], + "busco": [ + + ], + "cat": [ + + ], + "checkm": [ + + ], + "genomad": [ + + ], + "gtdbtk": [ + + ], + "gunc": [ + + ], + "metaeuk": [ + + ], + "prodigal": [ + + ], + "prokka": [ + + ], + "quast_bins": [ + + ], + "refined_bins": [ + + ], + "refined_unbins": [ + + ], + "short_reads": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "test_minigut_1.merged.fastq.gz:md5,b7eaf06531000c4d32a48d050c2df34f", + "test_minigut_2.merged.fastq.gz:md5,a1629ec2f2931892a2a0c43f0f677056" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R1.fastq.gz", + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R2.fastq.gz" + ] + ] + ], + "unbins": [ + + ], + "versions": [ + "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + ] + } + ], + "timestamp": "2023-12-14T11:38:30.168015353" + } +} \ No newline at end of file diff --git a/workflows/mag/tests/test_virus_identification.nf.test b/workflows/mag/tests/test_virus_identification.nf.test new file mode 100644 index 00000000..4a49ac43 --- /dev/null +++ b/workflows/mag/tests/test_virus_identification.nf.test @@ -0,0 +1,20 @@ +nextflow_workflow { + + name "Test workflow: MAG" + script "../main.nf" + workflow "MAG" + tag "workflows" + tag "mag" + tag "mag_test_virus_identification" + + test("Parameters: run_virus_identification = true") { + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out + ).match() } + ) + } + } +} diff --git a/workflows/mag/tests/test_virus_identification.nf.test.snap b/workflows/mag/tests/test_virus_identification.nf.test.snap new file mode 100644 index 00000000..a1894fe8 --- /dev/null +++ b/workflows/mag/tests/test_virus_identification.nf.test.snap @@ -0,0 +1,189 @@ +{ + "Parameters: run_virus_identification = true": { + "content": [ + { + "0": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "test_minigut_1.merged.fastq.gz:md5,b7eaf06531000c4d32a48d050c2df34f", + "test_minigut_2.merged.fastq.gz:md5,a1629ec2f2931892a2a0c43f0f677056" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R1.fastq.gz", + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R2.fastq.gz" + ] + ] + ], + "1": [ + [ + { + "id": "group-0", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-group-0.contigs.fa:md5,8419d4d6c72ac1006324a9fee1394646" + ] + ], + "10": [ + + ], + "11": [ + + ], + "12": [ + + ], + "13": [ + + ], + "14": [ + + ], + "15": [ + + ], + "16": [ + "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + ], + "2": [ + + ], + "3": [ + [ + { + "id": "group-0", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,b08861c8a938b3eaf8c1d8e533935042" + ] + ], + "4": [ + + ], + "5": [ + + ], + "6": [ + + ], + "7": [ + + ], + "8": [ + + ], + "9": [ + + ], + "assemblies": [ + [ + { + "id": "group-0", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-group-0.contigs.fa:md5,8419d4d6c72ac1006324a9fee1394646" + ] + ], + "bins": [ + + ], + "busco": [ + + ], + "cat": [ + + ], + "checkm": [ + + ], + "genomad": [ + [ + { + "id": "group-0", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,b08861c8a938b3eaf8c1d8e533935042" + ] + ], + "gtdbtk": [ + + ], + "gunc": [ + + ], + "metaeuk": [ + + ], + "prodigal": [ + + ], + "prokka": [ + + ], + "quast_bins": [ + + ], + "refined_bins": [ + + ], + "refined_unbins": [ + + ], + "short_reads": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "test_minigut_1.merged.fastq.gz:md5,b7eaf06531000c4d32a48d050c2df34f", + "test_minigut_2.merged.fastq.gz:md5,a1629ec2f2931892a2a0c43f0f677056" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R1.fastq.gz", + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R2.fastq.gz" + ] + ] + ], + "unbins": [ + + ], + "versions": [ + "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + ] + } + ], + "timestamp": "2023-12-14T11:48:19.010278472" + } +} \ No newline at end of file From 155a152aa866188602d8141f3e13b8b78bd98f23 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Thu, 14 Dec 2023 15:05:15 -0800 Subject: [PATCH 15/49] Fixed linting and CI --- .github/workflows/ci.yml | 2 +- workflows/mag/tests/tags.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5763fdd5..dcea5671 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,7 @@ jobs: - name: Run nf-test run: | - nf-test test --tag ${{ matrix.tags }} --profile mag_"${{ matrix.tags }},${{ matrix.profile }}" --junitxml=test.xml + nf-test test --tag ${{ matrix.tags }} --profile mag_${{ matrix.tags }},${{ matrix.profile }} --junitxml=test.xml - name: Output log on failure if: failure() diff --git a/workflows/mag/tests/tags.yml b/workflows/mag/tests/tags.yml index 3d9e9033..84857fb4 100644 --- a/workflows/mag/tests/tags.yml +++ b/workflows/mag/tests/tags.yml @@ -12,4 +12,3 @@ test_nothing: - ./** test_virus_identification: - ./** - From 857fb2d2b6469d4e4db51243e405d5fa588725e0 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Thu, 14 Dec 2023 15:07:02 -0800 Subject: [PATCH 16/49] Fixed CI part 2 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcea5671..5be595bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,7 @@ jobs: - name: Run nf-test run: | - nf-test test --tag ${{ matrix.tags }} --profile mag_${{ matrix.tags }},${{ matrix.profile }} --junitxml=test.xml + nf-test test --tag mag_${{ matrix.tags }} --profile ${{ matrix.tags }},${{ matrix.profile }} --junitxml=test.xml - name: Output log on failure if: failure() From 40b4ebff9f0f64bae3b1efbb0dba9ee2d232531f Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Fri, 15 Dec 2023 14:40:05 -0800 Subject: [PATCH 17/49] Added ancient_dna test and reproducibility for binning --- conf/base.config | 2 +- conf/test_ancient_dna.config | 44 ++++++++++--------- nextflow.config | 15 +++++++ workflows/mag/tests/tags.yml | 2 + workflows/mag/tests/test_ancient_dna.nf.test | 34 ++++++++++++++ .../mag/tests/test_ancient_dna.nf.test.snap | 31 +++++++++++++ 6 files changed, 107 insertions(+), 21 deletions(-) create mode 100644 workflows/mag/tests/test_ancient_dna.nf.test create mode 100644 workflows/mag/tests/test_ancient_dna.nf.test.snap diff --git a/conf/base.config b/conf/base.config index 4f4b3d44..714e0eb9 100644 --- a/conf/base.config +++ b/conf/base.config @@ -154,7 +154,7 @@ process { } //returns exit code 247 when running out of memory withName: BOWTIE2_ASSEMBLY_ALIGN { - cpus = { check_max (2 * task.attempt, 'cpus' ) } + cpus = { check_bowtie2_cpus (8, 'cpus' ) } memory = { check_max (8.GB * task.attempt, 'memory' ) } time = { check_max (8.h * task.attempt, 'time' ) } errorStrategy = { task.exitStatus in [143,137,104,134,139,247] ? 'retry' : 'finish' } diff --git a/conf/test_ancient_dna.config b/conf/test_ancient_dna.config index ea672651..26752ad3 100644 --- a/conf/test_ancient_dna.config +++ b/conf/test_ancient_dna.config @@ -15,27 +15,31 @@ params { config_profile_description = 'Minimal test dataset to check pipeline function for ancient DNA step' // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' + //max_cpus = 2 + //max_memory = '6.GB' max_time = '6.h' // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" - kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" - skip_krona = true - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz" - skip_gtdbtk = true - ancient_dna = true - binning_map_mode = 'own' - skip_spades = false - skip_spadeshybrid = true - bcftools_view_high_variant_quality = 0 - bcftools_view_medium_variant_quality = 0 - bcftools_view_minimal_allelesupport = 3 - refine_bins_dastool = true - refine_bins_dastool_threshold = 0 - skip_concoct = true + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + skip_clipping = true + keep_phix = true + kraken2_db = null + centrifuge_db = null + skip_krona = true + megahit_fix_cpu_1 = true + skip_spades = false + spades_fix_cpus = 1 + skip_spadeshybrid = true + ancient_dna = true + skip_quast = true + skip_prodigal = true + bowtie2_fix_cpu_1 = true + binning_map_mode = 'own' + metabat2_fix_cpu_1 = true + maxbin2_fix_cpu_1 = true + concoct_fix_cpu_1 = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true } diff --git a/nextflow.config b/nextflow.config index e5865f94..1e46b28c 100644 --- a/nextflow.config +++ b/nextflow.config @@ -141,6 +141,9 @@ params { metabat_rng_seed = 1 bowtie2_seed = 1 bowtie2_fix_cpu_1 = false + metabat2_fix_cpu_1 = false + maxbin2_fix_cpu_1 = false + concoct_fix_cpu_1 = false // Annotation options skip_metaeuk = false @@ -428,3 +431,15 @@ def check_spadeshybrid_cpus (x, attempt ) { if (params.spadeshybrid_fix_cpus != -1) return check_max (params.spadeshybrid_fix_cpus, 'cpus' ) else return check_max (x * attempt, 'cpus' ) } +def check_metabat2_cpus (x, attempt ) { + if (params.metabat2_fix_cpu_1) return 1 + else return check_max (x * attempt, 'cpus' ) +} +def check_maxbin2_cpus (x, attempt ) { + if (params.maxbin2_fix_cpu_1) return 1 + else return check_max (x * attempt, 'cpus' ) +} +def check_concoct_cpus (x, attempt ) { + if (params.concoct_fix_cpu_1) return 1 + else return check_max (x * attempt, 'cpus' ) +} diff --git a/workflows/mag/tests/tags.yml b/workflows/mag/tests/tags.yml index 84857fb4..7e7fe107 100644 --- a/workflows/mag/tests/tags.yml +++ b/workflows/mag/tests/tags.yml @@ -1,5 +1,7 @@ test_adapterremoval: - ./** +test_ancient_dna: + - ./** test_bbnorm: - ./** test_host_rm: diff --git a/workflows/mag/tests/test_ancient_dna.nf.test b/workflows/mag/tests/test_ancient_dna.nf.test new file mode 100644 index 00000000..01f9c3bb --- /dev/null +++ b/workflows/mag/tests/test_ancient_dna.nf.test @@ -0,0 +1,34 @@ +nextflow_workflow { + + name "Test workflow: MAG" + script "../main.nf" + workflow "MAG" + tag "workflows" + tag "mag" + tag "mag_test_ancient_dna" + + test("Parameters: ancient_dna = true") { + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out. + short_reads, + //workflow.out.assemblies, + //workflow.out.prodigal, + //workflow.out.genomad, + //workflow.out.busco, + //workflow.out.checkm, + //workflow.out.gunc, + //workflow.out.quast_bins, + //workflow.out.cat, + //workflow.out.gtdbtk, + //workflow.out.prokka, + //workflow.out.metaeuk, + //workflow.out.versions + ).match()} + ) + } + } +} diff --git a/workflows/mag/tests/test_ancient_dna.nf.test.snap b/workflows/mag/tests/test_ancient_dna.nf.test.snap new file mode 100644 index 00000000..21ba9f10 --- /dev/null +++ b/workflows/mag/tests/test_ancient_dna.nf.test.snap @@ -0,0 +1,31 @@ +{ + "Parameters: ancient_dna = true": { + "content": [ + [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_R1.fastq.gz", + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_R2.fastq.gz" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R1.fastq.gz", + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R2.fastq.gz" + ] + ] + ] + ], + "timestamp": "2023-12-15T13:36:01.487711685" + } +} \ No newline at end of file From 2188e5951656791919408a2c51cbc52a2cdf9ddb Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Thu, 4 Jan 2024 15:26:53 -0800 Subject: [PATCH 18/49] Added and fixed nf-tests --- bin/combine_tables.py | 2 + bin/summary_busco.py | 2 + conf/base.config | 7 + conf/test.config | 12 +- conf/test_adapterremoval.config | 3 +- conf/test_ancient_dna.config | 54 +- conf/test_bbnorm.config | 2 +- conf/test_binrefinement.config | 41 +- conf/test_busco_auto.config | 31 +- conf/test_virus_identification.config | 6 +- subworkflows/local/busco_qc.nf | 1 - workflows/mag/main.nf | 33 +- workflows/mag/tests/main.nf.test | 21 + workflows/mag/tests/main.nf.test.snap | 543 ++++++++++++++++++ .../tests/test_adapterremoval.nf.test.snap | 68 +-- workflows/mag/tests/test_ancient_dna.nf.test | 15 +- .../mag/tests/test_ancient_dna.nf.test.snap | 406 ++++++++++++- workflows/mag/tests/test_bbnorm.nf.test.snap | 52 +- .../mag/tests/test_binrefinement.nf.test | 21 + .../mag/tests/test_binrefinement.nf.test.snap | 335 +++++++++++ workflows/mag/tests/test_busco_auto.nf.test | 21 + .../mag/tests/test_busco_auto.nf.test.snap | 189 ++++++ workflows/mag/tests/test_host_rm.nf.test.snap | 40 +- workflows/mag/tests/test_hybrid.nf.test.snap | 40 +- .../tests/test_hybrid_host_rm.nf.test.snap | 40 +- workflows/mag/tests/test_nothing.nf.test.snap | 40 +- .../test_virus_identification.nf.test.snap | 40 +- 27 files changed, 1653 insertions(+), 412 deletions(-) create mode 100644 workflows/mag/tests/main.nf.test create mode 100644 workflows/mag/tests/main.nf.test.snap create mode 100644 workflows/mag/tests/test_binrefinement.nf.test create mode 100644 workflows/mag/tests/test_binrefinement.nf.test.snap create mode 100644 workflows/mag/tests/test_busco_auto.nf.test create mode 100644 workflows/mag/tests/test_busco_auto.nf.test.snap diff --git a/bin/combine_tables.py b/bin/combine_tables.py index 46da929b..c4eaeb02 100755 --- a/bin/combine_tables.py +++ b/bin/combine_tables.py @@ -93,6 +93,8 @@ def main(args=None): results, gtdbtk_results, left_on="bin", right_on="user_genome", how="outer" ) # assuming depths for all bins are given + # sort results for reproducibility + results.sort_values(by='bin', inplace=True, ignore_index=True) results.to_csv(args.out, sep="\t") diff --git a/bin/summary_busco.py b/bin/summary_busco.py index b4a8c99b..faff6718 100755 --- a/bin/summary_busco.py +++ b/bin/summary_busco.py @@ -186,6 +186,8 @@ def main(args=None): else: df_final = df_specific.append(df_failed) + # sort output file for reproducibility + df_final.sort_values(by='GenomeBin', inplace=True) df_final.to_csv(args.out, sep="\t", index=False) diff --git a/conf/base.config b/conf/base.config index 714e0eb9..172df705 100644 --- a/conf/base.config +++ b/conf/base.config @@ -129,6 +129,13 @@ process { time = { check_max (16.h * task.attempt, 'time' ) } errorStrategy = { task.exitStatus in [143,137,104,134,139,250] ? 'retry' : 'finish' } } + //bowtie2 returns exit code 250 when running out of memory + withName: BOWTIE2_PHIX_REMOVAL_ALIGN { + cpus = { check_bowtie2_cpus (8, task.attempt ) } + memory = { check_max (40.GB * task.attempt, 'memory' ) } + time = { check_max (16.h * task.attempt, 'time' ) } + errorStrategy = { task.exitStatus in [143,137,104,134,139,250] ? 'retry' : 'finish' } + } //MEGAHIT returns exit code 250 when running out of memory withName: MEGAHIT { cpus = { check_megahit_cpus (8, task.attempt ) } diff --git a/conf/test.config b/conf/test.config index 4168c871..5968e844 100644 --- a/conf/test.config +++ b/conf/test.config @@ -24,14 +24,16 @@ params { centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" skip_krona = true - min_length_unbinned_contigs = 1 + megahit_fix_cpu_1 = true + bowtie2_fix_cpu_1 = true + metabat2_fix_cpu_1 = true + maxbin2_fix_cpu_1 = true + concoct_fix_cpu_1 = true + binning_map_mode = 'own' + min_length_unbinned_contigs = 1000000 max_unbinned_contigs = 2 busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz" busco_clean = true skip_gtdbtk = true skip_concoct = true - megahit_fix_cpu_1 = true - spades_fix_cpus = 2 - spadeshybrid_fix_cpus = 2 - metabat_rng_seed = 1 } diff --git a/conf/test_adapterremoval.config b/conf/test_adapterremoval.config index 54bb7959..07b02a6c 100644 --- a/conf/test_adapterremoval.config +++ b/conf/test_adapterremoval.config @@ -20,10 +20,9 @@ params { max_time = '6.h' // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.euk.csv' clip_tool = 'adapterremoval' keep_phix = true - skip_adapter_trimming = true centrifuge_db = null kraken2_db = null skip_krona = true diff --git a/conf/test_ancient_dna.config b/conf/test_ancient_dna.config index 26752ad3..dcdda8fa 100644 --- a/conf/test_ancient_dna.config +++ b/conf/test_ancient_dna.config @@ -15,31 +15,37 @@ params { config_profile_description = 'Minimal test dataset to check pipeline function for ancient DNA step' // Limit resources so that this can run on GitHub Actions - //max_cpus = 2 - //max_memory = '6.GB' + max_cpus = 2 + max_memory = '6.GB' max_time = '6.h' // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - skip_clipping = true - keep_phix = true - kraken2_db = null - centrifuge_db = null - skip_krona = true - megahit_fix_cpu_1 = true - skip_spades = false - spades_fix_cpus = 1 - skip_spadeshybrid = true - ancient_dna = true - skip_quast = true - skip_prodigal = true - bowtie2_fix_cpu_1 = true - binning_map_mode = 'own' - metabat2_fix_cpu_1 = true - maxbin2_fix_cpu_1 = true - concoct_fix_cpu_1 = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + skip_clipping = true + keep_phix = true + kraken2_db = null + centrifuge_db = null + skip_krona = true + megahit_fix_cpu_1 = true + spades_fix_cpus = 1 + skip_spadeshybrid = true + ancient_dna = true + skip_quast = true + skip_prodigal = true + bowtie2_fix_cpu_1 = true + binning_map_mode = 'own' + metabat2_fix_cpu_1 = true + maxbin2_fix_cpu_1 = true + concoct_fix_cpu_1 = true + bcftools_view_high_variant_quality = 0 + bcftools_view_medium_variant_quality = 0 + bcftools_view_minimal_allelesupport = 3 + refine_bins_dastool = true + refine_bins_dastool_threshold = 0 + min_length_unbinned_contigs = 1 + max_unbinned_contigs = 2 + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true } diff --git a/conf/test_bbnorm.config b/conf/test_bbnorm.config index 8e7d6e94..c3fb8518 100644 --- a/conf/test_bbnorm.config +++ b/conf/test_bbnorm.config @@ -20,7 +20,7 @@ params { max_time = '6.h' // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' bbnorm = true keep_phix = true skip_adapter_trimming = true diff --git a/conf/test_binrefinement.config b/conf/test_binrefinement.config index bc1796d5..969dfc2c 100644 --- a/conf/test_binrefinement.config +++ b/conf/test_binrefinement.config @@ -20,18 +20,31 @@ params { max_time = '6.h' // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - assembly_input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/assembly_samplesheet.csv' - centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" - kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" - skip_krona = true - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2020-03-06.tar.gz" - skip_gtdbtk = true - refine_bins_dastool = true - refine_bins_dastool_threshold = 0 - // TODO not using 'both' until #489 merged - postbinning_input = 'refined_bins_only' - busco_clean = true + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + assembly_input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/assembly_samplesheet.csv' + bbnorm = true + keep_phix = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_quast = true + skip_prodigal = true + bowtie2_fix_cpu_1 = true + metabat2_fix_cpu_1 = true + maxbin2_fix_cpu_1 = true + concoct_fix_cpu_1 = true + binning_map_mode = 'own' + min_length_unbinned_contigs = 1 + max_unbinned_contigs = 2 + refine_bins_dastool = true + refine_bins_dastool_threshold = 0 + // TODO not using 'both' until #489 merged + postbinning_input = 'refined_bins_only' + skip_busco = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true + } diff --git a/conf/test_busco_auto.config b/conf/test_busco_auto.config index 6479012f..d99ea24b 100644 --- a/conf/test_busco_auto.config +++ b/conf/test_busco_auto.config @@ -20,13 +20,26 @@ params { max_time = '6.h' // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - skip_spades = true - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - skip_gtdbtk = true - skip_prokka = true - skip_prodigal = true - skip_quast = true - skip_concoct = true + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + keep_phix = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + megahit_fix_cpu_1 = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + bowtie2_fix_cpu_1 = true + metabat2_fix_cpu_1 = true + maxbin2_fix_cpu_1 = true + concoct_fix_cpu_1 = true + binning_map_mode = 'own' + min_length_unbinned_contigs = 1 + max_unbinned_contigs = 2 + skip_concoct = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true } diff --git a/conf/test_virus_identification.config b/conf/test_virus_identification.config index 635edb50..940f7e1d 100644 --- a/conf/test_virus_identification.config +++ b/conf/test_virus_identification.config @@ -21,10 +21,6 @@ params { // Input data input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' - run_virus_identification = true - genomad_splits = 7 - - // For computational efficiency keep_phix = true skip_clipping = true skip_adapter_trimming = true @@ -36,6 +32,8 @@ params { skip_spades = true skip_spadeshybrid = true skip_quast = true + run_virus_identification = true + genomad_splits = 7 skip_prodigal = true skip_binning = true skip_binqc = true diff --git a/subworkflows/local/busco_qc.nf b/subworkflows/local/busco_qc.nf index 15ab8cfa..f2686f62 100644 --- a/subworkflows/local/busco_qc.nf +++ b/subworkflows/local/busco_qc.nf @@ -76,7 +76,6 @@ workflow BUSCO_QC { ) emit: - summary_specific = BUSCO.out.summary_specific summary = BUSCO_SUMMARY.out.summary failed_bin = BUSCO.out.failed_bin.map{it[1]} multiqc = BUSCO.out.summary_domain.mix(BUSCO.out.summary_specific).map{it[1]} diff --git a/workflows/mag/main.nf b/workflows/mag/main.nf index 7ad1d9a5..880d41bf 100644 --- a/workflows/mag/main.nf +++ b/workflows/mag/main.nf @@ -225,10 +225,11 @@ if(params.metaeuk_db && !params.skip_metaeuk) { ch_binning_results_unbins = Channel.empty() ch_refined_bins = Channel.empty() ch_refined_unbins = Channel.empty() - ch_busco_summary_specific = Channel.empty() + ch_busco_summary = Channel.empty() ch_checkm_tsv = Channel.empty() ch_gunc_maxcss_level_tsv = Channel.empty() ch_quast_bin_summaries = Channel.empty() + ch_bin_summaries = Channel.empty() ch_cat_tax_classification_names = Channel.empty() ch_gtdbtk_summaries = Channel.empty() ch_prokka_faa = Channel.empty() @@ -881,7 +882,6 @@ workflow MAG { ch_input_bins_for_qc ) ch_busco_summary = BUSCO_QC.out.summary - ch_busco_summary_specific = BUSCO_QC.out.summary_specific ch_versions = ch_versions.mix(BUSCO_QC.out.versions.first()) // process information if BUSCO analysis failed for individual bins due to no matching genes BUSCO_QC.out @@ -1000,6 +1000,7 @@ workflow MAG { ch_quast_bins_summary.ifEmpty([]), ch_gtdbtk_summary.ifEmpty([]) ) + ch_bin_summaries = BIN_SUMMARY.out.summary } /* @@ -1120,39 +1121,13 @@ workflow MAG { emit: short_reads = ch_short_reads - // fastqc_raw = FASTQC_RAW.out.json - // adapterremoval_se = ADAPTERREMOVAL_SE.out.singles_truncated - // adapterremoval_pe = ADAPTERREMOVAL_PE.out.paired_truncated - // host_rm = BOWTIE2_HOST_REMOVAL_ALIGN.out.reads - // phix_rm = BOWTIE2_PHIX_REMOVAL_ALIGN.out.reads - // fastqc_trimmed = FASTQC_TRIMMED.out.json - // cat_fastq = CAT_FASTQ.out.reads - // seqtk = SEQTK_MERGEPE.out.reads - // bbmap = BBMAP_BBNORM.out.fastq - // nanoplot_raw = NANOPLOT_RAW.out.txt - // porechop = PORECHOP.out.reads - // nanolyse = NANOLYSE.out.reads - // filtlong = FILTLONG.out.reads - // nanoplot_filtered = NANOPLOT_FILTERED.out.txt - // kraken2 = KRAKEN2.out.report - // centrifuge = CENTRIFUGE.out.report assemblies = ch_assemblies - // megahit = ch_megahit_assemblies - // spades = ch_spades_assemblies - // spadeshybrid = ch_spadeshybrid_assemblies - // quast_contigs = QUAST.out.report prodigal = ch_prodigal_gene_annotations genomad = ch_genomad_virus_summary - bins = ch_binning_results_bins - unbins = ch_binning_results_unbins refined_bins = ch_refined_bins refined_unbins = ch_refined_unbins - busco = ch_busco_summary_specific - checkm = ch_checkm_tsv - gunc = ch_gunc_maxcss_level_tsv - quast_bins = ch_quast_bin_summaries + bin_summary = ch_bin_summaries cat = ch_cat_tax_classification_names - gtdbtk = ch_gtdbtk_summaries prokka = ch_prokka_faa metaeuk = ch_metaeuk_easypredict_faa versions = ch_versions diff --git a/workflows/mag/tests/main.nf.test b/workflows/mag/tests/main.nf.test new file mode 100644 index 00000000..b90b09b2 --- /dev/null +++ b/workflows/mag/tests/main.nf.test @@ -0,0 +1,21 @@ +nextflow_workflow { + + name "Test workflow: MAG" + script "../main.nf" + workflow "MAG" + tag "workflows" + tag "mag" + tag "mag_test" + + test("Default paramters") { + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out + ).match() } + ) + } + } +} diff --git a/workflows/mag/tests/main.nf.test.snap b/workflows/mag/tests/main.nf.test.snap new file mode 100644 index 00000000..1202ab2d --- /dev/null +++ b/workflows/mag/tests/main.nf.test.snap @@ -0,0 +1,543 @@ +{ + "Default paramters": { + "content": [ + { + "0": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "test_minigut_1.merged.fastq.gz:md5,fe81462f19e6acc0bbcec55528be7ad2", + "test_minigut_2.merged.fastq.gz:md5,88ebc2ac01082507f7be1761049e5eff" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "test_minigut_sample2_run0_phix_removed.unmapped_1.fastq.gz:md5,25580ab85c8288a6a7a0b2100d93f1d0", + "test_minigut_sample2_run0_phix_removed.unmapped_2.fastq.gz:md5,a91bde6d5c85dd6b47f5bc5cfab65572" + ] + ] + ], + "1": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut.contigs.fa:md5,f35393cdbcb64bdc7ae9db78a5601229" + ], + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "SPAdes" + }, + "SPAdes-test_minigut_scaffolds.fasta:md5,8bd449b393a7d3937ff74a24ce39cfc1" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut_sample2.contigs.fa:md5,2c0b7977b39fb2db31ee9b3adf75f8c6" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "SPAdes" + }, + "SPAdes-test_minigut_sample2_scaffolds.fasta:md5,bf8ebad421409a0dee62a7b14acfca88" + ] + ], + "10": [ + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", + "versions.yml:md5,18eeebfdf8c302c29cbb4abc4325628c", + "versions.yml:md5,200880072147033d455fede5cf4499dc", + "versions.yml:md5,2acb240e473bc70de2e0bb8a392748ac", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", + "versions.yml:md5,3550550b622b6d5f6803ffed9742e08b", + "versions.yml:md5,368d757df0f57b49bce09965641cc71a", + "versions.yml:md5,519a472d661281a99e0335305cdd1d85", + "versions.yml:md5,59e59108b091ee58d131ac74e9bfac43", + "versions.yml:md5,5cf45e79d026b3593199fa035f2d72ec", + "versions.yml:md5,7017c8959eb098440346ca9a2310c460", + "versions.yml:md5,7067399ef1687d4d1ba0aab93dc67754", + "versions.yml:md5,8187d1683f97b64a582620caba2b175a", + "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", + "versions.yml:md5,ea3029fd33017b2f39de14bce337c246" + ], + "2": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "test_minigut.gff.gz:md5,d90d8730f41d31fdecf7571555b598fe" + ], + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "SPAdes" + }, + "test_minigut.gff.gz:md5,80e8726ecc2d146187cfa2aa4a748d84" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "test_minigut_sample2.gff.gz:md5,3ee13f148386668e88c18a4cd6a14ad3" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "SPAdes" + }, + "test_minigut_sample2.gff.gz:md5,1e099a6a3def0c73010d7a1e138c8047" + ] + ], + "3": [ + + ], + "4": [ + + ], + "5": [ + + ], + "6": [ + "bin_summary.tsv:md5,cb825146f68ff4d1722fc7064ae9017b" + ], + "7": [ + + ], + "8": [ + [ + { + "id": "MEGAHIT-MaxBin2-test_minigut.001", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT", + "binner": "MaxBin2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "MEGAHIT-MaxBin2-test_minigut.001.faa:md5,eaccd450eed58eae731384315670ca58" + ], + [ + { + "id": "MEGAHIT-MaxBin2-test_minigut.002", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT", + "binner": "MaxBin2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "MEGAHIT-MaxBin2-test_minigut.002.faa:md5,78099bcfdbc895f9a9db06cd15f5edab" + ], + [ + { + "id": "MEGAHIT-MetaBAT2-test_minigut.1", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT", + "binner": "MetaBAT2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "MEGAHIT-MetaBAT2-test_minigut.1.faa:md5,b9bd97482ed409c39f8685a9bc898f01" + ], + [ + { + "id": "MEGAHIT-MetaBAT2-test_minigut.2", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT", + "binner": "MetaBAT2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "MEGAHIT-MetaBAT2-test_minigut.2.faa:md5,b3c4338146bcc7fb36a8c94d92922f5e" + ], + [ + { + "id": "SPAdes-MaxBin2-test_minigut.001", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "binner": "MaxBin2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "SPAdes-MaxBin2-test_minigut.001.faa:md5,7e773008946a8eeb529cabc153d197b5" + ], + [ + { + "id": "SPAdes-MaxBin2-test_minigut.002", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "binner": "MaxBin2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "SPAdes-MaxBin2-test_minigut.002.faa:md5,846a091d4f412975bcd66c5ef0ed49d9" + ], + [ + { + "id": "SPAdes-MaxBin2-test_minigut_sample2.001", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "binner": "MaxBin2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "SPAdes-MaxBin2-test_minigut_sample2.001.faa:md5,e33d7a55af8b070646de60f62d407481" + ], + [ + { + "id": "SPAdes-MetaBAT2-test_minigut.1", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "binner": "MetaBAT2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "SPAdes-MetaBAT2-test_minigut.1.faa:md5,2879b7a33cca4024e3c518b2fa4bc063" + ], + [ + { + "id": "SPAdes-MetaBAT2-test_minigut.2", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "binner": "MetaBAT2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "SPAdes-MetaBAT2-test_minigut.2.faa:md5,1cc11d484e49ff51111b0e0bf7a2349b" + ] + ], + "9": [ + + ], + "assemblies": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut.contigs.fa:md5,f35393cdbcb64bdc7ae9db78a5601229" + ], + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "SPAdes" + }, + "SPAdes-test_minigut_scaffolds.fasta:md5,8bd449b393a7d3937ff74a24ce39cfc1" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut_sample2.contigs.fa:md5,2c0b7977b39fb2db31ee9b3adf75f8c6" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "SPAdes" + }, + "SPAdes-test_minigut_sample2_scaffolds.fasta:md5,bf8ebad421409a0dee62a7b14acfca88" + ] + ], + "bin_summary": [ + "bin_summary.tsv:md5,cb825146f68ff4d1722fc7064ae9017b" + ], + "cat": [ + + ], + "genomad": [ + + ], + "metaeuk": [ + + ], + "prodigal": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "test_minigut.gff.gz:md5,d90d8730f41d31fdecf7571555b598fe" + ], + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "SPAdes" + }, + "test_minigut.gff.gz:md5,80e8726ecc2d146187cfa2aa4a748d84" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "test_minigut_sample2.gff.gz:md5,3ee13f148386668e88c18a4cd6a14ad3" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "SPAdes" + }, + "test_minigut_sample2.gff.gz:md5,1e099a6a3def0c73010d7a1e138c8047" + ] + ], + "prokka": [ + [ + { + "id": "MEGAHIT-MaxBin2-test_minigut.001", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT", + "binner": "MaxBin2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "MEGAHIT-MaxBin2-test_minigut.001.faa:md5,eaccd450eed58eae731384315670ca58" + ], + [ + { + "id": "MEGAHIT-MaxBin2-test_minigut.002", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT", + "binner": "MaxBin2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "MEGAHIT-MaxBin2-test_minigut.002.faa:md5,78099bcfdbc895f9a9db06cd15f5edab" + ], + [ + { + "id": "MEGAHIT-MetaBAT2-test_minigut.1", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT", + "binner": "MetaBAT2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "MEGAHIT-MetaBAT2-test_minigut.1.faa:md5,b9bd97482ed409c39f8685a9bc898f01" + ], + [ + { + "id": "MEGAHIT-MetaBAT2-test_minigut.2", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT", + "binner": "MetaBAT2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "MEGAHIT-MetaBAT2-test_minigut.2.faa:md5,b3c4338146bcc7fb36a8c94d92922f5e" + ], + [ + { + "id": "SPAdes-MaxBin2-test_minigut.001", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "binner": "MaxBin2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "SPAdes-MaxBin2-test_minigut.001.faa:md5,7e773008946a8eeb529cabc153d197b5" + ], + [ + { + "id": "SPAdes-MaxBin2-test_minigut.002", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "binner": "MaxBin2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "SPAdes-MaxBin2-test_minigut.002.faa:md5,846a091d4f412975bcd66c5ef0ed49d9" + ], + [ + { + "id": "SPAdes-MaxBin2-test_minigut_sample2.001", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "binner": "MaxBin2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "SPAdes-MaxBin2-test_minigut_sample2.001.faa:md5,e33d7a55af8b070646de60f62d407481" + ], + [ + { + "id": "SPAdes-MetaBAT2-test_minigut.1", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "binner": "MetaBAT2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "SPAdes-MetaBAT2-test_minigut.1.faa:md5,2879b7a33cca4024e3c518b2fa4bc063" + ], + [ + { + "id": "SPAdes-MetaBAT2-test_minigut.2", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "binner": "MetaBAT2", + "domain": "unclassified", + "refinement": "unrefined" + }, + "SPAdes-MetaBAT2-test_minigut.2.faa:md5,1cc11d484e49ff51111b0e0bf7a2349b" + ] + ], + "refined_bins": [ + + ], + "refined_unbins": [ + + ], + "short_reads": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "test_minigut_1.merged.fastq.gz:md5,fe81462f19e6acc0bbcec55528be7ad2", + "test_minigut_2.merged.fastq.gz:md5,88ebc2ac01082507f7be1761049e5eff" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "test_minigut_sample2_run0_phix_removed.unmapped_1.fastq.gz:md5,25580ab85c8288a6a7a0b2100d93f1d0", + "test_minigut_sample2_run0_phix_removed.unmapped_2.fastq.gz:md5,a91bde6d5c85dd6b47f5bc5cfab65572" + ] + ] + ], + "versions": [ + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", + "versions.yml:md5,18eeebfdf8c302c29cbb4abc4325628c", + "versions.yml:md5,200880072147033d455fede5cf4499dc", + "versions.yml:md5,2acb240e473bc70de2e0bb8a392748ac", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", + "versions.yml:md5,3550550b622b6d5f6803ffed9742e08b", + "versions.yml:md5,368d757df0f57b49bce09965641cc71a", + "versions.yml:md5,519a472d661281a99e0335305cdd1d85", + "versions.yml:md5,59e59108b091ee58d131ac74e9bfac43", + "versions.yml:md5,5cf45e79d026b3593199fa035f2d72ec", + "versions.yml:md5,7017c8959eb098440346ca9a2310c460", + "versions.yml:md5,7067399ef1687d4d1ba0aab93dc67754", + "versions.yml:md5,8187d1683f97b64a582620caba2b175a", + "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", + "versions.yml:md5,ea3029fd33017b2f39de14bce337c246" + ] + } + ], + "timestamp": "2024-01-04T14:50:41.377338936" + } +} \ No newline at end of file diff --git a/workflows/mag/tests/test_adapterremoval.nf.test.snap b/workflows/mag/tests/test_adapterremoval.nf.test.snap index b5b418b1..7e77be93 100644 --- a/workflows/mag/tests/test_adapterremoval.nf.test.snap +++ b/workflows/mag/tests/test_adapterremoval.nf.test.snap @@ -5,24 +5,24 @@ "0": [ [ { - "id": "test_minigut", + "id": "test_minigut_euk", "group": "0", "single_end": false }, [ - "test_minigut_1.merged.fastq.gz:md5,3ba44a474c09f2bc8770299fc6d327fb", - "test_minigut_2.merged.fastq.gz:md5,587c61c5f7bf3f0377f152ce9835bb88" + "test_minigut_euk_run0_ar2.pair1.truncated.fastq.gz:md5,e8f01a4fdf2e348330fc02a47eb3c5af", + "test_minigut_euk_run0_ar2.pair2.truncated.fastq.gz:md5,06510940505288d80a3d940db85d0ed5" ] ], [ { - "id": "test_minigut_sample2", + "id": "test_minigut_sample2_euk", "group": "0", "single_end": false }, [ - "test_minigut_sample2_run0_ar2.pair1.truncated.fastq.gz:md5,6ec5c5d6a273c2145096cd8a1488e159", - "test_minigut_sample2_run0_ar2.pair2.truncated.fastq.gz:md5,2c037d5d633998bf4b7006d30dc95610" + "test_minigut_sample2_euk_run0_ar2.pair1.truncated.fastq.gz:md5,92b8643e591f7490c980fe4d0879c955", + "test_minigut_sample2_euk_run0_ar2.pair2.truncated.fastq.gz:md5,7d4436e9532bb9665080b77efe441faf" ] ] ], @@ -30,26 +30,6 @@ ], "10": [ - - ], - "11": [ - - ], - "12": [ - - ], - "13": [ - - ], - "14": [ - - ], - "15": [ - - ], - "16": [ - "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", @@ -82,26 +62,14 @@ "assemblies": [ ], - "bins": [ - - ], - "busco": [ + "bin_summary": [ ], "cat": [ - ], - "checkm": [ - ], "genomad": [ - ], - "gtdbtk": [ - - ], - "gunc": [ - ], "metaeuk": [ @@ -111,9 +79,6 @@ ], "prokka": [ - ], - "quast_bins": [ - ], "refined_bins": [ @@ -124,33 +89,28 @@ "short_reads": [ [ { - "id": "test_minigut", + "id": "test_minigut_euk", "group": "0", "single_end": false }, [ - "test_minigut_1.merged.fastq.gz:md5,3ba44a474c09f2bc8770299fc6d327fb", - "test_minigut_2.merged.fastq.gz:md5,587c61c5f7bf3f0377f152ce9835bb88" + "test_minigut_euk_run0_ar2.pair1.truncated.fastq.gz:md5,e8f01a4fdf2e348330fc02a47eb3c5af", + "test_minigut_euk_run0_ar2.pair2.truncated.fastq.gz:md5,06510940505288d80a3d940db85d0ed5" ] ], [ { - "id": "test_minigut_sample2", + "id": "test_minigut_sample2_euk", "group": "0", "single_end": false }, [ - "test_minigut_sample2_run0_ar2.pair1.truncated.fastq.gz:md5,6ec5c5d6a273c2145096cd8a1488e159", - "test_minigut_sample2_run0_ar2.pair2.truncated.fastq.gz:md5,2c037d5d633998bf4b7006d30dc95610" + "test_minigut_sample2_euk_run0_ar2.pair1.truncated.fastq.gz:md5,92b8643e591f7490c980fe4d0879c955", + "test_minigut_sample2_euk_run0_ar2.pair2.truncated.fastq.gz:md5,7d4436e9532bb9665080b77efe441faf" ] ] - ], - "unbins": [ - ], "versions": [ - "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", @@ -158,6 +118,6 @@ ] } ], - "timestamp": "2023-12-14T11:34:38.395383683" + "timestamp": "2024-01-02T11:42:16.890174222" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_ancient_dna.nf.test b/workflows/mag/tests/test_ancient_dna.nf.test index 01f9c3bb..b0023c14 100644 --- a/workflows/mag/tests/test_ancient_dna.nf.test +++ b/workflows/mag/tests/test_ancient_dna.nf.test @@ -13,20 +13,7 @@ nextflow_workflow { assertAll( { assert workflow.success }, { assert snapshot( - workflow.out. - short_reads, - //workflow.out.assemblies, - //workflow.out.prodigal, - //workflow.out.genomad, - //workflow.out.busco, - //workflow.out.checkm, - //workflow.out.gunc, - //workflow.out.quast_bins, - //workflow.out.cat, - //workflow.out.gtdbtk, - //workflow.out.prokka, - //workflow.out.metaeuk, - //workflow.out.versions + workflow.out ).match()} ) } diff --git a/workflows/mag/tests/test_ancient_dna.nf.test.snap b/workflows/mag/tests/test_ancient_dna.nf.test.snap index 21ba9f10..c6c3e6b3 100644 --- a/workflows/mag/tests/test_ancient_dna.nf.test.snap +++ b/workflows/mag/tests/test_ancient_dna.nf.test.snap @@ -1,31 +1,399 @@ { "Parameters: ancient_dna = true": { "content": [ - [ - [ - { - "id": "test_minigut", - "group": "0", - "single_end": false - }, + { + "0": [ [ - "/nf-core/test-datasets/raw/mag/test_data/test_minigut_R1.fastq.gz", - "/nf-core/test-datasets/raw/mag/test_data/test_minigut_R2.fastq.gz" + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_R1.fastq.gz", + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_R2.fastq.gz" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R1.fastq.gz", + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R2.fastq.gz" + ] + ] + ], + "1": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut.contigs.fa:md5,3055734183fc34cba4b50afbda4f34cf" + ], + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "SPAdes" + }, + "SPAdes-test_minigut_scaffolds.fasta:md5,acb79980009afc87e0e81293a84fe54f" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut_sample2.contigs.fa:md5,093b87fe274ddb7f7dbf5d5e574495ad" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "SPAdes" + }, + "SPAdes-test_minigut_sample2_scaffolds.fasta:md5,98a1c0bf47c301bbd838b32a54b7537b" + ] + ], + "10": [ + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,3550550b622b6d5f6803ffed9742e08b", + "versions.yml:md5,368d757df0f57b49bce09965641cc71a", + "versions.yml:md5,47df50b983a036f55cb936247d920744", + "versions.yml:md5,586dc48e8e2cc8106cb0ee3b70ce0fef", + "versions.yml:md5,6d41fb977d31ce96d7ddbb4ea2dbe38b", + "versions.yml:md5,7017c8959eb098440346ca9a2310c460", + "versions.yml:md5,7ccffaba4e75be98b8faedb54ccea16f", + "versions.yml:md5,8187d1683f97b64a582620caba2b175a", + "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", + "versions.yml:md5,aee39cb21305f6d3fe88bf9863529706", + "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", + "versions.yml:md5,c777f00c6a48f3c0490d09b1aa8d81a6", + "versions.yml:md5,cc4a4972327dec07faf1f54ead653715", + "versions.yml:md5,d8d2321bec0c70de2f9750f258fc890d", + "versions.yml:md5,dc29f891cb297131cf940ba5580a6832", + "versions.yml:md5,df654dd4fdb4039c2d8cf0b6b49f4b55", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", + "versions.yml:md5,e9f8fe610b5abe68b28d77efc951e9e6" + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT", + "binner": "DASTool", + "refinement": "dastool_refined", + "domain": "unclassified" + }, + [ + "MEGAHIT-MetaBAT2Refined-test_minigut.1.fa:md5,c010aca18e5dfa00f17b24ce507988f3", + "MEGAHIT-MetaBAT2Refined-test_minigut.2.fa:md5,2cc38f264e02be8fab0867b1dfe9f9bd" + ] + ], + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "binner": "DASTool", + "refinement": "dastool_refined", + "domain": "unclassified" + }, + [ + "SPAdes-MaxBin2Refined-test_minigut.001.fa:md5,1e01c5c341311b94df0f5b0b97668cde", + "SPAdes-MaxBin2Refined-test_minigut.002.fa:md5,ca74ae68469c1e354a00d7a1bf0416d0" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "binner": "DASTool", + "refinement": "dastool_refined", + "domain": "unclassified" + }, + [ + "SPAdes-MaxBin2Refined-test_minigut_sample2.001.fa:md5,7c8439fb9c3b72ba4d89b9b61b538a66" + ] + ] + ], + "5": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT", + "refinement": "dastool_refined_unbinned", + "binner": "DASTool", + "domain": "unclassified" + }, + "MEGAHIT-DASToolUnbinned-test_minigut.fa:md5,4b022bfb579d9873385aadc9eaf549d6" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "refinement": "dastool_refined_unbinned", + "binner": "DASTool", + "domain": "unclassified" + }, + "SPAdes-DASToolUnbinned-test_minigut_sample2.fa:md5,dd179320159df68d6bf630d6b46079b5" + ] + ], + "6": [ + + ], + "7": [ + + ], + "8": [ + + ], + "9": [ + + ], + "assemblies": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut.contigs.fa:md5,3055734183fc34cba4b50afbda4f34cf" + ], + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "SPAdes" + }, + "SPAdes-test_minigut_scaffolds.fasta:md5,acb79980009afc87e0e81293a84fe54f" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut_sample2.contigs.fa:md5,093b87fe274ddb7f7dbf5d5e574495ad" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "SPAdes" + }, + "SPAdes-test_minigut_sample2_scaffolds.fasta:md5,98a1c0bf47c301bbd838b32a54b7537b" ] ], - [ - { - "id": "test_minigut_sample2", - "group": "0", - "single_end": false - }, + "bin_summary": [ + + ], + "cat": [ + + ], + "genomad": [ + + ], + "metaeuk": [ + + ], + "prodigal": [ + + ], + "prokka": [ + + ], + "refined_bins": [ [ - "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R1.fastq.gz", - "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R2.fastq.gz" + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT", + "binner": "DASTool", + "refinement": "dastool_refined", + "domain": "unclassified" + }, + [ + "MEGAHIT-MetaBAT2Refined-test_minigut.1.fa:md5,c010aca18e5dfa00f17b24ce507988f3", + "MEGAHIT-MetaBAT2Refined-test_minigut.2.fa:md5,2cc38f264e02be8fab0867b1dfe9f9bd" + ] + ], + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "binner": "DASTool", + "refinement": "dastool_refined", + "domain": "unclassified" + }, + [ + "SPAdes-MaxBin2Refined-test_minigut.001.fa:md5,1e01c5c341311b94df0f5b0b97668cde", + "SPAdes-MaxBin2Refined-test_minigut.002.fa:md5,ca74ae68469c1e354a00d7a1bf0416d0" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "binner": "DASTool", + "refinement": "dastool_refined", + "domain": "unclassified" + }, + [ + "SPAdes-MaxBin2Refined-test_minigut_sample2.001.fa:md5,7c8439fb9c3b72ba4d89b9b61b538a66" + ] + ] + ], + "refined_unbins": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT", + "refinement": "dastool_refined_unbinned", + "binner": "DASTool", + "domain": "unclassified" + }, + "MEGAHIT-DASToolUnbinned-test_minigut.fa:md5,4b022bfb579d9873385aadc9eaf549d6" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "SPAdes", + "refinement": "dastool_refined_unbinned", + "binner": "DASTool", + "domain": "unclassified" + }, + "SPAdes-DASToolUnbinned-test_minigut_sample2.fa:md5,dd179320159df68d6bf630d6b46079b5" ] + ], + "short_reads": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_R1.fastq.gz", + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_R2.fastq.gz" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R1.fastq.gz", + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R2.fastq.gz" + ] + ] + ], + "versions": [ + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,3550550b622b6d5f6803ffed9742e08b", + "versions.yml:md5,368d757df0f57b49bce09965641cc71a", + "versions.yml:md5,47df50b983a036f55cb936247d920744", + "versions.yml:md5,586dc48e8e2cc8106cb0ee3b70ce0fef", + "versions.yml:md5,6d41fb977d31ce96d7ddbb4ea2dbe38b", + "versions.yml:md5,7017c8959eb098440346ca9a2310c460", + "versions.yml:md5,7ccffaba4e75be98b8faedb54ccea16f", + "versions.yml:md5,8187d1683f97b64a582620caba2b175a", + "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", + "versions.yml:md5,aee39cb21305f6d3fe88bf9863529706", + "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", + "versions.yml:md5,c777f00c6a48f3c0490d09b1aa8d81a6", + "versions.yml:md5,cc4a4972327dec07faf1f54ead653715", + "versions.yml:md5,d8d2321bec0c70de2f9750f258fc890d", + "versions.yml:md5,dc29f891cb297131cf940ba5580a6832", + "versions.yml:md5,df654dd4fdb4039c2d8cf0b6b49f4b55", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", + "versions.yml:md5,e9f8fe610b5abe68b28d77efc951e9e6" ] - ] + } ], - "timestamp": "2023-12-15T13:36:01.487711685" + "timestamp": "2024-01-02T12:11:37.803431535" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_bbnorm.nf.test.snap b/workflows/mag/tests/test_bbnorm.nf.test.snap index fb848b61..89a37918 100644 --- a/workflows/mag/tests/test_bbnorm.nf.test.snap +++ b/workflows/mag/tests/test_bbnorm.nf.test.snap @@ -10,8 +10,8 @@ "single_end": false }, [ - "test_minigut_1.merged.fastq.gz:md5,fe81462f19e6acc0bbcec55528be7ad2", - "test_minigut_2.merged.fastq.gz:md5,88ebc2ac01082507f7be1761049e5eff" + "test_minigut_run0_fastp_1.fastp.fastq.gz:md5,bac85d89ef44638200c247bc941a66fb", + "test_minigut_run0_fastp_2.fastp.fastq.gz:md5,be6b3eb0d336c333e97d925b0e3a0ff6" ] ], [ @@ -30,31 +30,11 @@ ], "10": [ - - ], - "11": [ - - ], - "12": [ - - ], - "13": [ - - ], - "14": [ - - ], - "15": [ - - ], - "16": [ "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", - "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", "versions.yml:md5,67cbab915f2801633ea61219086a7735", "versions.yml:md5,67cbab915f2801633ea61219086a7735", "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" ], "2": [ @@ -84,26 +64,14 @@ "assemblies": [ ], - "bins": [ - - ], - "busco": [ + "bin_summary": [ ], "cat": [ - ], - "checkm": [ - ], "genomad": [ - ], - "gtdbtk": [ - - ], - "gunc": [ - ], "metaeuk": [ @@ -113,9 +81,6 @@ ], "prokka": [ - ], - "quast_bins": [ - ], "refined_bins": [ @@ -131,8 +96,8 @@ "single_end": false }, [ - "test_minigut_1.merged.fastq.gz:md5,fe81462f19e6acc0bbcec55528be7ad2", - "test_minigut_2.merged.fastq.gz:md5,88ebc2ac01082507f7be1761049e5eff" + "test_minigut_run0_fastp_1.fastp.fastq.gz:md5,bac85d89ef44638200c247bc941a66fb", + "test_minigut_run0_fastp_2.fastp.fastq.gz:md5,be6b3eb0d336c333e97d925b0e3a0ff6" ] ], [ @@ -146,22 +111,17 @@ "test_minigut_sample2_run0_fastp_2.fastp.fastq.gz:md5,a91bde6d5c85dd6b47f5bc5cfab65572" ] ] - ], - "unbins": [ - ], "versions": [ "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", - "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", "versions.yml:md5,67cbab915f2801633ea61219086a7735", "versions.yml:md5,67cbab915f2801633ea61219086a7735", "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" ] } ], - "timestamp": "2023-12-14T12:01:36.938067062" + "timestamp": "2024-01-02T13:06:14.895384486" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_binrefinement.nf.test b/workflows/mag/tests/test_binrefinement.nf.test new file mode 100644 index 00000000..277b06e5 --- /dev/null +++ b/workflows/mag/tests/test_binrefinement.nf.test @@ -0,0 +1,21 @@ +nextflow_workflow { + + name "Test workflow: MAG" + script "../main.nf" + workflow "MAG" + tag "workflows" + tag "mag" + tag "mag_test_binrefinement" + + test("Parameters: refine_bins_dastool = true") { + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out + ).match() } + ) + } + } +} diff --git a/workflows/mag/tests/test_binrefinement.nf.test.snap b/workflows/mag/tests/test_binrefinement.nf.test.snap new file mode 100644 index 00000000..c4cc01c2 --- /dev/null +++ b/workflows/mag/tests/test_binrefinement.nf.test.snap @@ -0,0 +1,335 @@ +{ + "Parameters: refine_bins_dastool = true": { + "content": [ + { + "0": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_R1.fastq.gz", + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_R2.fastq.gz" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R1.fastq.gz", + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R2.fastq.gz" + ] + ] + ], + "1": [ + [ + { + "id": "test_minigut", + "group": "0", + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut.contigs.fa:md5,49bd2dbd085e37b3f19a62d9d010a271" + ], + [ + { + "id": "test_minigut", + "group": "0", + "assembler": "SPAdes" + }, + "SPAdes-test_minigut_contigs.fasta:md5,86027592505360f0e51a395555b8a11b" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut_sample2.contigs.fa:md5,378c9d74964ef6a9a0fd2f2c77dfa607" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "assembler": "SPAdes" + }, + "SPAdes-test_minigut_sample2_contigs.fasta:md5,0c3fdd6f534d5e21b105be45351ee655" + ] + ], + "10": [ + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,3550550b622b6d5f6803ffed9742e08b", + "versions.yml:md5,586dc48e8e2cc8106cb0ee3b70ce0fef", + "versions.yml:md5,6d41fb977d31ce96d7ddbb4ea2dbe38b", + "versions.yml:md5,7017c8959eb098440346ca9a2310c460", + "versions.yml:md5,7ccffaba4e75be98b8faedb54ccea16f", + "versions.yml:md5,8187d1683f97b64a582620caba2b175a", + "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", + "versions.yml:md5,aee39cb21305f6d3fe88bf9863529706", + "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", + "versions.yml:md5,c777f00c6a48f3c0490d09b1aa8d81a6", + "versions.yml:md5,cc4a4972327dec07faf1f54ead653715", + "versions.yml:md5,d8d2321bec0c70de2f9750f258fc890d", + "versions.yml:md5,dc29f891cb297131cf940ba5580a6832", + "versions.yml:md5,df654dd4fdb4039c2d8cf0b6b49f4b55", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", + "versions.yml:md5,e9f8fe610b5abe68b28d77efc951e9e6", + "versions.yml:md5,ede79d94ca30c948c1707584760b230c", + "versions.yml:md5,ede79d94ca30c948c1707584760b230c", + "versions.yml:md5,ede79d94ca30c948c1707584760b230c", + "versions.yml:md5,ede79d94ca30c948c1707584760b230c" + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + [ + { + "id": "test_minigut", + "group": "0", + "assembler": "MEGAHIT", + "binner": "DASTool", + "refinement": "dastool_refined", + "domain": "unclassified" + }, + [ + "MEGAHIT-MetaBAT2Refined-test_minigut.1.fa:md5,489a5f7ba3795477e09d2d396fe535c3", + "MEGAHIT-MetaBAT2Refined-test_minigut.2.fa:md5,1addf634bca82af85d36621585f93f52" + ] + ], + [ + { + "id": "test_minigut", + "group": "0", + "assembler": "SPAdes", + "binner": "DASTool", + "refinement": "dastool_refined", + "domain": "unclassified" + }, + [ + "SPAdes-MaxBin2Refined-test_minigut.001_sub.fa:md5,b1633cb560c41afe3606b6e8f2082211", + "SPAdes-MetaBAT2Refined-test_minigut.2.fa:md5,7f8be703a086fd12d41d2df817a08e8d" + ] + ] + ], + "5": [ + [ + { + "id": "test_minigut", + "group": "0", + "assembler": "MEGAHIT", + "refinement": "dastool_refined_unbinned", + "binner": "DASTool", + "domain": "unclassified" + }, + "MEGAHIT-DASToolUnbinned-test_minigut.fa:md5,5b68d8b2df39cb4ab46299d9a09f1243" + ], + [ + { + "id": "test_minigut", + "group": "0", + "assembler": "SPAdes", + "refinement": "dastool_refined_unbinned", + "binner": "DASTool", + "domain": "unclassified" + }, + "SPAdes-DASToolUnbinned-test_minigut.fa:md5,067f6ad8e2f820ab3d6000a154150954" + ] + ], + "6": [ + + ], + "7": [ + + ], + "8": [ + + ], + "9": [ + + ], + "assemblies": [ + [ + { + "id": "test_minigut", + "group": "0", + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut.contigs.fa:md5,49bd2dbd085e37b3f19a62d9d010a271" + ], + [ + { + "id": "test_minigut", + "group": "0", + "assembler": "SPAdes" + }, + "SPAdes-test_minigut_contigs.fasta:md5,86027592505360f0e51a395555b8a11b" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut_sample2.contigs.fa:md5,378c9d74964ef6a9a0fd2f2c77dfa607" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "assembler": "SPAdes" + }, + "SPAdes-test_minigut_sample2_contigs.fasta:md5,0c3fdd6f534d5e21b105be45351ee655" + ] + ], + "bin_summary": [ + + ], + "cat": [ + + ], + "genomad": [ + + ], + "metaeuk": [ + + ], + "prodigal": [ + + ], + "prokka": [ + + ], + "refined_bins": [ + [ + { + "id": "test_minigut", + "group": "0", + "assembler": "MEGAHIT", + "binner": "DASTool", + "refinement": "dastool_refined", + "domain": "unclassified" + }, + [ + "MEGAHIT-MetaBAT2Refined-test_minigut.1.fa:md5,489a5f7ba3795477e09d2d396fe535c3", + "MEGAHIT-MetaBAT2Refined-test_minigut.2.fa:md5,1addf634bca82af85d36621585f93f52" + ] + ], + [ + { + "id": "test_minigut", + "group": "0", + "assembler": "SPAdes", + "binner": "DASTool", + "refinement": "dastool_refined", + "domain": "unclassified" + }, + [ + "SPAdes-MaxBin2Refined-test_minigut.001_sub.fa:md5,b1633cb560c41afe3606b6e8f2082211", + "SPAdes-MetaBAT2Refined-test_minigut.2.fa:md5,7f8be703a086fd12d41d2df817a08e8d" + ] + ] + ], + "refined_unbins": [ + [ + { + "id": "test_minigut", + "group": "0", + "assembler": "MEGAHIT", + "refinement": "dastool_refined_unbinned", + "binner": "DASTool", + "domain": "unclassified" + }, + "MEGAHIT-DASToolUnbinned-test_minigut.fa:md5,5b68d8b2df39cb4ab46299d9a09f1243" + ], + [ + { + "id": "test_minigut", + "group": "0", + "assembler": "SPAdes", + "refinement": "dastool_refined_unbinned", + "binner": "DASTool", + "domain": "unclassified" + }, + "SPAdes-DASToolUnbinned-test_minigut.fa:md5,067f6ad8e2f820ab3d6000a154150954" + ] + ], + "short_reads": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_R1.fastq.gz", + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_R2.fastq.gz" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R1.fastq.gz", + "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R2.fastq.gz" + ] + ] + ], + "versions": [ + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,3550550b622b6d5f6803ffed9742e08b", + "versions.yml:md5,586dc48e8e2cc8106cb0ee3b70ce0fef", + "versions.yml:md5,6d41fb977d31ce96d7ddbb4ea2dbe38b", + "versions.yml:md5,7017c8959eb098440346ca9a2310c460", + "versions.yml:md5,7ccffaba4e75be98b8faedb54ccea16f", + "versions.yml:md5,8187d1683f97b64a582620caba2b175a", + "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", + "versions.yml:md5,aee39cb21305f6d3fe88bf9863529706", + "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", + "versions.yml:md5,c777f00c6a48f3c0490d09b1aa8d81a6", + "versions.yml:md5,cc4a4972327dec07faf1f54ead653715", + "versions.yml:md5,d8d2321bec0c70de2f9750f258fc890d", + "versions.yml:md5,dc29f891cb297131cf940ba5580a6832", + "versions.yml:md5,df654dd4fdb4039c2d8cf0b6b49f4b55", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", + "versions.yml:md5,e9f8fe610b5abe68b28d77efc951e9e6", + "versions.yml:md5,ede79d94ca30c948c1707584760b230c", + "versions.yml:md5,ede79d94ca30c948c1707584760b230c", + "versions.yml:md5,ede79d94ca30c948c1707584760b230c", + "versions.yml:md5,ede79d94ca30c948c1707584760b230c" + ] + } + ], + "timestamp": "2024-01-02T13:35:37.404532095" + } +} \ No newline at end of file diff --git a/workflows/mag/tests/test_busco_auto.nf.test b/workflows/mag/tests/test_busco_auto.nf.test new file mode 100644 index 00000000..5fb41543 --- /dev/null +++ b/workflows/mag/tests/test_busco_auto.nf.test @@ -0,0 +1,21 @@ +nextflow_workflow { + + name "Test workflow: MAG" + script "../main.nf" + workflow "MAG" + tag "workflows" + tag "mag" + tag "mag_test_busco_auto" + + test("Parameters: skip_busco = false") { + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out + ).match() } + ) + } + } +} diff --git a/workflows/mag/tests/test_busco_auto.nf.test.snap b/workflows/mag/tests/test_busco_auto.nf.test.snap new file mode 100644 index 00000000..f7a26da6 --- /dev/null +++ b/workflows/mag/tests/test_busco_auto.nf.test.snap @@ -0,0 +1,189 @@ +{ + "Parameters: skip_busco = false": { + "content": [ + { + "0": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "test_minigut_run0_fastp_1.fastp.fastq.gz:md5,bac85d89ef44638200c247bc941a66fb", + "test_minigut_run0_fastp_2.fastp.fastq.gz:md5,be6b3eb0d336c333e97d925b0e3a0ff6" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "test_minigut_sample2_run0_fastp_1.fastp.fastq.gz:md5,25580ab85c8288a6a7a0b2100d93f1d0", + "test_minigut_sample2_run0_fastp_2.fastp.fastq.gz:md5,a91bde6d5c85dd6b47f5bc5cfab65572" + ] + ] + ], + "1": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut.contigs.fa:md5,c78fa53feb4b35380998bc0c339cdb0e" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut_sample2.contigs.fa:md5,2c0b7977b39fb2db31ee9b3adf75f8c6" + ] + ], + "10": [ + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", + "versions.yml:md5,18eeebfdf8c302c29cbb4abc4325628c", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,3550550b622b6d5f6803ffed9742e08b", + "versions.yml:md5,7017c8959eb098440346ca9a2310c460", + "versions.yml:md5,8187d1683f97b64a582620caba2b175a", + "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", + "versions.yml:md5,cc4a4972327dec07faf1f54ead653715", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + + ], + "5": [ + + ], + "6": [ + "bin_summary.tsv:md5,59e4568ef471db7c5d1f1105b3325216" + ], + "7": [ + + ], + "8": [ + + ], + "9": [ + + ], + "assemblies": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut.contigs.fa:md5,c78fa53feb4b35380998bc0c339cdb0e" + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false, + "assembler": "MEGAHIT" + }, + "MEGAHIT-test_minigut_sample2.contigs.fa:md5,2c0b7977b39fb2db31ee9b3adf75f8c6" + ] + ], + "bin_summary": [ + "bin_summary.tsv:md5,59e4568ef471db7c5d1f1105b3325216" + ], + "cat": [ + + ], + "genomad": [ + + ], + "metaeuk": [ + + ], + "prodigal": [ + + ], + "prokka": [ + + ], + "refined_bins": [ + + ], + "refined_unbins": [ + + ], + "short_reads": [ + [ + { + "id": "test_minigut", + "group": "0", + "single_end": false + }, + [ + "test_minigut_run0_fastp_1.fastp.fastq.gz:md5,bac85d89ef44638200c247bc941a66fb", + "test_minigut_run0_fastp_2.fastp.fastq.gz:md5,be6b3eb0d336c333e97d925b0e3a0ff6" + ] + ], + [ + { + "id": "test_minigut_sample2", + "group": "0", + "single_end": false + }, + [ + "test_minigut_sample2_run0_fastp_1.fastp.fastq.gz:md5,25580ab85c8288a6a7a0b2100d93f1d0", + "test_minigut_sample2_run0_fastp_2.fastp.fastq.gz:md5,a91bde6d5c85dd6b47f5bc5cfab65572" + ] + ] + ], + "versions": [ + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", + "versions.yml:md5,18eeebfdf8c302c29cbb4abc4325628c", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,3550550b622b6d5f6803ffed9742e08b", + "versions.yml:md5,7017c8959eb098440346ca9a2310c460", + "versions.yml:md5,8187d1683f97b64a582620caba2b175a", + "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", + "versions.yml:md5,cc4a4972327dec07faf1f54ead653715", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", + "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + ] + } + ], + "timestamp": "2024-01-02T15:06:33.650718766" + } +} \ No newline at end of file diff --git a/workflows/mag/tests/test_host_rm.nf.test.snap b/workflows/mag/tests/test_host_rm.nf.test.snap index 9a90030d..6103b621 100644 --- a/workflows/mag/tests/test_host_rm.nf.test.snap +++ b/workflows/mag/tests/test_host_rm.nf.test.snap @@ -30,24 +30,6 @@ ], "10": [ - - ], - "11": [ - - ], - "12": [ - - ], - "13": [ - - ], - "14": [ - - ], - "15": [ - - ], - "16": [ "versions.yml:md5,59050473c9f749b269ad7079fd4693df", "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", @@ -80,26 +62,14 @@ "assemblies": [ ], - "bins": [ - - ], - "busco": [ + "bin_summary": [ ], "cat": [ - ], - "checkm": [ - ], "genomad": [ - ], - "gtdbtk": [ - - ], - "gunc": [ - ], "metaeuk": [ @@ -109,9 +79,6 @@ ], "prokka": [ - ], - "quast_bins": [ - ], "refined_bins": [ @@ -142,9 +109,6 @@ "test_minigut_sample2_hg38host_run0_host_removed.unmapped_2.fastq.gz:md5,6ded2b5f426aca4e9b29877fac4e9bff" ] ] - ], - "unbins": [ - ], "versions": [ "versions.yml:md5,59050473c9f749b269ad7079fd4693df", @@ -154,6 +118,6 @@ ] } ], - "timestamp": "2023-12-14T14:23:51.5082671" + "timestamp": "2024-01-02T15:33:59.196829811" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_hybrid.nf.test.snap b/workflows/mag/tests/test_hybrid.nf.test.snap index 1b9556cd..6ec11bbd 100644 --- a/workflows/mag/tests/test_hybrid.nf.test.snap +++ b/workflows/mag/tests/test_hybrid.nf.test.snap @@ -36,24 +36,6 @@ ] ], "10": [ - - ], - "11": [ - - ], - "12": [ - - ], - "13": [ - - ], - "14": [ - - ], - "15": [ - - ], - "16": [ "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", "versions.yml:md5,1bd6bbec156534880e04e0320dbce063", "versions.yml:md5,5de8983775d3ef64cdbec6f382852619", @@ -108,26 +90,14 @@ "SPAdesHybrid-group-0_scaffolds.fasta:md5,84c68c6ba721ad5858588a1487e05b6f" ] ], - "bins": [ - - ], - "busco": [ + "bin_summary": [ ], "cat": [ - ], - "checkm": [ - ], "genomad": [ - ], - "gtdbtk": [ - - ], - "gunc": [ - ], "metaeuk": [ @@ -137,9 +107,6 @@ ], "prokka": [ - ], - "quast_bins": [ - ], "refined_bins": [ @@ -159,9 +126,6 @@ "minigut_run0_fastp_2.fastp.fastq.gz:md5,be6b3eb0d336c333e97d925b0e3a0ff6" ] ] - ], - "unbins": [ - ], "versions": [ "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", @@ -176,6 +140,6 @@ ] } ], - "timestamp": "2023-12-14T14:09:37.526745496" + "timestamp": "2024-01-02T16:01:13.719391405" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_hybrid_host_rm.nf.test.snap b/workflows/mag/tests/test_hybrid_host_rm.nf.test.snap index b4c23e10..3cb34ca8 100644 --- a/workflows/mag/tests/test_hybrid_host_rm.nf.test.snap +++ b/workflows/mag/tests/test_hybrid_host_rm.nf.test.snap @@ -27,24 +27,6 @@ ] ], "10": [ - - ], - "11": [ - - ], - "12": [ - - ], - "13": [ - - ], - "14": [ - - ], - "15": [ - - ], - "16": [ "versions.yml:md5,1bd6bbec156534880e04e0320dbce063", "versions.yml:md5,59050473c9f749b269ad7079fd4693df", "versions.yml:md5,5de8983775d3ef64cdbec6f382852619", @@ -89,26 +71,14 @@ "SPAdesHybrid-minigut_scaffolds.fasta:md5,d3c5e27f011a08daea597f9d2650436d" ] ], - "bins": [ - - ], - "busco": [ + "bin_summary": [ ], "cat": [ - ], - "checkm": [ - ], "genomad": [ - ], - "gtdbtk": [ - - ], - "gunc": [ - ], "metaeuk": [ @@ -118,9 +88,6 @@ ], "prokka": [ - ], - "quast_bins": [ - ], "refined_bins": [ @@ -140,9 +107,6 @@ "minigut_run0_host_removed.unmapped_2.fastq.gz:md5,b4246225aa4a15496991ac7c6b58563f" ] ] - ], - "unbins": [ - ], "versions": [ "versions.yml:md5,1bd6bbec156534880e04e0320dbce063", @@ -156,6 +120,6 @@ ] } ], - "timestamp": "2023-12-14T14:52:24.237611909" + "timestamp": "2024-01-02T15:43:00.831382408" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_nothing.nf.test.snap b/workflows/mag/tests/test_nothing.nf.test.snap index a1a58ed6..ec39fd6a 100644 --- a/workflows/mag/tests/test_nothing.nf.test.snap +++ b/workflows/mag/tests/test_nothing.nf.test.snap @@ -30,24 +30,6 @@ ], "10": [ - - ], - "11": [ - - ], - "12": [ - - ], - "13": [ - - ], - "14": [ - - ], - "15": [ - - ], - "16": [ "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" ], @@ -78,26 +60,14 @@ "assemblies": [ ], - "bins": [ - - ], - "busco": [ + "bin_summary": [ ], "cat": [ - ], - "checkm": [ - ], "genomad": [ - ], - "gtdbtk": [ - - ], - "gunc": [ - ], "metaeuk": [ @@ -107,9 +77,6 @@ ], "prokka": [ - ], - "quast_bins": [ - ], "refined_bins": [ @@ -140,9 +107,6 @@ "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R2.fastq.gz" ] ] - ], - "unbins": [ - ], "versions": [ "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", @@ -150,6 +114,6 @@ ] } ], - "timestamp": "2023-12-14T11:38:30.168015353" + "timestamp": "2024-01-02T16:14:07.274635084" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_virus_identification.nf.test.snap b/workflows/mag/tests/test_virus_identification.nf.test.snap index a1894fe8..e9b22af2 100644 --- a/workflows/mag/tests/test_virus_identification.nf.test.snap +++ b/workflows/mag/tests/test_virus_identification.nf.test.snap @@ -38,24 +38,6 @@ ] ], "10": [ - - ], - "11": [ - - ], - "12": [ - - ], - "13": [ - - ], - "14": [ - - ], - "15": [ - - ], - "16": [ "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" @@ -103,17 +85,11 @@ "MEGAHIT-group-0.contigs.fa:md5,8419d4d6c72ac1006324a9fee1394646" ] ], - "bins": [ - - ], - "busco": [ + "bin_summary": [ ], "cat": [ - ], - "checkm": [ - ], "genomad": [ [ @@ -125,12 +101,6 @@ }, "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,b08861c8a938b3eaf8c1d8e533935042" ] - ], - "gtdbtk": [ - - ], - "gunc": [ - ], "metaeuk": [ @@ -140,9 +110,6 @@ ], "prokka": [ - ], - "quast_bins": [ - ], "refined_bins": [ @@ -173,9 +140,6 @@ "/nf-core/test-datasets/raw/mag/test_data/test_minigut_sample2_R2.fastq.gz" ] ] - ], - "unbins": [ - ], "versions": [ "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", @@ -184,6 +148,6 @@ ] } ], - "timestamp": "2023-12-14T11:48:19.010278472" + "timestamp": "2024-01-03T09:34:15.379927608" } } \ No newline at end of file From edb4a80609b3b8e438a8d846fbdd79ef946111b3 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Thu, 4 Jan 2024 15:31:59 -0800 Subject: [PATCH 19/49] Updated CI tags --- workflows/mag/tests/tags.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/workflows/mag/tests/tags.yml b/workflows/mag/tests/tags.yml index 7e7fe107..6d995160 100644 --- a/workflows/mag/tests/tags.yml +++ b/workflows/mag/tests/tags.yml @@ -4,6 +4,10 @@ test_ancient_dna: - ./** test_bbnorm: - ./** +test_binrefinement: + - ./** +test_busco_auto: + - ./** test_host_rm: - ./** test_hybrid_host_rm: @@ -14,3 +18,5 @@ test_nothing: - ./** test_virus_identification: - ./** +test: + - ./** From 065c3a6d6aaeed06dd34f550fd87be40a16d020c Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Mon, 8 Jan 2024 09:37:33 -0800 Subject: [PATCH 20/49] Fixed snapshots and linting --- bin/combine_tables.py | 2 +- conf/test.config | 1 + conf/test_ancient_dna.config | 4 +-- workflows/mag/tests/main.nf.test.snap | 26 +++++++++---------- .../test_virus_identification.nf.test.snap | 2 +- 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/bin/combine_tables.py b/bin/combine_tables.py index c4eaeb02..ed3240fa 100755 --- a/bin/combine_tables.py +++ b/bin/combine_tables.py @@ -94,7 +94,7 @@ def main(args=None): ) # assuming depths for all bins are given # sort results for reproducibility - results.sort_values(by='bin', inplace=True, ignore_index=True) + results.sort_values(by="bin", inplace=True, ignore_index=True) results.to_csv(args.out, sep="\t") diff --git a/conf/test.config b/conf/test.config index 5968e844..d03d088e 100644 --- a/conf/test.config +++ b/conf/test.config @@ -25,6 +25,7 @@ params { kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" skip_krona = true megahit_fix_cpu_1 = true + spades_fix_cpus = 1 bowtie2_fix_cpu_1 = true metabat2_fix_cpu_1 = true maxbin2_fix_cpu_1 = true diff --git a/conf/test_ancient_dna.config b/conf/test_ancient_dna.config index dcdda8fa..017dcb66 100644 --- a/conf/test_ancient_dna.config +++ b/conf/test_ancient_dna.config @@ -15,8 +15,8 @@ params { config_profile_description = 'Minimal test dataset to check pipeline function for ancient DNA step' // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' + //max_cpus = 2 + //max_memory = '6.GB' max_time = '6.h' // Input data diff --git a/workflows/mag/tests/main.nf.test.snap b/workflows/mag/tests/main.nf.test.snap index 1202ab2d..cb3ddd39 100644 --- a/workflows/mag/tests/main.nf.test.snap +++ b/workflows/mag/tests/main.nf.test.snap @@ -43,7 +43,7 @@ "single_end": false, "assembler": "SPAdes" }, - "SPAdes-test_minigut_scaffolds.fasta:md5,8bd449b393a7d3937ff74a24ce39cfc1" + "SPAdes-test_minigut_scaffolds.fasta:md5,50301a321e7618b1f5c4a8664ae21765" ], [ { @@ -61,7 +61,7 @@ "single_end": false, "assembler": "SPAdes" }, - "SPAdes-test_minigut_sample2_scaffolds.fasta:md5,bf8ebad421409a0dee62a7b14acfca88" + "SPAdes-test_minigut_sample2_scaffolds.fasta:md5,235505d69a2822d77ad70ce061b80efe" ] ], "10": [ @@ -120,7 +120,7 @@ "single_end": false, "assembler": "SPAdes" }, - "test_minigut.gff.gz:md5,80e8726ecc2d146187cfa2aa4a748d84" + "test_minigut.gff.gz:md5,84429e564c6e5efe0e39c131ee23919f" ], [ { @@ -138,7 +138,7 @@ "single_end": false, "assembler": "SPAdes" }, - "test_minigut_sample2.gff.gz:md5,1e099a6a3def0c73010d7a1e138c8047" + "test_minigut_sample2.gff.gz:md5,4d9b7eb438adc2db0befb7ae87f25725" ] ], "3": [ @@ -227,7 +227,7 @@ "domain": "unclassified", "refinement": "unrefined" }, - "SPAdes-MaxBin2-test_minigut.002.faa:md5,846a091d4f412975bcd66c5ef0ed49d9" + "SPAdes-MaxBin2-test_minigut.002.faa:md5,ed025d6d6dceb1b79fafb2a062b5790c" ], [ { @@ -251,7 +251,7 @@ "domain": "unclassified", "refinement": "unrefined" }, - "SPAdes-MetaBAT2-test_minigut.1.faa:md5,2879b7a33cca4024e3c518b2fa4bc063" + "SPAdes-MetaBAT2-test_minigut.1.faa:md5,d378b57b012feaf4c43a7da0a78a6968" ], [ { @@ -286,7 +286,7 @@ "single_end": false, "assembler": "SPAdes" }, - "SPAdes-test_minigut_scaffolds.fasta:md5,8bd449b393a7d3937ff74a24ce39cfc1" + "SPAdes-test_minigut_scaffolds.fasta:md5,50301a321e7618b1f5c4a8664ae21765" ], [ { @@ -304,7 +304,7 @@ "single_end": false, "assembler": "SPAdes" }, - "SPAdes-test_minigut_sample2_scaffolds.fasta:md5,bf8ebad421409a0dee62a7b14acfca88" + "SPAdes-test_minigut_sample2_scaffolds.fasta:md5,235505d69a2822d77ad70ce061b80efe" ] ], "bin_summary": [ @@ -336,7 +336,7 @@ "single_end": false, "assembler": "SPAdes" }, - "test_minigut.gff.gz:md5,80e8726ecc2d146187cfa2aa4a748d84" + "test_minigut.gff.gz:md5,84429e564c6e5efe0e39c131ee23919f" ], [ { @@ -354,7 +354,7 @@ "single_end": false, "assembler": "SPAdes" }, - "test_minigut_sample2.gff.gz:md5,1e099a6a3def0c73010d7a1e138c8047" + "test_minigut_sample2.gff.gz:md5,4d9b7eb438adc2db0befb7ae87f25725" ] ], "prokka": [ @@ -428,7 +428,7 @@ "domain": "unclassified", "refinement": "unrefined" }, - "SPAdes-MaxBin2-test_minigut.002.faa:md5,846a091d4f412975bcd66c5ef0ed49d9" + "SPAdes-MaxBin2-test_minigut.002.faa:md5,ed025d6d6dceb1b79fafb2a062b5790c" ], [ { @@ -452,7 +452,7 @@ "domain": "unclassified", "refinement": "unrefined" }, - "SPAdes-MetaBAT2-test_minigut.1.faa:md5,2879b7a33cca4024e3c518b2fa4bc063" + "SPAdes-MetaBAT2-test_minigut.1.faa:md5,d378b57b012feaf4c43a7da0a78a6968" ], [ { @@ -538,6 +538,6 @@ ] } ], - "timestamp": "2024-01-04T14:50:41.377338936" + "timestamp": "2024-01-08T08:51:13.153323648" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_virus_identification.nf.test.snap b/workflows/mag/tests/test_virus_identification.nf.test.snap index e9b22af2..76009d2d 100644 --- a/workflows/mag/tests/test_virus_identification.nf.test.snap +++ b/workflows/mag/tests/test_virus_identification.nf.test.snap @@ -148,6 +148,6 @@ ] } ], - "timestamp": "2024-01-03T09:34:15.379927608" + "timestamp": "2024-01-04T16:01:40.636819119" } } \ No newline at end of file From 6110f5fdf08b4b92e8590cb50bd746819d1779f6 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Mon, 8 Jan 2024 09:37:58 -0800 Subject: [PATCH 21/49] Fixed resource requests --- conf/test_ancient_dna.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/test_ancient_dna.config b/conf/test_ancient_dna.config index 017dcb66..dcdda8fa 100644 --- a/conf/test_ancient_dna.config +++ b/conf/test_ancient_dna.config @@ -15,8 +15,8 @@ params { config_profile_description = 'Minimal test dataset to check pipeline function for ancient DNA step' // Limit resources so that this can run on GitHub Actions - //max_cpus = 2 - //max_memory = '6.GB' + max_cpus = 2 + max_memory = '6.GB' max_time = '6.h' // Input data From 2143e0ae11ea4e32301f745a07d83911c2371253 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Mon, 8 Jan 2024 09:40:22 -0800 Subject: [PATCH 22/49] Fixed linting --- bin/summary_busco.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/summary_busco.py b/bin/summary_busco.py index faff6718..5dd33f0c 100755 --- a/bin/summary_busco.py +++ b/bin/summary_busco.py @@ -187,7 +187,7 @@ def main(args=None): df_final = df_specific.append(df_failed) # sort output file for reproducibility - df_final.sort_values(by='GenomeBin', inplace=True) + df_final.sort_values(by="GenomeBin", inplace=True) df_final.to_csv(args.out, sep="\t", index=False) From 438ace331e2ce8b026888fac3c761744d633172a Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Mon, 8 Jan 2024 14:44:48 -0800 Subject: [PATCH 23/49] Updated snapshot --- .../test_virus_identification.nf.test.snap | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/workflows/mag/tests/test_virus_identification.nf.test.snap b/workflows/mag/tests/test_virus_identification.nf.test.snap index 76009d2d..f9cda1bc 100644 --- a/workflows/mag/tests/test_virus_identification.nf.test.snap +++ b/workflows/mag/tests/test_virus_identification.nf.test.snap @@ -43,7 +43,7 @@ "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" ], "2": [ - + ], "3": [ [ @@ -53,26 +53,26 @@ "single_end": false, "assembler": "MEGAHIT" }, - "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,b08861c8a938b3eaf8c1d8e533935042" + "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,95db91fff85f23a19e64ef7ea0f8d371" ] ], "4": [ - + ], "5": [ - + ], "6": [ - + ], "7": [ - + ], "8": [ - + ], "9": [ - + ], "assemblies": [ [ @@ -86,10 +86,10 @@ ] ], "bin_summary": [ - + ], "cat": [ - + ], "genomad": [ [ @@ -99,23 +99,23 @@ "single_end": false, "assembler": "MEGAHIT" }, - "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,b08861c8a938b3eaf8c1d8e533935042" + "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,95db91fff85f23a19e64ef7ea0f8d371" ] ], "metaeuk": [ - + ], "prodigal": [ - + ], "prokka": [ - + ], "refined_bins": [ - + ], "refined_unbins": [ - + ], "short_reads": [ [ @@ -150,4 +150,4 @@ ], "timestamp": "2024-01-04T16:01:40.636819119" } -} \ No newline at end of file +} From 6028f5a100e1dd1a6e3e5b2ca93628d02a9fb7b5 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Fri, 12 Jan 2024 16:12:33 +0000 Subject: [PATCH 24/49] Remove unused local modules --- modules/local/mag_merge_samplesheet.nf | 28 ------------- modules/local/mag_to_samplesheet.nf | 40 ------------------- .../create_phageannotator_samplesheet.nf | 2 - 3 files changed, 70 deletions(-) delete mode 100644 modules/local/mag_merge_samplesheet.nf delete mode 100644 modules/local/mag_to_samplesheet.nf diff --git a/modules/local/mag_merge_samplesheet.nf b/modules/local/mag_merge_samplesheet.nf deleted file mode 100644 index 34641de6..00000000 --- a/modules/local/mag_merge_samplesheet.nf +++ /dev/null @@ -1,28 +0,0 @@ -process MAG_MERGE_SAMPLESHEET { - - conda "conda-forge::sed=4.7" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : - 'nf-core/ubuntu:20.04' }" - - input: - path ('samplesheets/*') - - output: - path "*_samplesheet.csv", emit: samplesheet - path "versions.yml" , emit: versions - - script: - def prefix = task.ext.prefix ?: "${meta.id}" - """ - head -n 1 `ls ./samplesheets/* | head -n 1` > ${prefix}_samplesheet.csv - for fileid in `ls ./samplesheets/*`; do - awk 'NR>1' \$fileid >> ${prefix}_samplesheet.csv - done - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - sed: \$(echo \$(sed --version 2>&1) | sed 's/^.*GNU sed) //; s/ .*\$//') - END_VERSIONS - """ -} diff --git a/modules/local/mag_to_samplesheet.nf b/modules/local/mag_to_samplesheet.nf deleted file mode 100644 index a454bb44..00000000 --- a/modules/local/mag_to_samplesheet.nf +++ /dev/null @@ -1,40 +0,0 @@ -process MAG_TO_SAMPLESHEET { - tag "$meta.id" - - executor 'local' - memory 100.MB - - input: - val meta - val pipeline - - output: - tuple val(meta), path("*samplesheet.csv"), emit: samplesheet - - exec: - // - // Create samplesheet containing metadata - // - - // Add nf-core pipeline specific entries - if (pipeline) { - if (pipeline == 'phageannotator') { - pipeline_map = [ - sample : "${meta.id}", - group : "${meta.group}", - fastq_1 : meta.fastq_1, - fastq_2 : meta.fastq_2, - fasta : meta.fasta - ] - } - } - - // Create a samplesheet - samplesheet = pipeline_map.keySet().collect{ '"' + it + '"'}.join(",") + '\n' - samplesheet += pipeline_map.values().collect{ '"' + it + '"'}.join(",") - - // Write samplesheet to file - def samplesheet_file = task.workDir.resolve("${meta.id}.samplesheet.csv") - samplesheet_file.text = samplesheet - -} diff --git a/subworkflows/local/create_phageannotator_samplesheet.nf b/subworkflows/local/create_phageannotator_samplesheet.nf index 6423486e..1299febb 100644 --- a/subworkflows/local/create_phageannotator_samplesheet.nf +++ b/subworkflows/local/create_phageannotator_samplesheet.nf @@ -1,6 +1,4 @@ include { CAT_CAT } from '../../modules/nf-core/cat/cat/main' -include { MAG_TO_SAMPLESHEET } from '../../modules/local/mag_to_samplesheet' -include { MAG_MERGE_SAMPLESHEET } from '../../modules/local/mag_merge_samplesheet' workflow CREATE_PHAGEANNOTATOR_SAMPLESHEET { take: From d52798dc5c6925a969efbeb571d1e75fd3b9284c Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Wed, 17 Jan 2024 14:51:24 +0000 Subject: [PATCH 25/49] Updated nf-core modules and pinned prettier version --- .github/workflows/linting.yml | 2 +- conf/test_data.config | 764 ++++++++++++++++++ modules.json | 4 +- .../dumpsoftwareversions/environment.yml | 2 +- .../custom/dumpsoftwareversions/main.nf | 4 +- .../dumpsoftwareversions/tests/main.nf.test | 7 +- .../tests/main.nf.test.snap | 50 +- modules/nf-core/fastqc/tests/main.nf.test | 273 +++++-- .../nf-core/fastqc/tests/main.nf.test.snap | 12 +- nextflow.config | 1 + 10 files changed, 1014 insertions(+), 105 deletions(-) create mode 100644 conf/test_data.config diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 905c58e4..5c001706 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/setup-node@v4 - name: Install Prettier - run: npm install -g prettier + run: npm install -g prettier@3.1.0 - name: Run Prettier --check run: prettier --check ${GITHUB_WORKSPACE} diff --git a/conf/test_data.config b/conf/test_data.config new file mode 100644 index 00000000..81ca3fdf --- /dev/null +++ b/conf/test_data.config @@ -0,0 +1,764 @@ +// README: +// https://github.com/nf-core/test-datasets/blob/modules/README.md + +params { + // Base directory for test data + test_data_base = "https://raw.githubusercontent.com/nf-core/test-datasets/modules" + + test_data { + 'sarscov2' { + 'genome' { + genome_fasta = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.fasta" + genome_fasta_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.fasta.gz" + genome_fasta_fai = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.fasta.fai" + genome_fasta_txt_zst = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.fasta.txt.zst" + genome_dict = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.dict" + genome_gff3 = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.gff3" + genome_gff3_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.gff3.gz" + genome_gtf = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.gtf" + genome_paf = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.paf" + genome_sizes = "${params.test_data_base}/data/genomics/sarscov2/genome/genome.sizes" + transcriptome_fasta = "${params.test_data_base}/data/genomics/sarscov2/genome/transcriptome.fasta" + transcriptome_paf = "${params.test_data_base}/data/genomics/sarscov2/genome/transcriptome.paf" + proteome_fasta = "${params.test_data_base}/data/genomics/sarscov2/genome/proteome.fasta" + proteome_fasta_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/proteome.fasta.gz" + + test_bed = "${params.test_data_base}/data/genomics/sarscov2/genome/bed/test.bed" + test_bed_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/bed/test.bed.gz" + test2_bed = "${params.test_data_base}/data/genomics/sarscov2/genome/bed/test2.bed" + test_bed12 = "${params.test_data_base}/data/genomics/sarscov2/genome/bed/test.bed12" + baits_bed = "${params.test_data_base}/data/genomics/sarscov2/genome/bed/baits.bed" + bed_autosql = "${params.test_data_base}/data/genomics/sarscov2/genome/bed/bed6alt.as" + + reference_cnn = "${params.test_data_base}/data/genomics/sarscov2/genome/cnn/reference.cnn" + + kraken2 = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kraken2" + kraken2_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kraken2.tar.gz" + + kraken2_bracken = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kraken2_bracken" + kraken2_bracken_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kraken2_bracken.tar.gz" + + kaiju = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kaiju" + kaiju_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kaiju.tar.gz" + + kofamscan_profiles_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kofamscan/profiles.tar.gz" + kofamscan_ko_list_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/db/kofamscan/ko_list.gz" + + ncbi_taxmap_zip = "${params.test_data_base}/data/genomics/sarscov2/genome/db/maltextract/ncbi_taxmap.zip" + taxon_list_txt = "${params.test_data_base}/data/genomics/sarscov2/genome/db/maltextract/taxon_list.txt" + + mmseqs_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/db/mmseqs.tar.gz" + + all_sites_fas = "${params.test_data_base}/data/genomics/sarscov2/genome/alignment/all_sites.fas" + informative_sites_fas = "${params.test_data_base}/data/genomics/sarscov2/genome/alignment/informative_sites.fas" + + contigs_genome_maf_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/alignment/last/contigs.genome.maf.gz" + contigs_genome_par = "${params.test_data_base}/data/genomics/sarscov2/genome/alignment/last/contigs.genome.par" + lastdb_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/genome/alignment/last/lastdb.tar.gz" + + baits_interval_list = "${params.test_data_base}/data/genomics/sarscov2/genome/picard/baits.interval_list" + targets_interval_list = "${params.test_data_base}/data/genomics/sarscov2/genome/picard/targets.interval_list" + regions_txt = "${params.test_data_base}/data/genomics/sarscov2/genome/graphtyper/regions.txt" + lc_extrap_mr = "${params.test_data_base}/data/delete_me/preseq/SRR1003759_5M_subset.mr" + } + 'illumina' { + test_single_end_bam = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.single_end.bam" + test_single_end_sorted_bam = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.single_end.sorted.bam" + test_single_end_sorted_bam_bai = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.single_end.sorted.bam.bai" + test_paired_end_bam = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.paired_end.bam" + test_paired_end_sorted_bam = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam" + test_paired_end_sorted_bam_bai = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam.bai" + test_paired_end_methylated_bam = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.paired_end.methylated.bam" + test_paired_end_methylated_sorted_bam = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam" + test_paired_end_methylated_sorted_bam_bai = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai" + test_unaligned_bam = "${params.test_data_base}/data/genomics/sarscov2/illumina/bam/test.unaligned.bam" + + test_1_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test_1.fastq.gz" + test_2_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test_2.fastq.gz" + test_interleaved_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test_interleaved.fastq.gz" + test_1_fastq_txt_zst = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test_1.fastq.txt.zst" + test2_1_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test2_1.fastq.gz" + test2_2_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test2_2.fastq.gz" + test_methylated_1_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test.methylated_1.fastq.gz" + test_methylated_2_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastq/test.methylated_2.fastq.gz" + test_1_fastq_gz_fastqc_html = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastqc/test_fastqc.html" + test_1_fastq_gz_fastqc_zip = "${params.test_data_base}/data/genomics/sarscov2/illumina/fastqc/test_fastqc.zip" + + test_bedgraph = "${params.test_data_base}/data/genomics/sarscov2/illumina/bedgraph/test.bedgraph" + + test_bigwig = "${params.test_data_base}/data/genomics/sarscov2/illumina/bigwig/test.bigwig" + + test_wig_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/wig/test.wig.gz" + + test_baserecalibrator_table = "${params.test_data_base}/data/genomics/sarscov2/illumina/gatk/test.baserecalibrator.table" + + test_computematrix_mat_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/deeptools/test.computeMatrix.mat.gz" + + test_bcf = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test.bcf" + + test_vcf = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test.vcf" + test_vcf_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test.vcf.gz" + test_vcf_gz_tbi = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test.vcf.gz.tbi" + test2_vcf = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test2.vcf" + test2_vcf_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test2.vcf.gz" + test2_vcf_gz_tbi = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test2.vcf.gz.tbi" + test2_vcf_targets_tsv_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test2.targets.tsv.gz" + test3_vcf = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test3.vcf" + test3_vcf_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test3.vcf.gz" + test3_vcf_gz_tbi = "${params.test_data_base}/data/genomics/sarscov2/illumina/vcf/test3.vcf.gz.tbi" + + contigs_fasta = "${params.test_data_base}/data/genomics/sarscov2/illumina/fasta/contigs.fasta" + scaffolds_fasta = "${params.test_data_base}/data/genomics/sarscov2/illumina/fasta/scaffolds.fasta" + + assembly_gfa = "${params.test_data_base}/data/genomics/sarscov2/illumina/gfa/assembly.gfa" + assembly_gfa_bgz = "${params.test_data_base}/data/genomics/sarscov2/illumina/gfa/assembly.gfa.bgz" + assembly_gfa_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/gfa/assembly.gfa.gz" + assembly_gfa_zst = "${params.test_data_base}/data/genomics/sarscov2/illumina/gfa/assembly.gfa.zst" + + test_single_end_bam_readlist_txt = "${params.test_data_base}/data/genomics/sarscov2/illumina/picard/test.single_end.bam.readlist.txt" + + SRR13255544_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/sra/SRR13255544.tar.gz" + SRR11140744_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/sra/SRR11140744.tar.gz" + } + 'nanopore' { + test_sorted_bam = "${params.test_data_base}/data/genomics/sarscov2/nanopore/bam/test.sorted.bam" + test_sorted_bam_bai = "${params.test_data_base}/data/genomics/sarscov2/nanopore/bam/test.sorted.bam.bai" + + fast5_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/nanopore/fast5/fast5.tar.gz" + + test_fastq_gz = "${params.test_data_base}/data/genomics/sarscov2/nanopore/fastq/test.fastq.gz" + + test_sequencing_summary = "${params.test_data_base}/data/genomics/sarscov2/nanopore/sequencing_summary/test.sequencing_summary.txt" + } + 'metagenome' { + classified_reads_assignment = "${params.test_data_base}/data/genomics/sarscov2/metagenome/test_1.kraken2.reads.txt" + kraken_report = "${params.test_data_base}/data/genomics/sarscov2/metagenome/test_1.kraken2.report.txt" + krona_taxonomy = "${params.test_data_base}/data/genomics/sarscov2/metagenome/krona_taxonomy.tab" + seqid2taxid_map = "${params.test_data_base}/data/genomics/sarscov2/metagenome/seqid2taxid.map" + nodes_dmp = "${params.test_data_base}/data/genomics/sarscov2/metagenome/nodes.dmp" + names_dmp = "${params.test_data_base}/data/genomics/sarscov2/metagenome/names.dmp" + prot_nodes_dmp = "${params.test_data_base}/data/genomics/sarscov2/metagenome/prot_nodes.dmp" + prot_names_dmp = "${params.test_data_base}/data/genomics/sarscov2/metagenome/prot_names.dmp" + prot_accession2taxid_gz = "${params.test_data_base}/data/genomics/sarscov2/metagenome/prot.accession2taxid.gz" + } + } + 'mus_musculus' { + 'genome' { + rnaseq_samplesheet = "${params.test_data_base}/data/genomics/mus_musculus/rnaseq_expression/SRP254919.samplesheet.csv" + rnaseq_genemeta = "${params.test_data_base}/data/genomics/mus_musculus/rnaseq_expression/SRP254919.gene_meta.tsv" + rnaseq_contrasts = "${params.test_data_base}/data/genomics/mus_musculus/rnaseq_expression/SRP254919.contrasts.csv" + rnaseq_matrix = "${params.test_data_base}/data/genomics/mus_musculus/rnaseq_expression/SRP254919.salmon.merged.gene_counts.top1000cov.tsv" + rnaseq_lengths = "${params.test_data_base}/data/genomics/mus_musculus/rnaseq_expression/SRP254919.spoofed_lengths.tsv" + deseq_results = "${params.test_data_base}/data/genomics/mus_musculus/rnaseq_expression/SRP254919.salmon.merged.deseq2.results.tsv" + genome_19_fasta = "${params.test_data_base}/data/genomics/mus_musculus/genome/chr19.fa.gz" + genome_19_gtf = "${params.test_data_base}/data/genomics/mus_musculus/genome/chr19.filtered.gtf.gz" + } + 'illumina' { + test_1_fastq_gz = "${params.test_data_base}/data/genomics/mus_musculus/mageck/ERR376998.small.fastq.gz" + test_2_fastq_gz = "${params.test_data_base}/data/genomics/mus_musculus/mageck/ERR376999.small.fastq.gz" + genome_config = "${params.test_data_base}/data/genomics/mus_musculus/illumina/10xgenomics/multiome/cellranger_arc_mkref_test_mm39_chr19_config.json" + multiome_lib_csv = "${params.test_data_base}/data/genomics/mus_musculus/illumina/10xgenomics/multiome/lib.csv" + test_scARC_gex_R1_fastq_gz = "${params.test_data_base}/data/genomics/mus_musculus/illumina/10xgenomics/multiome/SRR18907480_chr19_sub_S1_L001_R1_001.fastq.gz" + test_scARC_gex_R2_fastq_gz = "${params.test_data_base}/data/genomics/mus_musculus/illumina/10xgenomics/multiome/SRR18907480_chr19_sub_S1_L001_R2_001.fastq.gz" + test_scARC_atac_R1_fastq_gz = "${params.test_data_base}/data/genomics/mus_musculus/illumina/10xgenomics/multiome/SRR18907481_chr19_sub_S1_L001_R1_001.fastq.gz" + test_scARC_atac_R2_fastq_gz = "${params.test_data_base}/data/genomics/mus_musculus/illumina/10xgenomics/multiome/SRR18907481_chr19_sub_S1_L001_R2_001.fastq.gz" + test_scARC_atac_I2_fastq_gz = "${params.test_data_base}/data/genomics/mus_musculus/illumina/10xgenomics/multiome/SRR18907481_chr19_sub_S1_L001_I2_001.fastq.gz" + } + 'csv' { + count_table = "${params.test_data_base}/data/genomics/mus_musculus/mageck/count_table.csv" + library = "${params.test_data_base}/data/genomics/mus_musculus/mageck/yusa_library.csv" + } + 'txt' { + design_matrix = "${params.test_data_base}/data/genomics/mus_musculus/mageck/design_matrix.txt" + } + } + 'homo_sapiens' { + '10xgenomics' { + cellranger { + test_10x_10k_pbmc_5fb_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5fb/subsampled_sc5p_v2_hs_PBMC_10k_5fb_S1_L001_R1_001.fastq.gz" + test_10x_10k_pbmc_5fb_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5fb/subsampled_sc5p_v2_hs_PBMC_10k_5fb_S1_L001_R2_001.fastq.gz" + test_10x_10k_pbmc_5gex_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5gex/subsampled_sc5p_v2_hs_PBMC_10k_5gex_S1_L001_R1_001.fastq.gz" + test_10x_10k_pbmc_5gex_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/5gex/5gex/subsampled_sc5p_v2_hs_PBMC_10k_5gex_S1_L001_R2_001.fastq.gz" + test_10x_10k_pbmc_b_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/bcell/subsampled_sc5p_v2_hs_PBMC_10k_b_S1_L001_R1_001.fastq.gz" + test_10x_10k_pbmc_b_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/bcell/subsampled_sc5p_v2_hs_PBMC_10k_b_S1_L001_R2_001.fastq.gz" + test_10x_10k_pbmc_t_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/tcell/subsampled_sc5p_v2_hs_PBMC_10k_t_S1_L001_R1_001.fastq.gz" + test_10x_10k_pbmc_t_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/fastqs/tcell/subsampled_sc5p_v2_hs_PBMC_10k_t_S1_L001_R2_001.fastq.gz" + test_10x_10k_pbmc_feature_ref_csv = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc/sc5p_v2_hs_PBMC_10k_multi_5gex_5fb_b_t_feature_ref.csv" + + test_10x_10k_pbmc_cmo_cmo_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/cmo/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_multiplexing_capture_S1_L001_R1_001.fastq.gz" + test_10x_10k_pbmc_cmo_cmo_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/cmo/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_multiplexing_capture_S1_L001_R2_001.fastq.gz" + test_10x_10k_pbmc_cmo_gex1_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/gex_1/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_gex_S2_L001_R1_001.fastq.gz" + test_10x_10k_pbmc_cmo_gex1_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/gex_1/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_1_gex_S2_L001_R2_001.fastq.gz" + test_10x_10k_pbmc_cmo_gex2_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/gex_2/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_2_gex_S1_L001_R1_001.fastq.gz" + test_10x_10k_pbmc_cmo_gex2_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/fastqs/gex_2/subsampled_SC3_v3_NextGem_DI_CellPlex_Human_PBMC_10K_2_gex_S1_L001_R2_001.fastq.gz" + test_10x_10k_pbmc_cmo_feature_ref_csv = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/10k_pbmc_cmo/10k_pbmc_cmo_count_feature_reference.csv" + + test_10x_5k_cmvpos_tcells_ab_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/ab/subsampled_5k_human_antiCMV_T_TBNK_connect_AB_S2_L004_R1_001.fastq.gz" + test_10x_5k_cmvpos_tcells_ab_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/ab/subsampled_5k_human_antiCMV_T_TBNK_connect_AB_S2_L004_R2_001.fastq.gz" + test_10x_5k_cmvpos_tcells_gex1_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/gex_1/subsampled_5k_human_antiCMV_T_TBNK_connect_GEX_1_S1_L001_R1_001.fastq.gz" + test_10x_5k_cmvpos_tcells_gex1_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/gex_1/subsampled_5k_human_antiCMV_T_TBNK_connect_GEX_1_S1_L001_R2_001.fastq.gz" + test_10x_5k_cmvpos_tcells_vdj_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/vdj/subsampled_5k_human_antiCMV_T_TBNK_connect_VDJ_S1_L001_R1_001.fastq.gz" + test_10x_5k_cmvpos_tcells_vdj_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/fastqs/vdj/subsampled_5k_human_antiCMV_T_TBNK_connect_VDJ_S1_L001_R2_001.fastq.gz" + test_10x_5k_cmvpos_tcells_feature_ref_csv = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/5k_cmvpos_tcells/5k_human_antiCMV_T_TBNK_connect_Multiplex_count_feature_reference.csv" + + test_10x_vdj_ref_json = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/references/vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-5.0.0/reference.json" + test_10x_vdj_ref_fasta = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/references/vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-5.0.0/fasta/regions.fa" + test_10x_vdj_ref_suppfasta = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger/references/vdj/refdata-cellranger-vdj-GRCh38-alts-ensembl-5.0.0/fasta/supp_regions.fa" + + test_scATAC_1_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger-atac/test_scATAC_S1_L001_R1_001.fastq.gz" + test_scATAC_2_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger-atac/test_scATAC_S1_L001_R2_001.fastq.gz" + test_scATAC_3_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger-atac/test_scATAC_S1_L001_R3_001.fastq.gz" + test_scATAC_I_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/cellranger-atac/test_scATAC_S1_L001_I1_001.fastq.gz" + } + spaceranger { + test_10x_ffpe_cytassist_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/spaceranger/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/CytAssist_11mm_FFPE_Human_Glioblastoma_2_S1_L001_R1_001.fastq.gz" + test_10x_ffpe_cytassist_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/spaceranger/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/CytAssist_11mm_FFPE_Human_Glioblastoma_2_S1_L001_R2_001.fastq.gz" + test_10x_ffpe_cytassist_image = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/spaceranger/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/CytAssist_11mm_FFPE_Human_Glioblastoma_image.tif" + test_10x_ffpe_cytassist_probeset = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/spaceranger/human-brain-cancer-11-mm-capture-area-ffpe-2-standard_v2_ffpe_cytassist/CytAssist_11mm_FFPE_Human_Glioblastoma_probe_set.csv" + + test_10x_ffpe_v1_fastq_1_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/spaceranger/human-ovarian-cancer-1-standard_v1_ffpe/Visium_FFPE_Human_Ovarian_Cancer_S1_L001_R1_001.fastq.gz" + test_10x_ffpe_v1_fastq_2_gz = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/spaceranger/human-ovarian-cancer-1-standard_v1_ffpe/Visium_FFPE_Human_Ovarian_Cancer_S1_L001_R2_001.fastq.gz" + test_10x_ffpe_v1_image = "${params.test_data_base}/data/genomics/homo_sapiens/10xgenomics/spaceranger/human-ovarian-cancer-1-standard_v1_ffpe/Visium_FFPE_Human_Ovarian_Cancer_image.jpg" + } + } + 'genome' { + genome_elfasta = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.elfasta" + genome_fasta = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.fasta" + genome_fasta_fai = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.fasta.fai" + genome_fasta_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.fasta.gz" + genome_fasta_gz_fai = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.fasta.gz.fai" + genome_fasta_gz_gzi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.fasta.gz.gzi" + genome_strtablefile = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome_strtablefile.zip" + genome_dict = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.dict" + genome_gff3 = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.gff3" + genome_gtf = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.gtf" + genome_interval_list = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.interval_list" + genome_multi_interval_bed = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.multi_intervals.bed" + genome_blacklist_interval_bed = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.blacklist_intervals.bed" + genome_sizes = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.sizes" + genome_bed = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.bed" + genome_header = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.header" + genome_bed_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.bed.gz" + genome_bed_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.bed.gz.tbi" + genome_elsites = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.elsites" + transcriptome_fasta = "${params.test_data_base}/data/genomics/homo_sapiens/genome/transcriptome.fasta" + genome2_fasta = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome2.fasta" + genome_chain_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.chain.gz" + genome_annotated_interval_tsv = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.annotated_intervals.tsv" + genome_mt_gb = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.NC_012920_1.gb" + genome_preprocessed_count_tsv = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.preprocessed_intervals.counts.tsv" + genome_preprocessed_interval_list = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.preprocessed_intervals.interval_list" + genome_ploidy_model = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.ploidy_model.tar.gz" + genome_ploidy_calls = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.ploidy_calls.tar.gz" + genome_germline_cnv_model = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.germline_cnv_model.tar.gz" + genome_germline_cnv_calls = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome.germline_cnv_calls.tar.gz" + genome_motifs = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome_motifs.txt" + genome_config = "${params.test_data_base}/data/genomics/homo_sapiens/genome/genome_config.json" + + genome_1_fasta = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr1/genome.fasta.gz" + genome_1_gtf = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr1/genome.gtf" + + genome_21_sdf = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/genome_sdf.tar.gz" + genome_21_fasta = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/genome.fasta" + genome_21_fasta_fai = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/genome.fasta.fai" + genome_21_gencode_gtf = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/chr21_gencode.gtf" + genome_21_dict = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/genome.dict" + genome_21_sizes = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/genome.sizes" + genome_21_interval_list = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/genome.interval_list" + genome_21_annotated_bed = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/annotated.bed" + genome_21_multi_interval_bed = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.bed" + genome_21_multi_interval_antitarget_bed = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.antitarget.bed" + genome_21_multi_interval_bed_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.bed.gz" + genome_21_multi_interval_bed_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.bed.gz.tbi" + genome_21_chromosomes_dir = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/chromosomes.tar.gz" + genome_21_reference_cnn = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/reference_chr21.cnn" + genome_21_eigenstrat_snp = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/chr_21.snp" + genome_21_stitch_posfile = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/sequence/dbsnp_138.hg38.first_10_biallelic_sites.tsv" + + dbsnp_146_hg38_elsites = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/dbsnp_146.hg38.elsites" + dbsnp_146_hg38_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/dbsnp_146.hg38.vcf.gz" + dbsnp_146_hg38_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/dbsnp_146.hg38.vcf.gz.tbi" + gnomad_r2_1_1_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/gnomAD.r2.1.1.vcf.gz" + gnomad_r2_1_1_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/gnomAD.r2.1.1.vcf.gz.tbi" + mills_and_1000g_indels_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/mills_and_1000G.indels.vcf.gz" + mills_and_1000g_indels_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/mills_and_1000G.indels.vcf.gz.tbi" + syntheticvcf_short_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/syntheticvcf_short.vcf.gz" + syntheticvcf_short_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/syntheticvcf_short.vcf.gz.tbi" + syntheticvcf_short_score = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/syntheticvcf_short.score" + gnomad_r2_1_1_sv_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/gnomAD.r2.1.1-sv.vcf.gz" + gnomad2_r2_1_1_sv_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/gnomAD2.r2.1.1-sv.vcf.gz" + + hapmap_3_3_hg38_21_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/hapmap_3.3.hg38.vcf.gz" + hapmap_3_3_hg38_21_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/hapmap_3.3.hg38.vcf.gz.tbi" + res_1000g_omni2_5_hg38_21_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/1000G_omni2.5.hg38.vcf.gz" + res_1000g_omni2_5_hg38_21_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/1000G_omni2.5.hg38.vcf.gz.tbi" + res_1000g_phase1_snps_hg38_21_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/1000G_phase1.snps.hg38.vcf.gz" + res_1000g_phase1_snps_hg38_21_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/1000G_phase1.snps.hg38.vcf.gz.tbi" + dbsnp_138_hg38_21_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/dbsnp_138.hg38.vcf.gz" + dbsnp_138_hg38_21_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/dbsnp_138.hg38.vcf.gz.tbi" + gnomad_r2_1_1_21_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/gnomAD.r2.1.1.vcf.gz" + gnomad_r2_1_1_21_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/gnomAD.r2.1.1.vcf.gz.tbi" + mills_and_1000g_indels_21_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/mills_and_1000G.indels.hg38.vcf.gz" + mills_and_1000g_indels_21_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/mills_and_1000G.indels.hg38.vcf.gz.tbi" + haplotype_map = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/haplotype_map.txt" + dbNSFP_4_1a_21_hg38_txt_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/dbNSFP4.1a.21.txt.gz" + dbNSFP_4_1a_21_hg38_txt_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/dbNSFP4.1a.21.txt.gz.tbi" + ngscheckmate_bed = "${params.test_data_base}/data/genomics/homo_sapiens/genome/chr21/germlineresources/SNP_GRCh38_hg38_wChr.bed" + + index_salmon = "${params.test_data_base}/data/genomics/homo_sapiens/genome/index/salmon" + repeat_expansions = "${params.test_data_base}/data/genomics/homo_sapiens/genome/loci/repeat_expansions.json" + justhusky_ped = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/ped/justhusky.ped" + justhusky_minimal_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/ped/justhusky_minimal.vcf.gz" + justhusky_minimal_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/ped/justhusky_minimal.vcf.gz.tbi" + + vcfanno_tar_gz = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/vcfanno/vcfanno_grch38_module_test.tar.gz" + vcfanno_toml = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vcf/vcfanno/vcfanno.toml" + updsites_bed = "${params.test_data_base}/data/genomics/homo_sapiens/genome/updsites.bed" + + prg_input = "${params.test_data_base}/data/genomics/homo_sapiens/genome/PRG_test.zip" + crispr_functional_counts = "${params.test_data_base}/data/genomics/homo_sapiens/genome/tsv/functional_genomics_counts.tsv" + crispr_functional_library = "${params.test_data_base}/data/genomics/homo_sapiens/genome/tsv/library_functional_genomics.tsv" + + vep_cache = "${params.test_data_base}/data/genomics/homo_sapiens/genome/vep.tar.gz" + affy_array_samplesheet = "${params.test_data_base}/data/genomics/homo_sapiens/array_expression/GSE38751.csv" + affy_array_celfiles_tar = "${params.test_data_base}/data/genomics/homo_sapiens/array_expression/GSE38751_RAW.tar" + + } + 'pangenome' { + pangenome_fa = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.fa" + pangenome_fa_bgzip = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.fa.gz" + pangenome_fa_bgzip_fai = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.fa.gz.fai" + pangenome_fa_bgzip_gzi = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.fa.gz.gzi" + pangenome_paf = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.paf" + pangenome_paf_gz = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.paf.gz" + pangenome_panacus_tsv = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.panacus.tsv" + pangenome_seqwish_gfa = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.seqwish.gfa" + pangenome_smoothxg_gfa = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.smoothxg.gfa" + pangenome_gfaffix_gfa = "${params.test_data_base}/data/pangenomics/homo_sapiens/pangenome.gfaffix.gfa" + 'odgi' { + pangenome_og = "${params.test_data_base}/data/pangenomics/homo_sapiens/odgi/pangenome.og" + pangenome_lay = "${params.test_data_base}/data/pangenomics/homo_sapiens/odgi/pangenome.lay" + } + } + 'illumina' { + test_paired_end_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam" + test_paired_end_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam.bai" + test_paired_end_name_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.name.sorted.bam" + test_paired_end_markduplicates_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.markduplicates.sorted.bam" + test_paired_end_markduplicates_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.markduplicates.sorted.bam.bai" + test_paired_end_markduplicates_sorted_referencesn_txt = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.markduplicates.sorted.referencesn.txt" + test_paired_end_recalibrated_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.recalibrated.sorted.bam" + test_paired_end_recalibrated_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.paired_end.recalibrated.sorted.bam.bai" + test_paired_end_umi_consensus_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.umi_consensus.bam" + test_paired_end_umi_converted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.umi_converted.bam" + test_paired_end_umi_grouped_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.umi_grouped.bam" + test_paired_end_umi_histogram_txt = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.umi_histogram.txt" + test_paired_end_umi_unsorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.umi_unsorted.bam" + test_paired_end_umi_unsorted_tagged_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.unsorted_tagged.bam" + test_paired_end_hla = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/example_hla_pe.bam" + test_paired_end_hla_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/example_hla_pe.sorted.bam" + test_paired_end_hla_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/example_hla_pe.sorted.bam.bai" + test_rna_paired_end_sorted_chr6_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.rna.paired_end.sorted.chr6.bam" + test_rna_paired_end_sorted_chr6_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test.rna.paired_end.sorted.chr6.bam.bai" + + test2_paired_end_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam" + test2_paired_end_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam.bai" + test2_paired_end_name_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test2.paired_end.name.sorted.bam" + test2_paired_end_markduplicates_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test2.paired_end.markduplicates.sorted.bam" + test2_paired_end_markduplicates_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test2.paired_end.markduplicates.sorted.bam.bai" + test2_paired_end_recalibrated_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test2.paired_end.recalibrated.sorted.bam" + test2_paired_end_recalibrated_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test2.paired_end.recalibrated.sorted.bam.bai" + test2_paired_end_umi_consensus_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test2.paired_end.umi_consensus.bam" + test2_paired_end_umi_converted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test2.paired_end.umi_converted.bam" + test2_paired_end_umi_grouped_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test2.paired_end.umi_grouped.bam" + test2_paired_end_umi_histogram_txt = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test2.paired_end.umi_histogram.txt" + test2_paired_end_umi_unsorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test2.paired_end.umi_unsorted.bam" + test2_paired_end_umi_unsorted_tagged_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test2.paired_end.unsorted_tagged.bam" + test_paired_end_duplex_umi_unmapped_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.duplex_umi_unmapped.bam" + test_paired_end_duplex_umi_mapped_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.duplex_umi_mapped.bam" + test_paired_end_duplex_umi_mapped_tagged_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.duplex_umi_mapped_tagged.bam" + test_paired_end_duplex_umi_grouped_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.duplex_umi_grouped.bam" + test_paired_end_duplex_umi_duplex_consensus_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/umi/test.paired_end.duplex_umi_duplex_consensus.bam" + + mitochon_standin_recalibrated_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/mitochon_standin.recalibrated.sorted.bam" + mitochon_standin_recalibrated_sorted_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/mitochon_standin.recalibrated.sorted.bam.bai" + test_illumina_mt_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test_illumina_mt.bam" + test_illumina_mt_bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test_illumina_mt.bam.bai" + + test3_single_end_markduplicates_sorted_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/test3.single_end.markduplicates.sorted.bam" + + read_group_settings_txt = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bam/read_group_settings.txt" + + test_paired_end_sorted_cram = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram" + test_paired_end_sorted_cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram.crai" + test_paired_end_markduplicates_sorted_cram = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test.paired_end.markduplicates.sorted.cram" + test_paired_end_markduplicates_sorted_cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test.paired_end.markduplicates.sorted.cram.crai" + test_paired_end_recalibrated_sorted_cram = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test.paired_end.recalibrated.sorted.cram" + test_paired_end_recalibrated_sorted_cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test.paired_end.recalibrated.sorted.cram.crai" + + test2_paired_end_sorted_cram = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.sorted.cram" + test2_paired_end_sorted_cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.sorted.cram.crai" + test2_paired_end_markduplicates_sorted_cram = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.markduplicates.sorted.cram" + test2_paired_end_markduplicates_sorted_cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.markduplicates.sorted.cram.crai" + test2_paired_end_recalibrated_sorted_cram = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.recalibrated.sorted.cram" + test2_paired_end_recalibrated_sorted_cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test2.paired_end.recalibrated.sorted.cram.crai" + test3_paired_end_recalibrated_sorted_cram = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test3.paired_end.recalibrated.sorted.cram" + test3_paired_end_recalibrated_sorted_cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/cram/test3.paired_end.recalibrated.sorted.cram.crai" + + test_1_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test_1.fastq.gz" + test_2_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test_2.fastq.gz" + test_umi_1_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test.umi_1.fastq.gz" + test_umi_2_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test.umi_2.fastq.gz" + test_airrseq_1_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test_airrseq_umi_R1.fastq.gz" + test_airrseq_2_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test_airrseq_R2.fastq.gz" + test2_1_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test2_1.fastq.gz" + test2_2_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test2_2.fastq.gz" + test2_umi_1_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test2.umi_1.fastq.gz" + test2_umi_2_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test2.umi_2.fastq.gz" + test_rnaseq_1_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test_rnaseq_1.fastq.gz" + test_rnaseq_2_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test_rnaseq_2.fastq.gz" + test_paired_end_duplex_umi_1_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test_duplex_umi_1.fastq.gz" + test_paired_end_duplex_umi_2_fastq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/fastq/test_duplex_umi_2.fastq.gz" + + test_baserecalibrator_table = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test.baserecalibrator.table" + test2_baserecalibrator_table = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test2.baserecalibrator.table" + test_pileups_table = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test.pileups.table" + test2_pileups_table = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test2.pileups.table" + + test_paired_end_sorted_dragstrmodel = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test_paired_end_sorted_dragstrmodel.txt" + + test_genomicsdb_tar_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test_genomicsdb.tar.gz" + test_pon_genomicsdb_tar_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test_pon_genomicsdb.tar.gz" + + test2_haplotc_ann_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/haplotypecaller_calls/test2_haplotc.ann.vcf.gz" + test2_haplotc_ann_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/haplotypecaller_calls/test2_haplotc.ann.vcf.gz.tbi" + test_haplotc_cnn_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/haplotypecaller_calls/test_haplotcaller.cnn.vcf.gz" + test_haplotc_cnn_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/haplotypecaller_calls/test_haplotcaller.cnn.vcf.gz.tbi" + + test2_haplotc_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/haplotypecaller_calls/test2_haplotc.vcf.gz" + test2_haplotc_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/haplotypecaller_calls/test2_haplotc.vcf.gz.tbi" + + test2_recal = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/variantrecalibrator/test2.recal" + test2_recal_idx = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/variantrecalibrator/test2.recal.idx" + test2_tranches = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/variantrecalibrator/test2.tranches" + test2_allele_specific_recal = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/variantrecalibrator/test2_allele_specific.recal" + test2_allele_specific_recal_idx = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/variantrecalibrator/test2_allele_specific.recal.idx" + test2_allele_specific_tranches = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/variantrecalibrator/test2_allele_specific.tranches" + + test_test2_paired_mutect2_calls_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/paired_mutect2_calls/test_test2_paired_mutect2_calls.vcf.gz" + test_test2_paired_mutect2_calls_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/paired_mutect2_calls/test_test2_paired_mutect2_calls.vcf.gz.tbi" + test_test2_paired_mutect2_calls_vcf_gz_stats = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/paired_mutect2_calls/test_test2_paired_mutect2_calls.vcf.gz.stats" + test_test2_paired_mutect2_calls_f1r2_tar_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/paired_mutect2_calls/test_test2_paired_mutect2_calls.f1r2.tar.gz" + test_test2_paired_mutect2_calls_artifact_prior_tar_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test_test2_paired_mutect2_calls.artifact-prior.tar.gz" + test_test2_paired_segmentation_table = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test_test2_paired.segmentation.table" + test_test2_paired_contamination_table = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/test_test2_paired.contamination.table" + + test_genome_vcf = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test.genome.vcf" + test_genome_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz" + test_genome_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.gz.tbi" + test_genome_vcf_idx = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test.genome.vcf.idx" + + test_genome_vcf_ud = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/svd/test.genome.vcf.UD" + test_genome_vcf_mu = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/svd/test.genome.vcf.mu" + test_genome_vcf_bed = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/svd/test.genome.vcf.bed" + + test2_genome_vcf = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test2.genome.vcf" + test2_genome_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test2.genome.vcf.gz" + test2_genome_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test2.genome.vcf.gz.tbi" + test2_genome_vcf_idx = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gvcf/test2.genome.vcf.idx" + + test_genome21_indels_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/test.genome_21.somatic_sv.vcf.gz" + test_genome21_indels_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/test.genome_21.somatic_sv.vcf.gz.tbi" + + test_mpileup = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/mpileup/test.mpileup.gz" + test2_mpileup = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/mpileup/test2.mpileup.gz" + + test_broadpeak = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/broadpeak/test.broadPeak" + test2_broadpeak = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/broadpeak/test2.broadPeak" + + test_narrowpeak = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/narrowpeak/test.narrowPeak" + test2_narrowpeak = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/narrowpeak/test2.narrowPeak" + + test_yak = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/yak/test.yak" + test2_yak = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/yak/test2.yak" + + cutandrun_bedgraph_test_1 = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bedgraph/cutandtag_h3k27me3_test_1.bedGraph" + cutandrun_bedgraph_test_2 = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bedgraph/cutandtag_igg_test_1.bedGraph" + na24385_chr22_coverage = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bedgraph/NA24385_coverage.bed" + + empty_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/empty.vcf.gz" + empty_vcf_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/empty.vcf.gz.tbi" + + simulated_sv = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/chr21/simulated_sv.vcf.gz" + simulated_sv_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/chr21/simulated_sv.vcf.gz.tbi" + simulated_sv2 = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/chr21/simulated_sv2.vcf.gz" + simulated_sv2_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/chr21/simulated_sv2.vcf.gz.tbi" + + test_rnaseq_vcf = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/test.rnaseq.vcf" + test_sv_vcf = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/sv_query.vcf.gz" + test_sv_vcf_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/sv_query.vcf.gz.tbi" + na24385_chr22_sv_vcf = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/NA24385_sv.vcf.gz" + na24385_chr22_sv_vcf_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/NA24385_sv.vcf.gz.tbi" + genmod_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/genmod.vcf.gz" + genmod_annotate_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/test_annotate.vcf.gz" + genmod_models_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/test_models.vcf.gz" + genmod_score_vcf_gz = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/test_score.vcf.gz" + + test_mito_vcf = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/vcf/NA12878_chrM.vcf.gz" + + test_pytor = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/pytor/test.pytor" + rank_model = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/genmod/svrank_model_-v1.8-.ini" + + test_flowcell = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bcl/flowcell.tar.gz" + test_flowcell_samplesheet = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/bcl/flowcell_samplesheet.csv" + + varlociraptor_scenario = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/varlociraptor/scenario.yml" + + contig_ploidy_priors_table = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/gatk/contig_ploidy_priors_table.tsv" + + purecn_ex1_bam = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/purecn/purecn_ex1.bam" + purecn_ex1_bai = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/purecn/purecn_ex1.bam.bai" + purecn_ex1_interval = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/purecn/purecn_ex1_intervals.txt" + purecn_ex1_normal = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/purecn/purecn_ex1_normal.txt.gz" + purecn_ex2_normal = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/purecn/purecn_ex2_normal.txt.gz" + purecn_normalpanel_vcf = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/purecn/purecn_normalpanel.vcf.gz" + purecn_normalpanel_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/illumina/purecn/purecn_normalpanel.vcf.gz.tbi" + } + 'pacbio' { + primers = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/fasta/primers.fasta" + alz = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.bam" + alzpbi = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.bam.pbi" + ccs = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.ccs.bam" + ccs_fa = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/fasta/alz.ccs.fasta" + ccs_fa_gz = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/fasta/alz.ccs.fasta.gz" + ccs_fq = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/fastq/alz.ccs.fastq" + ccs_fq_gz = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/fastq/alz.ccs.fastq.gz" + ccs_xml = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/xml/alz.ccs.consensusreadset.xml" + hifi = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/fastq/test_hifi.fastq.gz" + lima = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.bam" + refine = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.flnc.bam" + cluster = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.flnc.clustered.bam" + singletons = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.flnc.clustered.singletons.bam" + aligned = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.flnc.clustered.singletons.merged.aligned.bam" + alignedbai = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bam/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.flnc.clustered.singletons.merged.aligned.bam.bai" + genemodel1 = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bed/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.flnc.clustered.singletons.merged.aligned_tc.bed" + genemodel2 = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/bed/alz.ccs.fl.NEB_5p--NEB_Clontech_3p.flnc.clustered.singletons.merged.aligned_tc.2.bed" + filelist = "${params.test_data_base}/data/genomics/homo_sapiens/pacbio/txt/filelist.txt" + } + 'scramble' { + fasta = "${params.test_data_base}/data/genomics/homo_sapiens/scramble/test.fa" + fasta_fai = "${params.test_data_base}/data/genomics/homo_sapiens/scramble/test.fa.fai" + bam = "${params.test_data_base}/data/genomics/homo_sapiens/scramble/test.bam" + bam_bai = "${params.test_data_base}/data/genomics/homo_sapiens/scramble/test.bam.bai" + cram = "${params.test_data_base}/data/genomics/homo_sapiens/scramble/test.cram" + cram_crai = "${params.test_data_base}/data/genomics/homo_sapiens/scramble/test.cram.crai" + bed = "${params.test_data_base}/data/genomics/homo_sapiens/scramble/test.bed" + } + 'gene_set_analysis' { + gct = "${params.test_data_base}/data/genomics/homo_sapiens/gene_set_analysis/P53_6samples_collapsed_symbols.gct" + cls = "${params.test_data_base}/data/genomics/homo_sapiens/gene_set_analysis/P53_6samples.cls" + gmx = "${params.test_data_base}/data/genomics/homo_sapiens/gene_set_analysis/c1.symbols.reduced.gmx" + } + 'cnvkit' { + amplicon_cnr = "https://raw.githubusercontent.com/etal/cnvkit/v0.9.9/test/formats/amplicon.cnr" + amplicon_cns = "https://raw.githubusercontent.com/etal/cnvkit/v0.9.9/test/formats/amplicon.cns" + } + } + 'bacteroides_fragilis' { + 'genome' { + genome_fna_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.fna.gz" + genome_gbff_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.gbff.gz" + genome_paf = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.paf" + genome_gff_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/genome/genome.gff.gz" + + } + 'hamronization' { + genome_abricate_tsv = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/hamronization/genome.abricate.tsv" + genome_mapping_potential_arg = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/hamronization/genome.mapping.potential.ARG" + } + 'illumina' { + test1_contigs_fa_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fasta/test1.contigs.fa.gz" + test1_1_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test1_1.fastq.gz" + test1_2_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test1_2.fastq.gz" + test2_1_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test2_1.fastq.gz" + test2_2_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/fastq/test2_2.fastq.gz" + test1_paired_end_bam = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/bam/test1.bam" + test1_paired_end_sorted_bam = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/bam/test1.sorted.bam" + test1_paired_end_sorted_bam_bai = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/bam/test1.sorted.bam.bai" + test2_paired_end_bam = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/bam/test2.bam" + test2_paired_end_sorted_bam = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/bam/test2.sorted.bam" + test2_paired_end_sorted_bam_bai = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/illumina/bam/test2.sorted.bam.bai" + } + 'nanopore' { + test_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/nanopore/fastq/test.fastq.gz" + overlap_paf = "${params.test_data_base}/data/genomics/prokaryotes/bacteroides_fragilis/nanopore/overlap.paf" + } + } + 'candidatus_portiera_aleyrodidarum' { + 'genome' { + genome_fasta = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/genome.fasta" + genome_sizes = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/genome.sizes" + genome_aln_gz = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/genome.aln.gz" + genome_aln_nwk = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/genome.aln.nwk" + proteome_fasta = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/proteome.fasta" + test1_gff = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/gff/test1.gff" + test2_gff = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/gff/test2.gff" + test3_gff = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/gff/test3.gff" + } + 'illumina' { + test_1_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/illumina/fastq/test_1.fastq.gz" + test_2_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/illumina/fastq/test_2.fastq.gz" + test_se_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/illumina/fastq/test_se.fastq.gz" + } + 'nanopore' { + test_fastq_gz = "${params.test_data_base}/data/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/nanopore/fastq/test.fastq.gz" + } + } + 'haemophilus_influenzae' { + 'genome' { + genome_fna_gz = "${params.test_data_base}/data/genomics/prokaryotes/haemophilus_influenzae/genome/genome.fna.gz" + genome_aln_gz = "${params.test_data_base}/data/genomics/prokaryotes/haemophilus_influenzae/genome/genome.aln.gz" + genome_aln_nwk = "${params.test_data_base}/data/genomics/prokaryotes/haemophilus_influenzae/genome/genome.aln.nwk" + } + } + 'generic' { + 'csv' { + test_csv = "${params.test_data_base}/data/generic/csv/test.csv" + } + 'notebooks' { + rmarkdown = "${params.test_data_base}/data/generic/notebooks/rmarkdown/rmarkdown_notebook.Rmd" + ipython_md = "${params.test_data_base}/data/generic/notebooks/jupyter/ipython_notebook.md" + ipython_ipynb = "${params.test_data_base}/data/generic/notebooks/jupyter/ipython_notebook.ipynb" + } + 'tar' { + tar_gz = "${params.test_data_base}/data/generic/tar/hello.tar.gz" + } + 'tsv' { + test_tsv = "${params.test_data_base}/data/generic/tsv/test.tsv" + } + 'txt' { + hello = "${params.test_data_base}/data/generic/txt/hello.txt" + } + 'cooler'{ + test_pairix_pair_gz = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/cload/hg19/hg19.GM12878-MboI.pairs.subsample.blksrt.txt.gz" + test_pairix_pair_gz_px2 = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/cload/hg19/hg19.GM12878-MboI.pairs.subsample.blksrt.txt.gz.px2" + test_pairs_pair = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/cload/hg19/hg19.sample1.pairs" + test_tabix_pair_gz = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/cload/hg19/hg19.GM12878-MboI.pairs.subsample.sorted.possrt.txt.gz" + test_tabix_pair_gz_tbi = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/cload/hg19/hg19.GM12878-MboI.pairs.subsample.sorted.possrt.txt.gz.tbi" + hg19_chrom_sizes = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/cload/hg19/hg19.chrom.sizes" + test_merge_cool = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/merge/toy/toy.symm.upper.2.cool" + test_merge_cool_cp2 = "${params.test_data_base}/data/genomics/homo_sapiens/cooler/merge/toy/toy.symm.upper.2.cp2.cool" + + } + 'pairtools' { + mock_4dedup_pairsam = "${params.test_data_base}/data/genomics/homo_sapiens/pairtools/mock.4dedup.pairsam" + mock_4flip_pairs = "${params.test_data_base}/data/genomics/homo_sapiens/pairtools/mock.4flip.pairs" + mock_chrom_sizes = "${params.test_data_base}/data/genomics/homo_sapiens/pairtools/mock.chrom.sizes" + mock_pairsam = "${params.test_data_base}/data/genomics/homo_sapiens/pairtools/mock.pairsam" + mock_sam = "${params.test_data_base}/data/genomics/homo_sapiens/pairtools/mock.sam" + frag_bed = "${params.test_data_base}/data/genomics/homo_sapiens/pairtools/frag.bed" + } + 'estsfs'{ + config_file = "${params.test_data_base}/data/delete_me/estsfs/config-JC.txt" + data_file = "${params.test_data_base}/data/delete_me/estsfs/TEST-DATA.TXT" + seed_file = "${params.test_data_base}/data/delete_me/estsfs/seedfile.txt" + } + 'config' { + ncbi_user_settings = "${params.test_data_base}/data/generic/config/ncbi_user_settings.mkfg" + } + 'unsorted_data' { + 'unsorted_text' { + genome_file = "${params.test_data_base}/data/generic/unsorted_data/unsorted_text/test.genome" + intervals = "${params.test_data_base}/data/generic/unsorted_data/unsorted_text/test.bed" + numbers_csv = "${params.test_data_base}/data/generic/unsorted_data/unsorted_text/test.csv" + } + } + } + 'proteomics' { + 'msspectra' { + ups_file1 = "${params.test_data_base}/data/proteomics/msspectra/OVEMB150205_12.raw" + ups_file2 = "${params.test_data_base}/data/proteomics/msspectra/OVEMB150205_14.raw" + profile_spectra = "${params.test_data_base}/data/proteomics/msspectra/peakpicker_tutorial_1.mzML" + } + 'database' { + yeast_ups = "${params.test_data_base}/data/proteomics/database/yeast_UPS.fasta" + } + 'maxquant' { + mq_contrasts = "${params.test_data_base}/data/proteomics/maxquant/MaxQuant_contrasts.csv" + mq_proteingroups = "${params.test_data_base}/data/proteomics/maxquant/MaxQuant_proteinGroups.txt" + mq_samplesheet = "${params.test_data_base}/data/proteomics/maxquant/MaxQuant_samplesheet.tsv" + mq_proteus_mat = "${params.test_data_base}/data/proteomics/maxquant/proteus.raw_MaxQuant_proteingroups_tab.tsv" + } + 'parameter' { + maxquant = "${params.test_data_base}/data/proteomics/parameter/mqpar.xml" + } + 'openms' { + idxml1 = "${params.test_data_base}/data/proteomics/openms/HepG2_rep1_small.idXML" + idxml2 = "${params.test_data_base}/data/proteomics/openms/HepG2_rep2_small.idXML" + } + 'pdb' { + tim1_pdb = "${params.test_data_base}/data/proteomics/pdb/1tim.pdb" + tim8_pdb = "${params.test_data_base}/data/proteomics/pdb/8tim.pdb" + } + } + 'galaxea_fascicularis' { + hic { + pretext = "${params.test_data_base}/data/genomics/eukaryotes/galaxea_fascicularis/hic/jaGalFasc40_2.pretext" + } + } + 'deilephila_porcellus' { + 'mito' { + ref_fa = "${params.test_data_base}/data/genomics/eukaryotes/deilephila_porcellus/mito/MW539688.1.fasta" + ref_gb = "${params.test_data_base}/data/genomics/eukaryotes/deilephila_porcellus/mito/MW539688.1.gb" + hifi_reads = "${params.test_data_base}/data/genomics/eukaryotes/deilephila_porcellus/mito/ilDeiPorc1.HiFi.reads.fa" + contigs = "${params.test_data_base}/data/genomics/eukaryotes/deilephila_porcellus/mito/ilDeiPorc1.contigs.fa" + } + } + 'imaging' { + 'h5' { + plant_wga = "${params.test_data_base}/data/imaging/h5/plant_wga.h5" + plant_wga_prob = "${params.test_data_base}/data/imaging/h5/plant_wga_probabilities.h5" + } + 'ilp' { + plant_wga_multicut = "${params.test_data_base}/data/imaging/ilp/plant_wga.multicut.ilp" + plant_wga_pixel_class = "${params.test_data_base}/data/imaging/ilp/plant_wga.pixel_prob.ilp" + } + 'tiff' { + mouse_heart_wga = "${params.test_data_base}/data/imaging/tiff/mindagap.mouse_heart.wga.tiff" + } + 'ome-tiff' { + cycif_tonsil_channels = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-channels.csv" + cycif_tonsil_cycle1 = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-cycle1.ome.tif" + cycif_tonsil_cycle2 = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-cycle2.ome.tif" + cycif_tonsil_cycle3 = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-cycle3.ome.tif" + cycif_tonsil_dfp = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-dfp.ome.tif" + cycif_tonsil_ffp = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-ffp.ome.tif" + } + 'registration' { + markers = "${params.test_data_base}/data/imaging/registration/markers.csv" + cycle1 = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-cycle1.ome.tif" + cycle2 = "${params.test_data_base}/data/imaging/ome-tiff/cycif-tonsil-cycle2.ome.tif" + } + 'segmentation' { + markers = "${params.test_data_base}/data/imaging/segmentation/markers.csv" + image = "${params.test_data_base}/data/imaging/segmentation/cycif_tonsil_registered.ome.tif" + } + 'quantification' { + markers = "${params.test_data_base}/data/imaging/quantification/markers.csv" + image = "${params.test_data_base}/data/imaging/quantification/cycif_tonsil_registered.ome.tif" + mask = "${params.test_data_base}/data/imaging/quantification/cell.ome.tif" + } + 'downstream' { + markers = "${params.test_data_base}/data/imaging/downstream/markers.csv" + cell_feature_array = "${params.test_data_base}/data/imaging/downstream/cycif_tonsil_cell.csv" + } + 'background_subtraction' { + markers = "${params.test_data_base}/data/imaging/background_subtraction/markers.csv" + image = "${params.test_data_base}/data/imaging/background_subtraction/cycif_tonsil_registered.ome.tif" + } + 'core_detection' { + image = "${params.test_data_base}/data/imaging/core_detection/single_core_dapi.tif" + } + } + } +} diff --git a/modules.json b/modules.json index 89828a1c..8f4b03a6 100644 --- a/modules.json +++ b/modules.json @@ -78,7 +78,7 @@ }, "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "bba7e362e4afead70653f84d8700588ea28d0f9e", + "git_sha": "8ec825f465b9c17f9d83000022995b4f7de6fe93", "installed_by": ["modules"] }, "dastool/dastool": { @@ -98,7 +98,7 @@ }, "fastqc": { "branch": "master", - "git_sha": "65ad3e0b9a4099592e1102e92e10455dc661cf53", + "git_sha": "617777a807a1770f73deb38c80004bac06807eef", "installed_by": ["modules"] }, "freebayes": { diff --git a/modules/nf-core/custom/dumpsoftwareversions/environment.yml b/modules/nf-core/custom/dumpsoftwareversions/environment.yml index f0c63f69..9b3272bc 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/environment.yml +++ b/modules/nf-core/custom/dumpsoftwareversions/environment.yml @@ -4,4 +4,4 @@ channels: - bioconda - defaults dependencies: - - bioconda::multiqc=1.17 + - bioconda::multiqc=1.19 diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf index 7685b33c..f2187611 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -4,8 +4,8 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.17--pyhdfd78af_0' : - 'biocontainers/multiqc:1.17--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.19--pyhdfd78af_0' : + 'biocontainers/multiqc:1.19--pyhdfd78af_0' }" input: path versions diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test index eec1db10..b1e1630b 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test +++ b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test @@ -31,7 +31,12 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot( + process.out.versions, + file(process.out.mqc_yml[0]).readLines()[0..10], + file(process.out.yml[0]).readLines()[0..7] + ).match() + } ) } } diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap index 4274ed57..5f59a936 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap +++ b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap @@ -1,27 +1,33 @@ { "Should run without failures": { "content": [ - { - "0": [ - "software_versions.yml:md5,1c851188476409cda5752ce971b20b58" - ], - "1": [ - "software_versions_mqc.yml:md5,2570f4ba271ad08357b0d3d32a9cf84d" - ], - "2": [ - "versions.yml:md5,3843ac526e762117eedf8825b40683df" - ], - "mqc_yml": [ - "software_versions_mqc.yml:md5,2570f4ba271ad08357b0d3d32a9cf84d" - ], - "versions": [ - "versions.yml:md5,3843ac526e762117eedf8825b40683df" - ], - "yml": [ - "software_versions.yml:md5,1c851188476409cda5752ce971b20b58" - ] - } + [ + "versions.yml:md5,76d454d92244589d32455833f7c1ba6d" + ], + [ + "data: \"\\n\\n \\n \\n \\n \\n \\n \\n \\n\\", + " \\n\\n\\n \\n \\n\\", + " \\ \\n\\n\\n\\n \\n \\", + " \\ \\n \\n\\n\\n\\n\\", + " \\n\\n \\n \\n\\", + " \\ \\n\\n\\n\\n\\n\\n \\n\\", + " \\ \\n \\n\\n\\n\\n\\", + " \\n\\n \\n \\n\\" + ], + [ + "CUSTOM_DUMPSOFTWAREVERSIONS:", + " python: 3.11.7", + " yaml: 5.4.1", + "TOOL1:", + " tool1: 0.11.9", + "TOOL2:", + " tool2: '1.9'", + "Workflow:" + ] ], - "timestamp": "2023-11-03T14:43:22.157011" + "timestamp": "2024-01-09T23:01:18.710682" } -} +} \ No newline at end of file diff --git a/modules/nf-core/fastqc/tests/main.nf.test b/modules/nf-core/fastqc/tests/main.nf.test index 25b2931b..ad9bc54f 100644 --- a/modules/nf-core/fastqc/tests/main.nf.test +++ b/modules/nf-core/fastqc/tests/main.nf.test @@ -3,95 +3,218 @@ nextflow_process { name "Test Process FASTQC" script "../main.nf" process "FASTQC" + tag "modules" tag "modules_nfcore" tag "fastqc" - test("Single-Read") { + test("sarscov2 single-end [fastq]") { when { - params { - outdir = "$outputDir" - } process { """ input[0] = [ - [ id: 'test', single_end:true ], - [ + [ id: 'test', single_end:true ], + [ + file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) + ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + + // NOTE The report contains the date inside it, which means that the md5sum is stable per day, but not longer than that. So you can't md5sum it. + // looks like this:
Mon 2 Oct 2023
test.gz
+ // https://github.com/nf-core/modules/pull/3903#issuecomment-1743620039 + + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 paired-end [fastq]") { + + when { + process { + """ + input[0] = [ + [id: 'test', single_end: false], // meta map + [ + file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) + ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, + { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, + { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, + { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, + { assert path(process.out.html[0][1][0]).text.contains("") }, + { assert path(process.out.html[0][1][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 interleaved [fastq]") { + + when { + process { + """ + input[0] = [ + [id: 'test', single_end: false], // meta map + file(params.test_data['sarscov2']['illumina']['test_interleaved_fastq_gz'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 paired-end [bam]") { + + when { + process { + """ + input[0] = [ + [id: 'test', single_end: false], // meta map + file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 multiple [fastq]") { + + when { + process { + """ + input[0] = [ + [id: 'test', single_end: false], // meta map + [ + file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['test2_1_fastq_gz'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['test2_2_fastq_gz'], checkIfExists: true) + ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, + { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, + { assert process.out.html[0][1][2] ==~ ".*/test_3_fastqc.html" }, + { assert process.out.html[0][1][3] ==~ ".*/test_4_fastqc.html" }, + { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, + { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, + { assert process.out.zip[0][1][2] ==~ ".*/test_3_fastqc.zip" }, + { assert process.out.zip[0][1][3] ==~ ".*/test_4_fastqc.zip" }, + { assert path(process.out.html[0][1][0]).text.contains("") }, + { assert path(process.out.html[0][1][1]).text.contains("") }, + { assert path(process.out.html[0][1][2]).text.contains("") }, + { assert path(process.out.html[0][1][3]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 custom_prefix") { + + when { + process { + """ + input[0] = [ + [ id:'mysample', single_end:true ], // meta map file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) - ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1] ==~ ".*/mysample_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/mysample_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 single-end [fastq] - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id: 'test', single_end:true ], + [ + file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) + ] ] """ } } then { -// TODO -// // -// // Test with paired-end data -// // -// workflow test_fastqc_paired_end { -// input = [ -// [id: 'test', single_end: false], // meta map -// [ -// file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), -// file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) -// ] -// ] - -// FASTQC ( input ) -// } - -// // -// // Test with interleaved data -// // -// workflow test_fastqc_interleaved { -// input = [ -// [id: 'test', single_end: false], // meta map -// file(params.test_data['sarscov2']['illumina']['test_interleaved_fastq_gz'], checkIfExists: true) -// ] - -// FASTQC ( input ) -// } - -// // -// // Test with bam data -// // -// workflow test_fastqc_bam { -// input = [ -// [id: 'test', single_end: false], // meta map -// file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true) -// ] - -// FASTQC ( input ) -// } - -// // -// // Test with multiple samples -// // -// workflow test_fastqc_multiple { -// input = [ -// [id: 'test', single_end: false], // meta map -// [ -// file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), -// file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true), -// file(params.test_data['sarscov2']['illumina']['test2_1_fastq_gz'], checkIfExists: true), -// file(params.test_data['sarscov2']['illumina']['test2_2_fastq_gz'], checkIfExists: true) -// ] -// ] - -// FASTQC ( input ) -// } - -// // -// // Test with custom prefix -// // -// workflow test_fastqc_custom_prefix { -// input = [ -// [ id:'mysample', single_end:true ], // meta map -// file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) -// ] - -// FASTQC ( input ) -// } + assertAll ( + { assert process.success }, + { assert snapshot(process.out.html.collect { file(it[1]).getName() } + + process.out.zip.collect { file(it[1]).getName() } + + process.out.versions ).match() } + ) + } + } + } diff --git a/modules/nf-core/fastqc/tests/main.nf.test.snap b/modules/nf-core/fastqc/tests/main.nf.test.snap index 636a32ce..5ef5afbd 100644 --- a/modules/nf-core/fastqc/tests/main.nf.test.snap +++ b/modules/nf-core/fastqc/tests/main.nf.test.snap @@ -1,10 +1,20 @@ { + "sarscov2 single-end [fastq] - stub": { + "content": [ + [ + "test.html", + "test.zip", + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ] + ], + "timestamp": "2023-12-29T02:48:05.126117287" + }, "versions": { "content": [ [ "versions.yml:md5,e1cc25ca8af856014824abd842e93978" ] ], - "timestamp": "2023-10-09T23:40:54+0000" + "timestamp": "2023-12-29T02:46:49.507942667" } } \ No newline at end of file diff --git a/nextflow.config b/nextflow.config index c8856a45..67ed2975 100644 --- a/nextflow.config +++ b/nextflow.config @@ -306,6 +306,7 @@ profiles { executor.cpus = 4 executor.memory = 8.GB } + test_data { includeConfig 'conf/test_data.config' } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } test_host_rm { includeConfig 'conf/test_host_rm.config' } From a3c984a8c373d0ff611037997b20a713a23b327b Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Wed, 17 Jan 2024 14:53:10 +0000 Subject: [PATCH 26/49] Added test_data config to CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a7ddff2..c63ab065 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,7 @@ jobs: - name: Run nf-test run: | - nf-test test --tag mag_${{ matrix.tags }} --profile ${{ matrix.tags }},${{ matrix.profile }} --junitxml=test.xml + nf-test test --tag mag_${{ matrix.tags }} --profile ${{ matrix.tags }},${{ matrix.profile }},test_data --junitxml=test.xml - name: Output log on failure if: failure() From 319142cd47dfce185746f4773e524b0d8652334d Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Wed, 17 Jan 2024 14:57:34 +0000 Subject: [PATCH 27/49] Fixed tags for nf-core modules --- .github/workflows/ci.yml | 2 +- nextflow.config | 30 +++++++++++++++--------------- workflows/mag/tests/tags.yml | 22 +++++++++++----------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c63ab065..5143d314 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,7 @@ jobs: - name: Run nf-test run: | - nf-test test --tag mag_${{ matrix.tags }} --profile ${{ matrix.tags }},${{ matrix.profile }},test_data --junitxml=test.xml + nf-test test --tag ${{ matrix.tags }} --profile ${{ matrix.tags }},${{ matrix.profile }},test_data --junitxml=test.xml - name: Output log on failure if: failure() diff --git a/nextflow.config b/nextflow.config index 67ed2975..1053a50d 100644 --- a/nextflow.config +++ b/nextflow.config @@ -306,21 +306,21 @@ profiles { executor.cpus = 4 executor.memory = 8.GB } - test_data { includeConfig 'conf/test_data.config' } - test { includeConfig 'conf/test.config' } - test_full { includeConfig 'conf/test_full.config' } - test_host_rm { includeConfig 'conf/test_host_rm.config' } - test_hybrid { includeConfig 'conf/test_hybrid.config' } - test_hybrid_host_rm { includeConfig 'conf/test_hybrid_host_rm.config' } - test_busco_auto { includeConfig 'conf/test_busco_auto.config' } - test_ancient_dna { includeConfig 'conf/test_ancient_dna.config' } - test_adapterremoval { includeConfig 'conf/test_adapterremoval.config' } - test_binning_entry { includeConfig 'conf/test_binning_entry.config' } - test_binrefinement { includeConfig 'conf/test_binrefinement.config' } - test_no_clipping { includeConfig 'conf/test_no_clipping.config' } - test_bbnorm { includeConfig 'conf/test_bbnorm.config' } - test_nothing { includeConfig 'conf/test_nothing.config' } - test_virus_identification { includeConfig 'conf/test_virus_identification.config' } + test_data { includeConfig 'conf/test_data.config' } + mag_test { includeConfig 'conf/test.config' } + mag_test_full { includeConfig 'conf/test_full.config' } + mag_test_host_rm { includeConfig 'conf/test_host_rm.config' } + mag_test_hybrid { includeConfig 'conf/test_hybrid.config' } + mag_test_hybrid_host_rm { includeConfig 'conf/test_hybrid_host_rm.config' } + mag_test_busco_auto { includeConfig 'conf/test_busco_auto.config' } + mag_test_ancient_dna { includeConfig 'conf/test_ancient_dna.config' } + mag_test_adapterremoval { includeConfig 'conf/test_adapterremoval.config' } + mag_test_binning_entry { includeConfig 'conf/test_binning_entry.config' } + mag_test_binrefinement { includeConfig 'conf/test_binrefinement.config' } + mag_test_no_clipping { includeConfig 'conf/test_no_clipping.config' } + mag_test_bbnorm { includeConfig 'conf/test_bbnorm.config' } + mag_test_nothing { includeConfig 'conf/test_nothing.config' } + mag_test_virus_identification { includeConfig 'conf/test_virus_identification.config' } } // Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile diff --git a/workflows/mag/tests/tags.yml b/workflows/mag/tests/tags.yml index 6d995160..15329330 100644 --- a/workflows/mag/tests/tags.yml +++ b/workflows/mag/tests/tags.yml @@ -1,22 +1,22 @@ -test_adapterremoval: +mag_test_adapterremoval: - ./** -test_ancient_dna: +mag_test_ancient_dna: - ./** -test_bbnorm: +mag_test_bbnorm: - ./** -test_binrefinement: +mag_test_binrefinement: - ./** -test_busco_auto: +mag_test_busco_auto: - ./** -test_host_rm: +mag_test_host_rm: - ./** -test_hybrid_host_rm: +mag_test_hybrid_host_rm: - ./** -test_hybrid: +mag_test_hybrid: - ./** -test_nothing: +mag_test_nothing: - ./** -test_virus_identification: +mag_test_virus_identification: - ./** -test: +mag_test: - ./** From 0126a63654b0ccdaa5006db39666120962015095 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Wed, 17 Jan 2024 10:29:48 -0800 Subject: [PATCH 28/49] Updated snapshots to reflect new nf-core modules --- .github/workflows/ci.yml | 2 +- conf/test.config | 23 +-------- conf/test_adapterremoval.config | 40 --------------- conf/test_ancient_dna.config | 51 ------------------- conf/test_bbnorm.config | 40 --------------- conf/test_binrefinement.config | 50 ------------------ conf/test_busco_auto.config | 45 ---------------- conf/test_host_rm.config | 42 --------------- conf/test_hybrid.config | 42 --------------- conf/test_hybrid_host_rm.config | 42 --------------- conf/test_nothing.config | 41 --------------- conf/test_virus_identification.config | 43 ---------------- nextflow.config | 15 +----- workflows/mag/tests/main.nf.test | 22 ++++++++ workflows/mag/tests/main.nf.test.snap | 18 +++---- .../mag/tests/test_adapterremoval.nf.test | 21 ++++++++ .../tests/test_adapterremoval.nf.test.snap | 14 ++--- workflows/mag/tests/test_ancient_dna.nf.test | 33 ++++++++++++ .../mag/tests/test_ancient_dna.nf.test.snap | 6 +-- workflows/mag/tests/test_bbnorm.nf.test | 22 ++++++++ workflows/mag/tests/test_bbnorm.nf.test.snap | 14 ++--- .../mag/tests/test_binrefinement.nf.test | 31 +++++++++++ .../mag/tests/test_binrefinement.nf.test.snap | 6 +-- workflows/mag/tests/test_busco_auto.nf.test | 27 ++++++++++ .../mag/tests/test_busco_auto.nf.test.snap | 18 +++---- workflows/mag/tests/test_host_rm.nf.test | 24 +++++++++ workflows/mag/tests/test_host_rm.nf.test.snap | 14 ++--- workflows/mag/tests/test_hybrid.nf.test | 24 +++++++++ workflows/mag/tests/test_hybrid.nf.test.snap | 14 ++--- .../mag/tests/test_hybrid_host_rm.nf.test | 24 +++++++++ .../tests/test_hybrid_host_rm.nf.test.snap | 10 ++-- workflows/mag/tests/test_nothing.nf.test | 23 +++++++++ workflows/mag/tests/test_nothing.nf.test.snap | 6 +-- .../tests/test_virus_identification.nf.test | 26 ++++++++++ .../test_virus_identification.nf.test.snap | 44 ++++++++-------- 35 files changed, 363 insertions(+), 554 deletions(-) delete mode 100644 conf/test_adapterremoval.config delete mode 100644 conf/test_ancient_dna.config delete mode 100644 conf/test_bbnorm.config delete mode 100644 conf/test_binrefinement.config delete mode 100644 conf/test_busco_auto.config delete mode 100644 conf/test_host_rm.config delete mode 100644 conf/test_hybrid.config delete mode 100644 conf/test_hybrid_host_rm.config delete mode 100644 conf/test_nothing.config delete mode 100644 conf/test_virus_identification.config diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5143d314..030ff298 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,7 @@ jobs: - name: Run nf-test run: | - nf-test test --tag ${{ matrix.tags }} --profile ${{ matrix.tags }},${{ matrix.profile }},test_data --junitxml=test.xml + nf-test test --tag ${{ matrix.tags }} --profile test,${{ matrix.profile }},test_data --junitxml=test.xml - name: Output log on failure if: failure() diff --git a/conf/test.config b/conf/test.config index ab1857f7..bf071202 100644 --- a/conf/test.config +++ b/conf/test.config @@ -15,26 +15,7 @@ params { config_profile_description = 'Minimal test dataset to check pipeline function' // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' + //max_cpus = 2 + //max_memory = '6.GB' max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' - centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" - kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" - skip_krona = true - megahit_fix_cpu_1 = true - spades_fix_cpus = 1 - bowtie2_fix_cpu_1 = true - metabat2_fix_cpu_1 = true - maxbin2_fix_cpu_1 = true - concoct_fix_cpu_1 = true - binning_map_mode = 'own' - min_length_unbinned_contigs = 1000000 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz" - busco_clean = true - skip_gtdbtk = true - skip_concoct = true } diff --git a/conf/test_adapterremoval.config b/conf/test_adapterremoval.config deleted file mode 100644 index 07b02a6c..00000000 --- a/conf/test_adapterremoval.config +++ /dev/null @@ -1,40 +0,0 @@ -/* -======================================================================================== - Nextflow config file for running minimal tests -======================================================================================== - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run nf-core/mag -profile test, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Test profile for running with AdapterRemoval and domain classification' - config_profile_description = 'Minimal test dataset to check pipeline function with AdapterRemoval data and domain classification.' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.euk.csv' - clip_tool = 'adapterremoval' - keep_phix = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - skip_megahit = true - skip_spades = true - skip_spadeshybrid = true - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true - -} diff --git a/conf/test_ancient_dna.config b/conf/test_ancient_dna.config deleted file mode 100644 index dcdda8fa..00000000 --- a/conf/test_ancient_dna.config +++ /dev/null @@ -1,51 +0,0 @@ -/* -======================================================================================== - Nextflow config file for running minimal tests -======================================================================================== - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run nf-core/mag -profile test_ancient_dna, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Ancient DNA test profile ' - config_profile_description = 'Minimal test dataset to check pipeline function for ancient DNA step' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - skip_clipping = true - keep_phix = true - kraken2_db = null - centrifuge_db = null - skip_krona = true - megahit_fix_cpu_1 = true - spades_fix_cpus = 1 - skip_spadeshybrid = true - ancient_dna = true - skip_quast = true - skip_prodigal = true - bowtie2_fix_cpu_1 = true - binning_map_mode = 'own' - metabat2_fix_cpu_1 = true - maxbin2_fix_cpu_1 = true - concoct_fix_cpu_1 = true - bcftools_view_high_variant_quality = 0 - bcftools_view_medium_variant_quality = 0 - bcftools_view_minimal_allelesupport = 3 - refine_bins_dastool = true - refine_bins_dastool_threshold = 0 - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true -} diff --git a/conf/test_bbnorm.config b/conf/test_bbnorm.config deleted file mode 100644 index c3fb8518..00000000 --- a/conf/test_bbnorm.config +++ /dev/null @@ -1,40 +0,0 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Nextflow config file for running minimal tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run nf-core/mag -profile test, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Test profile' - config_profile_description = 'Minimal test dataset to check pipeline function' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - bbnorm = true - keep_phix = true - skip_adapter_trimming = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - skip_megahit = true - skip_spades = true - skip_spadeshybrid = true - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true -} diff --git a/conf/test_binrefinement.config b/conf/test_binrefinement.config deleted file mode 100644 index 969dfc2c..00000000 --- a/conf/test_binrefinement.config +++ /dev/null @@ -1,50 +0,0 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Nextflow config file for running minimal tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run nf-core/mag -profile test_binrefinement, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Test profile' - config_profile_description = 'Minimal test dataset to check pipeline function' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - assembly_input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/assembly_samplesheet.csv' - bbnorm = true - keep_phix = true - skip_adapter_trimming = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - skip_quast = true - skip_prodigal = true - bowtie2_fix_cpu_1 = true - metabat2_fix_cpu_1 = true - maxbin2_fix_cpu_1 = true - concoct_fix_cpu_1 = true - binning_map_mode = 'own' - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - refine_bins_dastool = true - refine_bins_dastool_threshold = 0 - // TODO not using 'both' until #489 merged - postbinning_input = 'refined_bins_only' - skip_busco = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true - -} diff --git a/conf/test_busco_auto.config b/conf/test_busco_auto.config deleted file mode 100644 index d99ea24b..00000000 --- a/conf/test_busco_auto.config +++ /dev/null @@ -1,45 +0,0 @@ -/* -======================================================================================== - Nextflow config file for running minimal tests -======================================================================================== - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run nf-core/mag -profile test_busco_auto, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Test profile' - config_profile_description = 'Minimal test dataset to check pipeline function' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - keep_phix = true - skip_adapter_trimming = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - megahit_fix_cpu_1 = true - skip_spades = true - skip_spadeshybrid = true - skip_quast = true - skip_prodigal = true - bowtie2_fix_cpu_1 = true - metabat2_fix_cpu_1 = true - maxbin2_fix_cpu_1 = true - concoct_fix_cpu_1 = true - binning_map_mode = 'own' - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - skip_concoct = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true -} diff --git a/conf/test_host_rm.config b/conf/test_host_rm.config deleted file mode 100644 index 906c68a7..00000000 --- a/conf/test_host_rm.config +++ /dev/null @@ -1,42 +0,0 @@ -/* -======================================================================================== - Nextflow config file for running minimal tests -======================================================================================== - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run nf-core/mag -profile test_host_rm, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Test profile' - config_profile_description = 'Minimal test dataset to check pipeline function' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.host_rm.csv' - keep_phix = true - host_fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/host_reference/genome.hg38.chr21_10000bp_region.fa" - bowtie2_fix_cpu_1 = true - skip_clipping = true - skip_adapter_trimming = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - skip_megahit = true - skip_spades = true - skip_spadeshybrid = true - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true -} diff --git a/conf/test_hybrid.config b/conf/test_hybrid.config deleted file mode 100644 index d0fb0bcc..00000000 --- a/conf/test_hybrid.config +++ /dev/null @@ -1,42 +0,0 @@ -/* -======================================================================================== - Nextflow config file for running minimal tests -======================================================================================== - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run nf-core/mag -profile test_hybrid, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Test profile' - config_profile_description = 'Minimal test dataset to check pipeline function' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.hybrid.csv' - keep_phix = true - skip_clipping = false - skip_adapter_trimming = false - centrifuge_db = null - kraken2_db = null - skip_krona = true - coassemble_group = true - megahit_fix_cpu_1 = true - skip_spades = true - skip_spadeshybrid = false - spadeshybrid_fix_cpus = 2 - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true -} diff --git a/conf/test_hybrid_host_rm.config b/conf/test_hybrid_host_rm.config deleted file mode 100644 index 05b2f8b4..00000000 --- a/conf/test_hybrid_host_rm.config +++ /dev/null @@ -1,42 +0,0 @@ -/* -======================================================================================== - Nextflow config file for running minimal tests -======================================================================================== - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run nf-core/mag -profile test_hybrid_host_rm, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Test profile' - config_profile_description = 'Minimal test dataset to check pipeline function' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.hybrid_host_rm.csv' - host_fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/host_reference/genome.hg38.chr21_10000bp_region.fa" - bowtie2_fix_cpu_1 = true - keep_phix = true - skip_clipping = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - skip_megahit = true - skip_spades = true - skip_spadeshybrid = false - spadeshybrid_fix_cpus = 2 - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true -} diff --git a/conf/test_nothing.config b/conf/test_nothing.config deleted file mode 100644 index 8904d2f6..00000000 --- a/conf/test_nothing.config +++ /dev/null @@ -1,41 +0,0 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Nextflow config file for running minimal tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Runs input data but skipping all possible steps to allow for a fast testing - profile for input checks etc. - - Use as follows: - nextflow run nf-core/mag -profile test_nothing, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Test profile' - config_profile_description = 'Minimal test dataset to check pipeline function' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' - keep_phix = true - skip_clipping = true - skip_adapter_trimming = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - skip_megahit = true - skip_spades = true - skip_spadeshybrid = true - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true -} diff --git a/conf/test_virus_identification.config b/conf/test_virus_identification.config deleted file mode 100644 index 940f7e1d..00000000 --- a/conf/test_virus_identification.config +++ /dev/null @@ -1,43 +0,0 @@ -/* -======================================================================================== - Nextflow config file for running minimal tests -======================================================================================== - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run nf-core/mag -profile test_virus_identification, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Test profile for running virus_identification' - config_profile_description = 'Minimal test dataset to check pipeline function virus identification' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' - keep_phix = true - skip_clipping = true - skip_adapter_trimming = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - coassemble_group = true - megahit_fix_cpu_1 = true - skip_spades = true - skip_spadeshybrid = true - skip_quast = true - run_virus_identification = true - genomad_splits = 7 - skip_prodigal = true - skip_binning = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true -} diff --git a/nextflow.config b/nextflow.config index 1053a50d..322d9792 100644 --- a/nextflow.config +++ b/nextflow.config @@ -307,20 +307,7 @@ profiles { executor.memory = 8.GB } test_data { includeConfig 'conf/test_data.config' } - mag_test { includeConfig 'conf/test.config' } - mag_test_full { includeConfig 'conf/test_full.config' } - mag_test_host_rm { includeConfig 'conf/test_host_rm.config' } - mag_test_hybrid { includeConfig 'conf/test_hybrid.config' } - mag_test_hybrid_host_rm { includeConfig 'conf/test_hybrid_host_rm.config' } - mag_test_busco_auto { includeConfig 'conf/test_busco_auto.config' } - mag_test_ancient_dna { includeConfig 'conf/test_ancient_dna.config' } - mag_test_adapterremoval { includeConfig 'conf/test_adapterremoval.config' } - mag_test_binning_entry { includeConfig 'conf/test_binning_entry.config' } - mag_test_binrefinement { includeConfig 'conf/test_binrefinement.config' } - mag_test_no_clipping { includeConfig 'conf/test_no_clipping.config' } - mag_test_bbnorm { includeConfig 'conf/test_bbnorm.config' } - mag_test_nothing { includeConfig 'conf/test_nothing.config' } - mag_test_virus_identification { includeConfig 'conf/test_virus_identification.config' } + test { includeConfig 'conf/test.config' } } // Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile diff --git a/workflows/mag/tests/main.nf.test b/workflows/mag/tests/main.nf.test index b90b09b2..b960e7cf 100644 --- a/workflows/mag/tests/main.nf.test +++ b/workflows/mag/tests/main.nf.test @@ -9,6 +9,28 @@ nextflow_workflow { test("Default paramters") { + when { + params { + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' + centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" + kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" + skip_krona = true + megahit_fix_cpu_1 = true + spades_fix_cpus = 1 + bowtie2_fix_cpu_1 = true + metabat2_fix_cpu_1 = true + maxbin2_fix_cpu_1 = true + concoct_fix_cpu_1 = true + binning_map_mode = 'own' + min_length_unbinned_contigs = 1000000 + max_unbinned_contigs = 2 + busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz" + busco_clean = true + skip_gtdbtk = true + skip_concoct = true + } + } + then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/main.nf.test.snap b/workflows/mag/tests/main.nf.test.snap index cb3ddd39..7b3f8e57 100644 --- a/workflows/mag/tests/main.nf.test.snap +++ b/workflows/mag/tests/main.nf.test.snap @@ -87,9 +87,7 @@ "versions.yml:md5,7067399ef1687d4d1ba0aab93dc67754", "versions.yml:md5,8187d1683f97b64a582620caba2b175a", "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", @@ -99,8 +97,10 @@ "versions.yml:md5,bb7110425028f7e7abcbf17152405079", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", "versions.yml:md5,ea3029fd33017b2f39de14bce337c246" ], "2": [ @@ -520,9 +520,7 @@ "versions.yml:md5,7067399ef1687d4d1ba0aab93dc67754", "versions.yml:md5,8187d1683f97b64a582620caba2b175a", "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", @@ -532,12 +530,14 @@ "versions.yml:md5,bb7110425028f7e7abcbf17152405079", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", "versions.yml:md5,ea3029fd33017b2f39de14bce337c246" ] } ], - "timestamp": "2024-01-08T08:51:13.153323648" + "timestamp": "2024-01-17T10:27:28.027473013" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_adapterremoval.nf.test b/workflows/mag/tests/test_adapterremoval.nf.test index 9d05d884..781b0e3c 100644 --- a/workflows/mag/tests/test_adapterremoval.nf.test +++ b/workflows/mag/tests/test_adapterremoval.nf.test @@ -9,6 +9,27 @@ nextflow_workflow { test("Parameters: clip_tool = 'adapterremoval'") { + when { + params { + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.euk.csv' + clip_tool = 'adapterremoval' + keep_phix = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_megahit = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true + } + } + then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_adapterremoval.nf.test.snap b/workflows/mag/tests/test_adapterremoval.nf.test.snap index 7e77be93..a52e2142 100644 --- a/workflows/mag/tests/test_adapterremoval.nf.test.snap +++ b/workflows/mag/tests/test_adapterremoval.nf.test.snap @@ -30,9 +30,9 @@ ], "10": [ - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", "versions.yml:md5,eca9f4c6e135848b1247916040e52b99" ], "2": [ @@ -111,13 +111,13 @@ ] ], "versions": [ - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", "versions.yml:md5,eca9f4c6e135848b1247916040e52b99" ] } ], - "timestamp": "2024-01-02T11:42:16.890174222" + "timestamp": "2024-01-17T09:17:39.984696852" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_ancient_dna.nf.test b/workflows/mag/tests/test_ancient_dna.nf.test index b0023c14..55ca203e 100644 --- a/workflows/mag/tests/test_ancient_dna.nf.test +++ b/workflows/mag/tests/test_ancient_dna.nf.test @@ -9,6 +9,39 @@ nextflow_workflow { test("Parameters: ancient_dna = true") { + when { + params { + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + skip_clipping = true + keep_phix = true + kraken2_db = null + centrifuge_db = null + skip_krona = true + megahit_fix_cpu_1 = true + spades_fix_cpus = 1 + skip_spadeshybrid = true + ancient_dna = true + skip_quast = true + skip_prodigal = true + bowtie2_fix_cpu_1 = true + binning_map_mode = 'own' + metabat2_fix_cpu_1 = true + maxbin2_fix_cpu_1 = true + concoct_fix_cpu_1 = true + bcftools_view_high_variant_quality = 0 + bcftools_view_medium_variant_quality = 0 + bcftools_view_minimal_allelesupport = 3 + refine_bins_dastool = true + refine_bins_dastool_threshold = 0 + min_length_unbinned_contigs = 1 + max_unbinned_contigs = 2 + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true + } + } + then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_ancient_dna.nf.test.snap b/workflows/mag/tests/test_ancient_dna.nf.test.snap index c6c3e6b3..0698a573 100644 --- a/workflows/mag/tests/test_ancient_dna.nf.test.snap +++ b/workflows/mag/tests/test_ancient_dna.nf.test.snap @@ -88,6 +88,7 @@ "versions.yml:md5,7ccffaba4e75be98b8faedb54ccea16f", "versions.yml:md5,8187d1683f97b64a582620caba2b175a", "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", "versions.yml:md5,aee39cb21305f6d3fe88bf9863529706", "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", @@ -104,7 +105,6 @@ "versions.yml:md5,dc29f891cb297131cf940ba5580a6832", "versions.yml:md5,df654dd4fdb4039c2d8cf0b6b49f4b55", "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", "versions.yml:md5,e9f8fe610b5abe68b28d77efc951e9e6" ], "2": [ @@ -373,6 +373,7 @@ "versions.yml:md5,7ccffaba4e75be98b8faedb54ccea16f", "versions.yml:md5,8187d1683f97b64a582620caba2b175a", "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", "versions.yml:md5,aee39cb21305f6d3fe88bf9863529706", "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", @@ -389,11 +390,10 @@ "versions.yml:md5,dc29f891cb297131cf940ba5580a6832", "versions.yml:md5,df654dd4fdb4039c2d8cf0b6b49f4b55", "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", "versions.yml:md5,e9f8fe610b5abe68b28d77efc951e9e6" ] } ], - "timestamp": "2024-01-02T12:11:37.803431535" + "timestamp": "2024-01-17T08:28:03.926093976" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_bbnorm.nf.test b/workflows/mag/tests/test_bbnorm.nf.test index 5a7dd6ba..2b80cd40 100644 --- a/workflows/mag/tests/test_bbnorm.nf.test +++ b/workflows/mag/tests/test_bbnorm.nf.test @@ -9,6 +9,28 @@ nextflow_workflow { test("Parameters: bbnorm = true") { + when { + params { + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + bbnorm = true + keep_phix = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_megahit = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true + } + } + then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_bbnorm.nf.test.snap b/workflows/mag/tests/test_bbnorm.nf.test.snap index 89a37918..f7b0a04f 100644 --- a/workflows/mag/tests/test_bbnorm.nf.test.snap +++ b/workflows/mag/tests/test_bbnorm.nf.test.snap @@ -33,9 +33,9 @@ "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", "versions.yml:md5,67cbab915f2801633ea61219086a7735", "versions.yml:md5,67cbab915f2801633ea61219086a7735", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0" ], "2": [ @@ -116,12 +116,12 @@ "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", "versions.yml:md5,67cbab915f2801633ea61219086a7735", "versions.yml:md5,67cbab915f2801633ea61219086a7735", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0" ] } ], - "timestamp": "2024-01-02T13:06:14.895384486" + "timestamp": "2024-01-17T09:19:27.939943074" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_binrefinement.nf.test b/workflows/mag/tests/test_binrefinement.nf.test index 277b06e5..0de723bd 100644 --- a/workflows/mag/tests/test_binrefinement.nf.test +++ b/workflows/mag/tests/test_binrefinement.nf.test @@ -9,6 +9,37 @@ nextflow_workflow { test("Parameters: refine_bins_dastool = true") { + when { + params { + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + assembly_input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/assembly_samplesheet.csv' + bbnorm = true + keep_phix = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_quast = true + skip_prodigal = true + bowtie2_fix_cpu_1 = true + metabat2_fix_cpu_1 = true + maxbin2_fix_cpu_1 = true + concoct_fix_cpu_1 = true + binning_map_mode = 'own' + min_length_unbinned_contigs = 1 + max_unbinned_contigs = 2 + refine_bins_dastool = true + refine_bins_dastool_threshold = 0 + // TODO not using 'both' until #489 merged + postbinning_input = 'refined_bins_only' + skip_busco = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true + } + } + then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_binrefinement.nf.test.snap b/workflows/mag/tests/test_binrefinement.nf.test.snap index c4cc01c2..1ebe267b 100644 --- a/workflows/mag/tests/test_binrefinement.nf.test.snap +++ b/workflows/mag/tests/test_binrefinement.nf.test.snap @@ -72,6 +72,7 @@ "versions.yml:md5,7ccffaba4e75be98b8faedb54ccea16f", "versions.yml:md5,8187d1683f97b64a582620caba2b175a", "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", "versions.yml:md5,aee39cb21305f6d3fe88bf9863529706", "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", @@ -86,7 +87,6 @@ "versions.yml:md5,d8d2321bec0c70de2f9750f258fc890d", "versions.yml:md5,dc29f891cb297131cf940ba5580a6832", "versions.yml:md5,df654dd4fdb4039c2d8cf0b6b49f4b55", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", "versions.yml:md5,e9f8fe610b5abe68b28d77efc951e9e6", "versions.yml:md5,ede79d94ca30c948c1707584760b230c", "versions.yml:md5,ede79d94ca30c948c1707584760b230c", @@ -307,6 +307,7 @@ "versions.yml:md5,7ccffaba4e75be98b8faedb54ccea16f", "versions.yml:md5,8187d1683f97b64a582620caba2b175a", "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", "versions.yml:md5,aee39cb21305f6d3fe88bf9863529706", "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", @@ -321,7 +322,6 @@ "versions.yml:md5,d8d2321bec0c70de2f9750f258fc890d", "versions.yml:md5,dc29f891cb297131cf940ba5580a6832", "versions.yml:md5,df654dd4fdb4039c2d8cf0b6b49f4b55", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a", "versions.yml:md5,e9f8fe610b5abe68b28d77efc951e9e6", "versions.yml:md5,ede79d94ca30c948c1707584760b230c", "versions.yml:md5,ede79d94ca30c948c1707584760b230c", @@ -330,6 +330,6 @@ ] } ], - "timestamp": "2024-01-02T13:35:37.404532095" + "timestamp": "2024-01-17T09:39:03.032859012" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_busco_auto.nf.test b/workflows/mag/tests/test_busco_auto.nf.test index 5fb41543..58293e64 100644 --- a/workflows/mag/tests/test_busco_auto.nf.test +++ b/workflows/mag/tests/test_busco_auto.nf.test @@ -9,6 +9,33 @@ nextflow_workflow { test("Parameters: skip_busco = false") { + when { + params { + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + keep_phix = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + megahit_fix_cpu_1 = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + bowtie2_fix_cpu_1 = true + metabat2_fix_cpu_1 = true + maxbin2_fix_cpu_1 = true + concoct_fix_cpu_1 = true + binning_map_mode = 'own' + min_length_unbinned_contigs = 1 + max_unbinned_contigs = 2 + skip_concoct = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true + } + } + then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_busco_auto.nf.test.snap b/workflows/mag/tests/test_busco_auto.nf.test.snap index f7a26da6..b10490c2 100644 --- a/workflows/mag/tests/test_busco_auto.nf.test.snap +++ b/workflows/mag/tests/test_busco_auto.nf.test.snap @@ -57,16 +57,16 @@ "versions.yml:md5,7017c8959eb098440346ca9a2310c460", "versions.yml:md5,8187d1683f97b64a582620caba2b175a", "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", "versions.yml:md5,cc4a4972327dec07faf1f54ead653715", - "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0" ], "2": [ @@ -171,19 +171,19 @@ "versions.yml:md5,7017c8959eb098440346ca9a2310c460", "versions.yml:md5,8187d1683f97b64a582620caba2b175a", "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", "versions.yml:md5,bb7110425028f7e7abcbf17152405079", "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", "versions.yml:md5,cc4a4972327dec07faf1f54ead653715", - "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0" ] } ], - "timestamp": "2024-01-02T15:06:33.650718766" + "timestamp": "2024-01-17T09:53:53.262636096" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_host_rm.nf.test b/workflows/mag/tests/test_host_rm.nf.test index 83ff9862..478a490d 100644 --- a/workflows/mag/tests/test_host_rm.nf.test +++ b/workflows/mag/tests/test_host_rm.nf.test @@ -9,6 +9,30 @@ nextflow_workflow { test("Parameters: host_fasta != null") { + when { + params { + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.host_rm.csv' + keep_phix = true + host_fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/host_reference/genome.hg38.chr21_10000bp_region.fa" + bowtie2_fix_cpu_1 = true + skip_clipping = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_megahit = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true + } + } + then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_host_rm.nf.test.snap b/workflows/mag/tests/test_host_rm.nf.test.snap index 6103b621..3e64f68d 100644 --- a/workflows/mag/tests/test_host_rm.nf.test.snap +++ b/workflows/mag/tests/test_host_rm.nf.test.snap @@ -31,9 +31,9 @@ ], "10": [ "versions.yml:md5,59050473c9f749b269ad7079fd4693df", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0" ], "2": [ @@ -112,12 +112,12 @@ ], "versions": [ "versions.yml:md5,59050473c9f749b269ad7079fd4693df", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0" ] } ], - "timestamp": "2024-01-02T15:33:59.196829811" + "timestamp": "2024-01-17T09:55:27.641552946" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_hybrid.nf.test b/workflows/mag/tests/test_hybrid.nf.test index 424bd4af..66c89d7c 100644 --- a/workflows/mag/tests/test_hybrid.nf.test +++ b/workflows/mag/tests/test_hybrid.nf.test @@ -9,6 +9,30 @@ nextflow_workflow { test("Parameters: long reads included") { + when { + params { + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.hybrid.csv' + keep_phix = true + skip_clipping = false + skip_adapter_trimming = false + centrifuge_db = null + kraken2_db = null + skip_krona = true + coassemble_group = true + megahit_fix_cpu_1 = true + skip_spades = true + skip_spadeshybrid = false + spadeshybrid_fix_cpus = 2 + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true + } + } + then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_hybrid.nf.test.snap b/workflows/mag/tests/test_hybrid.nf.test.snap index 6ec11bbd..643ef8e9 100644 --- a/workflows/mag/tests/test_hybrid.nf.test.snap +++ b/workflows/mag/tests/test_hybrid.nf.test.snap @@ -39,12 +39,12 @@ "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", "versions.yml:md5,1bd6bbec156534880e04e0320dbce063", "versions.yml:md5,5de8983775d3ef64cdbec6f382852619", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", "versions.yml:md5,b94aaba1559ac9deaf7339181c398405", "versions.yml:md5,cc143ba1b806a335a9963a434c887eba", "versions.yml:md5,d13508b80e270ac6f1543725b431a682", - "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0" ], "2": [ @@ -131,15 +131,15 @@ "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", "versions.yml:md5,1bd6bbec156534880e04e0320dbce063", "versions.yml:md5,5de8983775d3ef64cdbec6f382852619", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", "versions.yml:md5,b94aaba1559ac9deaf7339181c398405", "versions.yml:md5,cc143ba1b806a335a9963a434c887eba", "versions.yml:md5,d13508b80e270ac6f1543725b431a682", - "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0" ] } ], - "timestamp": "2024-01-02T16:01:13.719391405" + "timestamp": "2024-01-17T10:12:08.554896181" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_hybrid_host_rm.nf.test b/workflows/mag/tests/test_hybrid_host_rm.nf.test index ffa38654..92da398c 100644 --- a/workflows/mag/tests/test_hybrid_host_rm.nf.test +++ b/workflows/mag/tests/test_hybrid_host_rm.nf.test @@ -9,6 +9,30 @@ nextflow_workflow { test("Parameters: long reads & host removal included") { + when { + params { + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.hybrid_host_rm.csv' + host_fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/host_reference/genome.hg38.chr21_10000bp_region.fa" + bowtie2_fix_cpu_1 = true + keep_phix = true + skip_clipping = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_megahit = true + skip_spades = true + skip_spadeshybrid = false + spadeshybrid_fix_cpus = 2 + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true + } + } + then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_hybrid_host_rm.nf.test.snap b/workflows/mag/tests/test_hybrid_host_rm.nf.test.snap index 3cb34ca8..a4f438fb 100644 --- a/workflows/mag/tests/test_hybrid_host_rm.nf.test.snap +++ b/workflows/mag/tests/test_hybrid_host_rm.nf.test.snap @@ -30,11 +30,11 @@ "versions.yml:md5,1bd6bbec156534880e04e0320dbce063", "versions.yml:md5,59050473c9f749b269ad7079fd4693df", "versions.yml:md5,5de8983775d3ef64cdbec6f382852619", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", "versions.yml:md5,b94aaba1559ac9deaf7339181c398405", "versions.yml:md5,cc143ba1b806a335a9963a434c887eba", "versions.yml:md5,d13508b80e270ac6f1543725b431a682", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0" ], "2": [ @@ -112,14 +112,14 @@ "versions.yml:md5,1bd6bbec156534880e04e0320dbce063", "versions.yml:md5,59050473c9f749b269ad7079fd4693df", "versions.yml:md5,5de8983775d3ef64cdbec6f382852619", - "versions.yml:md5,9e5858bca5fb800545dc5c11f8503cc4", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", "versions.yml:md5,b94aaba1559ac9deaf7339181c398405", "versions.yml:md5,cc143ba1b806a335a9963a434c887eba", "versions.yml:md5,d13508b80e270ac6f1543725b431a682", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0" ] } ], - "timestamp": "2024-01-02T15:43:00.831382408" + "timestamp": "2024-01-17T10:03:17.006107632" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_nothing.nf.test b/workflows/mag/tests/test_nothing.nf.test index 210cf184..cbeafb06 100644 --- a/workflows/mag/tests/test_nothing.nf.test +++ b/workflows/mag/tests/test_nothing.nf.test @@ -8,6 +8,29 @@ nextflow_workflow { tag "mag_test_nothing" test("Parameters: test_nothing") { + + when { + params { + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' + keep_phix = true + skip_clipping = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_megahit = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true + } + } + then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_nothing.nf.test.snap b/workflows/mag/tests/test_nothing.nf.test.snap index ec39fd6a..81e6c78d 100644 --- a/workflows/mag/tests/test_nothing.nf.test.snap +++ b/workflows/mag/tests/test_nothing.nf.test.snap @@ -31,7 +31,7 @@ ], "10": [ "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400" ], "2": [ @@ -110,10 +110,10 @@ ], "versions": [ "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400" ] } ], - "timestamp": "2024-01-02T16:14:07.274635084" + "timestamp": "2024-01-17T08:28:52.988011944" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_virus_identification.nf.test b/workflows/mag/tests/test_virus_identification.nf.test index 4a49ac43..54b72b0c 100644 --- a/workflows/mag/tests/test_virus_identification.nf.test +++ b/workflows/mag/tests/test_virus_identification.nf.test @@ -8,6 +8,32 @@ nextflow_workflow { tag "mag_test_virus_identification" test("Parameters: run_virus_identification = true") { + + when { + params { + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' + keep_phix = true + skip_clipping = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + coassemble_group = true + megahit_fix_cpu_1 = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + run_virus_identification = true + genomad_splits = 7 + skip_prodigal = true + skip_binning = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true + } + } + then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_virus_identification.nf.test.snap b/workflows/mag/tests/test_virus_identification.nf.test.snap index f9cda1bc..2901435c 100644 --- a/workflows/mag/tests/test_virus_identification.nf.test.snap +++ b/workflows/mag/tests/test_virus_identification.nf.test.snap @@ -39,11 +39,11 @@ ], "10": [ "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", - "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0" ], "2": [ - + ], "3": [ [ @@ -53,26 +53,26 @@ "single_end": false, "assembler": "MEGAHIT" }, - "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,95db91fff85f23a19e64ef7ea0f8d371" + "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,b08861c8a938b3eaf8c1d8e533935042" ] ], "4": [ - + ], "5": [ - + ], "6": [ - + ], "7": [ - + ], "8": [ - + ], "9": [ - + ], "assemblies": [ [ @@ -86,10 +86,10 @@ ] ], "bin_summary": [ - + ], "cat": [ - + ], "genomad": [ [ @@ -99,23 +99,23 @@ "single_end": false, "assembler": "MEGAHIT" }, - "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,95db91fff85f23a19e64ef7ea0f8d371" + "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,b08861c8a938b3eaf8c1d8e533935042" ] ], "metaeuk": [ - + ], "prodigal": [ - + ], "prokka": [ - + ], "refined_bins": [ - + ], "refined_unbins": [ - + ], "short_reads": [ [ @@ -143,11 +143,11 @@ ], "versions": [ "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", - "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", - "versions.yml:md5,e9b46b38f7df41da088ff41127d3752a" + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0" ] } ], - "timestamp": "2024-01-04T16:01:40.636819119" + "timestamp": "2024-01-17T09:14:33.005055708" } -} +} \ No newline at end of file From c5a0b869431da01d06aeb103c735e2a78801a9fb Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Wed, 17 Jan 2024 10:36:48 -0800 Subject: [PATCH 29/49] Fixed CPU./mem requests --- conf/test.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/test.config b/conf/test.config index bf071202..c063074c 100644 --- a/conf/test.config +++ b/conf/test.config @@ -15,7 +15,7 @@ params { config_profile_description = 'Minimal test dataset to check pipeline function' // Limit resources so that this can run on GitHub Actions - //max_cpus = 2 - //max_memory = '6.GB' + max_cpus = 2 + max_memory = '6.GB' max_time = '6.h' } From bdaa7e64348b549a413023e3cd15d5968e43dbde Mon Sep 17 00:00:00 2001 From: Carson J Miller <68351153+CarsonJM@users.noreply.github.com> Date: Wed, 17 Jan 2024 21:11:49 +0000 Subject: [PATCH 30/49] Update genomad hash --- .../test_virus_identification.nf.test.snap | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/workflows/mag/tests/test_virus_identification.nf.test.snap b/workflows/mag/tests/test_virus_identification.nf.test.snap index 2901435c..6e68b049 100644 --- a/workflows/mag/tests/test_virus_identification.nf.test.snap +++ b/workflows/mag/tests/test_virus_identification.nf.test.snap @@ -43,7 +43,7 @@ "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0" ], "2": [ - + ], "3": [ [ @@ -53,26 +53,26 @@ "single_end": false, "assembler": "MEGAHIT" }, - "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,b08861c8a938b3eaf8c1d8e533935042" + "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,95db91fff85f23a19e64ef7ea0f8d371" ] ], "4": [ - + ], "5": [ - + ], "6": [ - + ], "7": [ - + ], "8": [ - + ], "9": [ - + ], "assemblies": [ [ @@ -86,10 +86,10 @@ ] ], "bin_summary": [ - + ], "cat": [ - + ], "genomad": [ [ @@ -99,23 +99,23 @@ "single_end": false, "assembler": "MEGAHIT" }, - "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,b08861c8a938b3eaf8c1d8e533935042" + "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,95db91fff85f23a19e64ef7ea0f8d371" ] ], "metaeuk": [ - + ], "prodigal": [ - + ], "prokka": [ - + ], "refined_bins": [ - + ], "refined_unbins": [ - + ], "short_reads": [ [ @@ -150,4 +150,4 @@ ], "timestamp": "2024-01-17T09:14:33.005055708" } -} \ No newline at end of file +} From fafb1b20bc07d00101f608c9efabd5fdcbf4b9ea Mon Sep 17 00:00:00 2001 From: Carson J Miller <68351153+CarsonJM@users.noreply.github.com> Date: Wed, 17 Jan 2024 21:21:08 +0000 Subject: [PATCH 31/49] Removed unecessary reproducibility options and updated schema --- conf/modules.config | 3 +-- nextflow.config | 11 ----------- nextflow_schema.json | 16 ++++++++++++---- workflows/mag/tests/main.nf.test | 2 -- workflows/mag/tests/test_ancient_dna.nf.test | 2 -- workflows/mag/tests/test_binrefinement.nf.test | 2 -- workflows/mag/tests/test_busco_auto.nf.test | 2 -- 7 files changed, 13 insertions(+), 25 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 6bd8537f..faa46def 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -116,8 +116,7 @@ process { withName: BOWTIE2_HOST_REMOVAL_ALIGN { ext.args = [ - params.host_removal_verysensitive ? "--very-sensitive" : "--sensitive", - "--seed ${params.bowtie2_seed}" + params.host_removal_verysensitive ? "--very-sensitive" : "--sensitive" ].join(' ').trim() ext.args2 = params.host_removal_save_ids ? "--host_removal_save_ids" : '' ext.prefix = { "${meta.id}_run${meta.run}_host_removed" } diff --git a/nextflow.config b/nextflow.config index 322d9792..4ee05d36 100644 --- a/nextflow.config +++ b/nextflow.config @@ -139,11 +139,8 @@ params { spades_fix_cpus = -1 spadeshybrid_fix_cpus = -1 metabat_rng_seed = 1 - bowtie2_seed = 1 bowtie2_fix_cpu_1 = false - metabat2_fix_cpu_1 = false maxbin2_fix_cpu_1 = false - concoct_fix_cpu_1 = false // Annotation options skip_metaeuk = false @@ -429,15 +426,7 @@ def check_spadeshybrid_cpus (x, attempt ) { if (params.spadeshybrid_fix_cpus != -1) return check_max (params.spadeshybrid_fix_cpus, 'cpus' ) else return check_max (x * attempt, 'cpus' ) } -def check_metabat2_cpus (x, attempt ) { - if (params.metabat2_fix_cpu_1) return 1 - else return check_max (x * attempt, 'cpus' ) -} def check_maxbin2_cpus (x, attempt ) { if (params.maxbin2_fix_cpu_1) return 1 else return check_max (x * attempt, 'cpus' ) } -def check_concoct_cpus (x, attempt ) { - if (params.concoct_fix_cpu_1) return 1 - else return check_max (x * attempt, 'cpus' ) -} diff --git a/nextflow_schema.json b/nextflow_schema.json index 99d30c06..6fe08aab 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -278,6 +278,9 @@ "description": "Use these parameters to also enable reproducible results from the individual assembly and binning tools .", "default": "", "properties": { + "bowtie2_fix_cpu_1": { + "type": "boolean" + }, "megahit_fix_cpu_1": { "type": "boolean", "description": "Fix number of CPUs for MEGAHIT to 1. Not increased with retries.", @@ -295,6 +298,11 @@ "description": "Fix number of CPUs used by SPAdes hybrid. Not increased with retries.", "help_text": "SPAdes is designed to be deterministic for a given number of threads. To generate reproducible results fix the number of CPUs using this parameter.\n\nWhen using this parameter do not change the number of CPUs for the `spadeshybrid` process with a custom config file. This would result in an error.\n\nDefault: -1 (the number of CPUs is specified in the `base.config` or in a custom config file, and increased with each retry)." }, + "maxbin2_fix_cpu_1": { + "type": "number", + "description": "Fix number of CPUS used by MaxBin2. Not increased with retries", + "default": -1 + }, "metabat_rng_seed": { "type": "integer", "default": 1, @@ -530,7 +538,7 @@ }, "gtdbtk_min_completeness": { "type": "number", - "default": 50.0, + "default": 50, "description": "Min. bin completeness (in %) required to apply GTDB-tk classification.", "help_text": "Completeness assessed with BUSCO analysis (100% - %Missing). Must be greater than 0 (min. 0.01) to avoid GTDB-tk errors. If too low, GTDB-tk classification results can be impaired due to not enough marker genes!", "minimum": 0.01, @@ -538,7 +546,7 @@ }, "gtdbtk_max_contamination": { "type": "number", - "default": 10.0, + "default": 10, "description": "Max. bin contamination (in %) allowed to apply GTDB-tk classification.", "help_text": "Contamination approximated based on BUSCO analysis (%Complete and duplicated). If too high, GTDB-tk classification results can be impaired due to contamination!", "minimum": 0, @@ -546,7 +554,7 @@ }, "gtdbtk_min_perc_aa": { "type": "number", - "default": 10.0, + "default": 10, "description": "Min. fraction of AA (in %) in the MSA for bins to be kept.", "minimum": 0, "maximum": 100 @@ -560,7 +568,7 @@ }, "gtdbtk_pplacer_cpus": { "type": "number", - "default": 1.0, + "default": 1, "description": "Number of CPUs used for the by GTDB-Tk run tool pplacer.", "help_text": "A low number of CPUs helps to reduce the memory required/reported by GTDB-Tk. See also the [GTDB-Tk documentation](https://ecogenomics.github.io/GTDBTk/faq.html#gtdb-tk-reaches-the-memory-limit-pplacer-crashes)." }, diff --git a/workflows/mag/tests/main.nf.test b/workflows/mag/tests/main.nf.test index b960e7cf..66a50f50 100644 --- a/workflows/mag/tests/main.nf.test +++ b/workflows/mag/tests/main.nf.test @@ -18,9 +18,7 @@ nextflow_workflow { megahit_fix_cpu_1 = true spades_fix_cpus = 1 bowtie2_fix_cpu_1 = true - metabat2_fix_cpu_1 = true maxbin2_fix_cpu_1 = true - concoct_fix_cpu_1 = true binning_map_mode = 'own' min_length_unbinned_contigs = 1000000 max_unbinned_contigs = 2 diff --git a/workflows/mag/tests/test_ancient_dna.nf.test b/workflows/mag/tests/test_ancient_dna.nf.test index 55ca203e..6b46a24a 100644 --- a/workflows/mag/tests/test_ancient_dna.nf.test +++ b/workflows/mag/tests/test_ancient_dna.nf.test @@ -25,9 +25,7 @@ nextflow_workflow { skip_prodigal = true bowtie2_fix_cpu_1 = true binning_map_mode = 'own' - metabat2_fix_cpu_1 = true maxbin2_fix_cpu_1 = true - concoct_fix_cpu_1 = true bcftools_view_high_variant_quality = 0 bcftools_view_medium_variant_quality = 0 bcftools_view_minimal_allelesupport = 3 diff --git a/workflows/mag/tests/test_binrefinement.nf.test b/workflows/mag/tests/test_binrefinement.nf.test index 0de723bd..9883b286 100644 --- a/workflows/mag/tests/test_binrefinement.nf.test +++ b/workflows/mag/tests/test_binrefinement.nf.test @@ -22,9 +22,7 @@ nextflow_workflow { skip_quast = true skip_prodigal = true bowtie2_fix_cpu_1 = true - metabat2_fix_cpu_1 = true maxbin2_fix_cpu_1 = true - concoct_fix_cpu_1 = true binning_map_mode = 'own' min_length_unbinned_contigs = 1 max_unbinned_contigs = 2 diff --git a/workflows/mag/tests/test_busco_auto.nf.test b/workflows/mag/tests/test_busco_auto.nf.test index 58293e64..5e7aaf98 100644 --- a/workflows/mag/tests/test_busco_auto.nf.test +++ b/workflows/mag/tests/test_busco_auto.nf.test @@ -23,9 +23,7 @@ nextflow_workflow { skip_quast = true skip_prodigal = true bowtie2_fix_cpu_1 = true - metabat2_fix_cpu_1 = true maxbin2_fix_cpu_1 = true - concoct_fix_cpu_1 = true binning_map_mode = 'own' min_length_unbinned_contigs = 1 max_unbinned_contigs = 2 From 972ec9297cdfb2794d0ab7efd72756ea270cc91e Mon Sep 17 00:00:00 2001 From: Carson J Miller <68351153+CarsonJM@users.noreply.github.com> Date: Wed, 17 Jan 2024 22:14:44 +0000 Subject: [PATCH 32/49] Fixed linting --- .github/workflows/ci.yml | 27 +++++++++++++++++++++++---- .github/workflows/linting.yml | 2 +- .nf-core.yml | 1 + nextflow_schema.json | 8 ++++---- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 030ff298..9034d1db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,3 +1,4 @@ +# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors name: nf-core CI on: pull_request: @@ -12,7 +13,10 @@ on: env: NXF_ANSI_LOG: false - NFTEST_VER: "0.8.1" + NFT_VER: "0.8.3" + NFT_WORKDIR: "~" + NFT_DIFF: "pdiff" + NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2" concurrency: group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" @@ -71,7 +75,17 @@ jobs: - name: Install Nextflow uses: nf-core/setup-nextflow@v1 with: - version: "23.10.0" + version: "${{ matrix.NXF_VER }}" + + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + architecture: "x64" + + - name: Install pdiff to see diff between nf-test snapshots + run: | + python -m pip install --upgrade pip + pip install pdiff - name: Cache nf-test installation id: cache-software @@ -80,7 +94,7 @@ jobs: path: | /usr/local/bin/nf-test /home/runner/.nf-test/nf-test.jar - key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest + key: ${{ runner.os }}-${{ env.NFT_VER }}-nftest - name: Install nf-test if: steps.cache-software.outputs.cache-hit != 'true' @@ -90,7 +104,12 @@ jobs: - name: Run nf-test run: | - nf-test test --tag ${{ matrix.tags }} --profile test,${{ matrix.profile }},test_data --junitxml=test.xml + nf-test test --verbose --tag ${{ matrix.tags }} --profile "${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap + + - uses: pcolby/tap-summary@v1 + with: + path: >- + test.tap - name: Output log on failure if: failure() diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 5c001706..905c58e4 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/setup-node@v4 - name: Install Prettier - run: npm install -g prettier@3.1.0 + run: npm install -g prettier - name: Run Prettier --check run: prettier --check ${GITHUB_WORKSPACE} diff --git a/.nf-core.yml b/.nf-core.yml index feabf6ad..073c81b8 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -3,3 +3,4 @@ repository_type: pipeline lint: files_unchanged: - lib/NfcoreTemplate.groovy + actions_ci: false diff --git a/nextflow_schema.json b/nextflow_schema.json index 6fe08aab..67fb8a4a 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -279,7 +279,8 @@ "default": "", "properties": { "bowtie2_fix_cpu_1": { - "type": "boolean" + "type": "boolean", + "description": "Fix number of CPUS used by MaxBin2. Not increased with retries" }, "megahit_fix_cpu_1": { "type": "boolean", @@ -299,9 +300,8 @@ "help_text": "SPAdes is designed to be deterministic for a given number of threads. To generate reproducible results fix the number of CPUs using this parameter.\n\nWhen using this parameter do not change the number of CPUs for the `spadeshybrid` process with a custom config file. This would result in an error.\n\nDefault: -1 (the number of CPUs is specified in the `base.config` or in a custom config file, and increased with each retry)." }, "maxbin2_fix_cpu_1": { - "type": "number", - "description": "Fix number of CPUS used by MaxBin2. Not increased with retries", - "default": -1 + "type": "boolean", + "description": "Fix number of CPUS used by MaxBin2. Not increased with retries" }, "metabat_rng_seed": { "type": "integer", From f4c7633c9b6023aab6bdbd9155dabf5a648aaf2a Mon Sep 17 00:00:00 2001 From: Carson J Miller <68351153+CarsonJM@users.noreply.github.com> Date: Wed, 17 Jan 2024 22:16:50 +0000 Subject: [PATCH 33/49] Fixed CI yet again --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9034d1db..ec2de1f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,7 @@ jobs: - name: Run nf-test run: | - nf-test test --verbose --tag ${{ matrix.tags }} --profile "${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap + nf-test test --verbose --tag ${{ matrix.tags }} --profile test,test_data,"${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap - uses: pcolby/tap-summary@v1 with: From 2cadf7b676ce5cb3f10d1362a022043278ac864c Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 17 Jan 2024 22:44:00 +0000 Subject: [PATCH 34/49] [automated] Fix linting with Prettier --- .devcontainer/devcontainer.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 4ecfbfe3..4a9bc5c7 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -18,11 +18,11 @@ "python.linting.flake8Path": "/opt/conda/bin/flake8", "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle", "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle", - "python.linting.pylintPath": "/opt/conda/bin/pylint" + "python.linting.pylintPath": "/opt/conda/bin/pylint", }, // Add the IDs of extensions you want installed when the container is created. - "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"] - } - } + "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"], + }, + }, } From 332b7c75f084e426eb3dff26c51896f44bf33a76 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 17 Jan 2024 23:12:32 +0000 Subject: [PATCH 35/49] [automated] Fix linting with Prettier --- .devcontainer/devcontainer.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 4ecfbfe3..4a9bc5c7 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -18,11 +18,11 @@ "python.linting.flake8Path": "/opt/conda/bin/flake8", "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle", "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle", - "python.linting.pylintPath": "/opt/conda/bin/pylint" + "python.linting.pylintPath": "/opt/conda/bin/pylint", }, // Add the IDs of extensions you want installed when the container is created. - "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"] - } - } + "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"], + }, + }, } From 96c48ccb8a17b8ad09e771f6767b526b04a87637 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Fri, 2 Feb 2024 19:35:24 +0000 Subject: [PATCH 36/49] Updated cat/cat and bin_summary --- .devcontainer/devcontainer.json | 8 +- modules.json | 2 +- modules/nf-core/cat/cat/main.nf | 23 +++- modules/nf-core/cat/cat/tests/main.nf.test | 32 ++++- .../nf-core/cat/cat/tests/main.nf.test.snap | 92 ++++++++------ nextflow.config | 4 +- workflows/mag/main.nf | 114 +++++++++--------- 7 files changed, 168 insertions(+), 107 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 4a9bc5c7..4ecfbfe3 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -18,11 +18,11 @@ "python.linting.flake8Path": "/opt/conda/bin/flake8", "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle", "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle", - "python.linting.pylintPath": "/opt/conda/bin/pylint", + "python.linting.pylintPath": "/opt/conda/bin/pylint" }, // Add the IDs of extensions you want installed when the container is created. - "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"], - }, - }, + "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"] + } + } } diff --git a/modules.json b/modules.json index 50b8f434..00ff9e0d 100644 --- a/modules.json +++ b/modules.json @@ -38,7 +38,7 @@ }, "cat/cat": { "branch": "master", - "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", + "git_sha": "81f27e75847087865299cc46605deb3b09b4e0a2", "installed_by": ["modules"] }, "cat/fastq": { diff --git a/modules/nf-core/cat/cat/main.nf b/modules/nf-core/cat/cat/main.nf index 4264a92c..adbdbd7b 100644 --- a/modules/nf-core/cat/cat/main.nf +++ b/modules/nf-core/cat/cat/main.nf @@ -22,6 +22,8 @@ process CAT_CAT { def args2 = task.ext.args2 ?: '' def file_list = files_in.collect { it.toString() } + // choose appropriate concatenation tool depending on input and output format + // | input | output | command1 | command2 | // |-----------|------------|----------|----------| // | gzipped | gzipped | cat | | @@ -30,11 +32,15 @@ process CAT_CAT { // | ungzipped | gzipped | cat | pigz | // Use input file ending as default - prefix = task.ext.prefix ?: "${meta.id}${file_list[0].substring(file_list[0].lastIndexOf('.'))}" + prefix = task.ext.prefix ?: "${meta.id}${getFileSuffix(file_list[0])}" out_zip = prefix.endsWith('.gz') in_zip = file_list[0].endsWith('.gz') command1 = (in_zip && !out_zip) ? 'zcat' : 'cat' command2 = (!in_zip && out_zip) ? "| pigz -c -p $task.cpus $args2" : '' + if(file_list.contains(prefix.trim())) { + error "The name of the input file can't be the same as for the output prefix in the " + + "module CAT_CAT (currently `$prefix`). Please choose a different one." + } """ $command1 \\ $args \\ @@ -49,8 +55,12 @@ process CAT_CAT { """ stub: - def file_list = files_in.collect { it.toString() } - prefix = task.ext.prefix ?: "${meta.id}${file_list[0].substring(file_list[0].lastIndexOf('.'))}" + def file_list = files_in.collect { it.toString() } + prefix = task.ext.prefix ?: "${meta.id}${file_list[0].substring(file_list[0].lastIndexOf('.'))}" + if(file_list.contains(prefix.trim())) { + error "The name of the input file can't be the same as for the output prefix in the " + + "module CAT_CAT (currently `$prefix`). Please choose a different one." + } """ touch $prefix @@ -60,3 +70,10 @@ process CAT_CAT { END_VERSIONS """ } + +// for .gz files also include the second to last extension if it is present. E.g., .fasta.gz +def getFileSuffix(filename) { + def match = filename =~ /^.*?((\.\w{1,5})?(\.\w{1,5}\.gz$))/ + return match ? match[0][1] : filename.substring(filename.lastIndexOf('.')) +} + diff --git a/modules/nf-core/cat/cat/tests/main.nf.test b/modules/nf-core/cat/cat/tests/main.nf.test index 5766daaf..aaae04f9 100644 --- a/modules/nf-core/cat/cat/tests/main.nf.test +++ b/modules/nf-core/cat/cat/tests/main.nf.test @@ -8,6 +8,32 @@ nextflow_process { tag "cat" tag "cat/cat" + test("test_cat_name_conflict") { + when { + params { + outdir = "${outputDir}" + } + process { + """ + input[0] = + [ + [ id:'genome', single_end:true ], + [ + file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true), + file(params.test_data['sarscov2']['genome']['genome_sizes'], checkIfExists: true) + ] + ] + """ + } + } + then { + assertAll( + { assert !process.success }, + { assert process.stdout.toString().contains("The name of the input file can't be the same as for the output prefix") } + ) + } + } + test("test_cat_unzipped_unzipped") { when { params { @@ -57,8 +83,7 @@ nextflow_process { def lines = path(process.out.file_out.get(0).get(1)).linesGzip assertAll( { assert process.success }, - { assert snapshot(lines[0..5]).match("test_cat_zipped_zipped_lines") }, - { assert snapshot(lines.size()).match("test_cat_zipped_zipped_size")} + { assert snapshot(process.out).match() } ) } } @@ -116,8 +141,7 @@ nextflow_process { def lines = path(process.out.file_out.get(0).get(1)).linesGzip assertAll( { assert process.success }, - { assert snapshot(lines[0..5]).match("test_cat_unzipped_zipped_lines") }, - { assert snapshot(lines.size()).match("test_cat_unzipped_zipped_size")} + { assert snapshot(process.out).match() } ) } } diff --git a/modules/nf-core/cat/cat/tests/main.nf.test.snap b/modules/nf-core/cat/cat/tests/main.nf.test.snap index 423571ba..0c9bfe8d 100644 --- a/modules/nf-core/cat/cat/tests/main.nf.test.snap +++ b/modules/nf-core/cat/cat/tests/main.nf.test.snap @@ -1,10 +1,4 @@ { - "test_cat_unzipped_zipped_size": { - "content": [ - 375 - ], - "timestamp": "2023-10-16T14:33:08.049445686" - }, "test_cat_unzipped_unzipped": { "content": [ { @@ -67,31 +61,36 @@ ], "timestamp": "2023-10-16T14:32:49.642741302" }, - "test_cat_zipped_zipped_lines": { - "content": [ - [ - "MT192765.1\tGenbank\ttranscript\t259\t29667\t.\t+\t.\tID=unknown_transcript_1;geneID=orf1ab;gene_name=orf1ab", - "MT192765.1\tGenbank\tgene\t259\t21548\t.\t+\t.\tParent=unknown_transcript_1", - "MT192765.1\tGenbank\tCDS\t259\t13461\t.\t+\t0\tParent=unknown_transcript_1;exception=\"ribosomal slippage\";gbkey=CDS;gene=orf1ab;note=\"pp1ab;translated=by -1 ribosomal frameshift\";product=\"orf1ab polyprotein\";protein_id=QIK50426.1", - "MT192765.1\tGenbank\tCDS\t13461\t21548\t.\t+\t0\tParent=unknown_transcript_1;exception=\"ribosomal slippage\";gbkey=CDS;gene=orf1ab;note=\"pp1ab;translated=by -1 ribosomal frameshift\";product=\"orf1ab polyprotein\";protein_id=QIK50426.1", - "MT192765.1\tGenbank\tCDS\t21556\t25377\t.\t+\t0\tParent=unknown_transcript_1;gbkey=CDS;gene=S;note=\"structural protein\";product=\"surface glycoprotein\";protein_id=QIK50427.1", - "MT192765.1\tGenbank\tgene\t21556\t25377\t.\t+\t.\tParent=unknown_transcript_1" - ] - ], - "timestamp": "2023-10-16T14:32:33.629048645" - }, - "test_cat_unzipped_zipped_lines": { + "test_cat_zipped_zipped": { "content": [ - [ - ">MT192765.1 Severe acute respiratory syndrome coronavirus 2 isolate SARS-CoV-2/human/USA/PC00101P/2020, complete genome", - "GTTTATACCTTCCCAGGTAACAAACCAACCAACTTTCGATCTCTTGTAGATCTGTTCTCTAAACGAACTTTAAAATCTGT", - "GTGGCTGTCACTCGGCTGCATGCTTAGTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAG", - "TAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTTTGTCCGG", - "GTGTGACCGAAAGGTAAGATGGAGAGCCTTGTCCCTGGTTTCAACGAGAAAACACACGTCCAACTCAGTTTGCCTGTTTT", - "ACAGGTTCGCGACGTGCTCGTACGTGGCTTTGGAGACTCCGTGGAGGAGGTCTTATCAGAGGCACGTCAACATCTTAAAG" - ] + { + "0": [ + [ + { + "id": "test", + "single_end": true + }, + "test.gff3.gz:md5,c439d3b60e7bc03e8802a451a0d9a5d9" + ] + ], + "1": [ + "versions.yml:md5,115ed6177ebcff24eb99d503fa5ef894" + ], + "file_out": [ + [ + { + "id": "test", + "single_end": true + }, + "test.gff3.gz:md5,c439d3b60e7bc03e8802a451a0d9a5d9" + ] + ], + "versions": [ + "versions.yml:md5,115ed6177ebcff24eb99d503fa5ef894" + ] + } ], - "timestamp": "2023-10-16T14:33:08.038830506" + "timestamp": "2024-01-12T14:02:02.999254641" }, "test_cat_one_file_unzipped_zipped_lines": { "content": [ @@ -106,16 +105,41 @@ ], "timestamp": "2023-10-16T14:33:21.39642399" }, - "test_cat_zipped_zipped_size": { + "test_cat_unzipped_zipped": { "content": [ - 78 + { + "0": [ + [ + { + "id": "test", + "single_end": true + }, + "cat.txt.gz:md5,f44b33a0e441ad58b2d3700270e2dbe2" + ] + ], + "1": [ + "versions.yml:md5,115ed6177ebcff24eb99d503fa5ef894" + ], + "file_out": [ + [ + { + "id": "test", + "single_end": true + }, + "cat.txt.gz:md5,f44b33a0e441ad58b2d3700270e2dbe2" + ] + ], + "versions": [ + "versions.yml:md5,115ed6177ebcff24eb99d503fa5ef894" + ] + } ], - "timestamp": "2023-10-16T14:32:33.641869244" + "timestamp": "2024-01-12T14:08:26.948048418" }, "test_cat_one_file_unzipped_zipped_size": { "content": [ 374 ], - "timestamp": "2023-10-16T14:33:21.4094373" + "timestamp": "2024-01-12T14:10:22.445700266" } -} \ No newline at end of file +} diff --git a/nextflow.config b/nextflow.config index e9c138d1..f81b08d9 100644 --- a/nextflow.config +++ b/nextflow.config @@ -305,10 +305,12 @@ profiles { executor.cpus = 4 executor.memory = 8.GB } - test_data { includeConfig 'conf/test_data.config' } test { includeConfig 'conf/test.config' } } +// include test data config by default +includeConfig 'conf/test_data.config' + // Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile // Will not be used unless Apptainer / Docker / Podman / Singularity are enabled // Set to your registry if you have a mirror of containers diff --git a/workflows/mag/main.nf b/workflows/mag/main.nf index 880d41bf..544584e7 100644 --- a/workflows/mag/main.nf +++ b/workflows/mag/main.nf @@ -369,15 +369,21 @@ workflow MAG { .groupTuple() .branch { meta, reads -> - cat: ( meta.single_end && reads.size() == 1 ) || ( !meta.single_end && reads.size() >= 2 ) + cat: reads.size() >= 2 // SE: [[meta], [S1_R1, S2_R1]]; PE: [[meta], [[S1_R1, S1_R2], [S2_R1, S2_R2]]] skip_cat: true // Can skip merging if only single lanes } CAT_FASTQ ( ch_short_reads_forcat.cat.map { meta, reads -> [ meta, reads.flatten() ]} ) - - ch_short_reads = Channel.empty() - ch_short_reads = CAT_FASTQ.out.reads.mix( ch_short_reads_forcat.skip_cat ).map { meta, reads -> [ meta, reads.flatten() ]} - ch_versions = ch_versions.mix(CAT_FASTQ.out.versions.first()) + // Ensure we don't have nests of nests so that structure is in form expected for assembly + ch_short_reads_catskipped = ch_short_reads_forcat.skip_cat + .map { meta, reads -> + def new_reads = meta.single_end ? reads[0] : reads.flatten() + [ meta, new_reads ] + } + // Combine single run and multi-run-merged data + ch_short_reads = Channel.empty() + ch_short_reads = CAT_FASTQ.out.reads.mix(ch_short_reads_catskipped) + ch_versions = ch_versions.mix(CAT_FASTQ.out.versions.first()) if ( params.bbnorm ) { if ( params.coassemble_group ) { @@ -474,27 +480,29 @@ workflow MAG { if ( !ch_centrifuge_db_file.isEmpty() ) { if ( ch_centrifuge_db_file.extension in ['gz', 'tgz'] ) { // Expects to be tar.gz! - ch_db_for_centrifuge = CENTRIFUGE_DB_PREPARATION ( ch_centrifuge_db_file ).db + ch_db_for_centrifuge = CENTRIFUGE_DB_PREPARATION ( ch_centrifuge_db_file ) + .db + .collect() + .map{ + db -> + def db_name = db[0].getBaseName().split('\\.')[0] + [ db_name, db ] + } } else if ( ch_centrifuge_db_file.isDirectory() ) { ch_db_for_centrifuge = Channel .fromPath( "${ch_centrifuge_db_file}/*.cf" ) + .collect() + .map{ + db -> + def db_name = db[0].getBaseName().split('\\.')[0] + [ db_name, db ] + } } else { ch_db_for_centrifuge = Channel.empty() } } else { ch_db_for_centrifuge = Channel.empty() } - - // Centrifuge val(db_name) has to be the basename of any of the - // index files up to but not including the final .1.cf - ch_db_for_centrifuge = ch_db_for_centrifuge - .collect() - .map{ - db -> - def db_name = db[0].getBaseName().split('\\.')[0] - [ db_name, db ] - } - CENTRIFUGE ( ch_short_reads, ch_db_for_centrifuge @@ -850,8 +858,6 @@ workflow MAG { } else if ( params.postbinning_input == 'refined_bins_only' ) { ch_input_for_postbinning_bins = ch_refined_bins ch_input_for_postbinning_bins_unbins = ch_refined_bins.mix(ch_refined_unbins) - // TODO REACTIVATE ONCE PR #489 IS READY! - // TODO RE-ADD BOTH TO SCHEMA ONCE RE-ADDING } else if ( params.postbinning_input == 'both' ) { ch_all_bins = ch_binning_results_bins.mix(ch_refined_bins) ch_input_for_postbinning_bins = ch_all_bins @@ -933,14 +939,16 @@ workflow MAG { def new_bins = bins.flatten() [meta, new_bins] } - QUAST_BINS ( ch_input_for_quast_bins ) ch_versions = ch_versions.mix(QUAST_BINS.out.versions.first()) - ch_quast_bin_summaries = QUAST_BINS.out.quast_bin_summaries - QUAST_BINS_SUMMARY ( QUAST_BINS.out.quast_bin_summaries.collect() ) + ch_quast_bin_summary = QUAST_BINS.out.quast_bin_summaries + .collectFile(keepHeader: true) { + meta, summary -> + ["${meta.id}.tsv", summary] + } + QUAST_BINS_SUMMARY ( ch_quast_bin_summary.collect() ) ch_quast_bins_summary = QUAST_BINS_SUMMARY.out.summary } - /* * CAT: Bin Annotation Tool (BAT) are pipelines for the taxonomic classification of long DNA sequences and metagenome assembled genomes (MAGs/bins) */ @@ -956,27 +964,36 @@ workflow MAG { ch_input_for_postbinning_bins_unbins, ch_cat_db ) + // Group all classification results for each sample in a single file + ch_cat_summary = CAT.out.tax_classification_names + .collectFile(keepHeader: true) { + meta, classification -> + ["${meta.id}.txt", classification] + } + // Group all classification results for the whole run in a single file CAT_SUMMARY( - CAT.out.tax_classification_names.collect() + ch_cat_summary.collect() ) - ch_cat_tax_classification_names = CAT.out.tax_classification_names ch_versions = ch_versions.mix(CAT.out.versions.first()) ch_versions = ch_versions.mix(CAT_SUMMARY.out.versions) + // If CAT is not run, then the CAT global summary should be an empty channel + if ( params.cat_db_generate || params.cat_db) { + ch_cat_global_summary = CAT_SUMMARY.out.summary + ch_cat_global_summary = CAT_SUMMARY.out.combined + } else { + ch_cat_global_summary = Channel.empty() + } /* * GTDB-tk: taxonomic classifications using GTDB reference */ - if ( !params.skip_gtdbtk ) { - ch_gtdbtk_summary = Channel.empty() if ( gtdb ){ - ch_gtdb_bins = ch_input_for_postbinning_bins_unbins .filter { meta, bins -> meta.domain != "eukarya" } - GTDBTK ( ch_gtdb_bins, ch_busco_summary, @@ -985,28 +1002,24 @@ workflow MAG { gtdb_mash ) ch_versions = ch_versions.mix(GTDBTK.out.versions.first()) - ch_gtdbtk_summaries = GTDBTK.out.gtdbtk_summaries ch_gtdbtk_summary = GTDBTK.out.summary } } else { ch_gtdbtk_summary = Channel.empty() } - if ( ( !params.skip_binqc ) || !params.skip_quast || !params.skip_gtdbtk){ BIN_SUMMARY ( ch_input_for_binsummary, ch_busco_summary.ifEmpty([]), ch_checkm_summary.ifEmpty([]), ch_quast_bins_summary.ifEmpty([]), - ch_gtdbtk_summary.ifEmpty([]) + ch_gtdbtk_summary.ifEmpty([]), + ch_cat_global_summary.ifEmpty([]) ) - ch_bin_summaries = BIN_SUMMARY.out.summary } - /* * Prokka: Genome annotation */ - if (!params.skip_prokka){ ch_bins_for_prokka = ch_input_for_postbinning_bins_unbins.transpose() .map { meta, bin -> @@ -1016,7 +1029,6 @@ workflow MAG { .filter { meta, bin -> meta.domain != "eukarya" } - PROKKA ( ch_bins_for_prokka, [], @@ -1025,7 +1037,6 @@ workflow MAG { ch_prokka_faa = PROKKA.out.faa ch_versions = ch_versions.mix(PROKKA.out.versions.first()) } - if (!params.skip_metaeuk && (params.metaeuk_db || params.metaeuk_mmseqs_db)) { ch_bins_for_metaeuk = ch_input_for_postbinning_bins_unbins.transpose() .filter { meta, bin -> @@ -1035,88 +1046,66 @@ workflow MAG { def meta_new = meta + [id: bin.getBaseName()] [ meta_new, bin ] } - METAEUK_EASYPREDICT (ch_bins_for_metaeuk, ch_metaeuk_db) ch_metaeuk_easypredict_faa = METAEUK_EASYPREDICT.out.faa ch_versions = ch_versions.mix(METAEUK_EASYPREDICT.out.versions) } } - CUSTOM_DUMPSOFTWAREVERSIONS ( ch_versions.unique().collectFile(name: 'collated_versions.yml') ) - // // MODULE: MultiQC // workflow_summary = WorkflowMag.paramsSummaryMultiqc(workflow, summary_params) ch_workflow_summary = Channel.value(workflow_summary) - methods_description = WorkflowMag.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description, params) ch_methods_description = Channel.value(methods_description) - ch_multiqc_files = Channel.empty() ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(FASTQC_RAW.out.zip.collect{it[1]}.ifEmpty([])) - if (!params.assembly_input) { - if ( !params.skip_clipping && params.clip_tool == 'adapterremoval' ) { ch_multiqc_files = ch_multiqc_files.mix(ADAPTERREMOVAL_PE.out.settings.collect{it[1]}.ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(ADAPTERREMOVAL_SE.out.settings.collect{it[1]}.ifEmpty([])) - } else if ( !params.skip_clipping && params.clip_tool == 'fastp' ) { ch_multiqc_files = ch_multiqc_files.mix(FASTP.out.json.collect{it[1]}.ifEmpty([])) } - if (!(params.keep_phix && params.skip_clipping && !(params.host_genome || params.host_fasta))) { ch_multiqc_files = ch_multiqc_files.mix(FASTQC_TRIMMED.out.zip.collect{it[1]}.ifEmpty([])) } - if ( params.host_fasta || params.host_genome ) { ch_multiqc_files = ch_multiqc_files.mix(BOWTIE2_HOST_REMOVAL_ALIGN.out.log.collect{it[1]}.ifEmpty([])) } - if(!params.keep_phix) { ch_multiqc_files = ch_multiqc_files.mix(BOWTIE2_PHIX_REMOVAL_ALIGN.out.log.collect{it[1]}.ifEmpty([])) } - } - ch_multiqc_files = ch_multiqc_files.mix(CENTRIFUGE.out.kreport.collect{it[1]}.ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(KRAKEN2.out.report.collect{it[1]}.ifEmpty([])) - if (!params.skip_quast){ ch_multiqc_files = ch_multiqc_files.mix(QUAST.out.report.collect().ifEmpty([])) - if ( !params.skip_binning ) { ch_multiqc_files = ch_multiqc_files.mix(QUAST_BINS.out.dir.collect().ifEmpty([])) } } - if ( !params.skip_binning || params.ancient_dna ) { ch_multiqc_files = ch_multiqc_files.mix(BINNING_PREPARATION.out.bowtie2_assembly_multiqc.collect().ifEmpty([])) } - if (!params.skip_binning && !params.skip_prokka){ ch_multiqc_files = ch_multiqc_files.mix(PROKKA.out.txt.collect{it[1]}.ifEmpty([])) } - if (!params.skip_binning && !params.skip_binqc && params.binqc_tool == 'busco'){ ch_multiqc_files = ch_multiqc_files.mix(BUSCO_QC.out.multiqc.collect().ifEmpty([])) } - - MULTIQC ( ch_multiqc_files.collect(), ch_multiqc_config.toList(), ch_multiqc_custom_config.toList(), ch_multiqc_logo.toList() ) - multiqc_report = MULTIQC.out.report.toList() emit: @@ -1127,7 +1116,6 @@ workflow MAG { refined_bins = ch_refined_bins refined_unbins = ch_refined_unbins bin_summary = ch_bin_summaries - cat = ch_cat_tax_classification_names prokka = ch_prokka_faa metaeuk = ch_metaeuk_easypredict_faa versions = ch_versions @@ -1149,6 +1137,12 @@ workflow.onComplete { NfcoreTemplate.IM_notification(workflow, params, summary_params, projectDir, log) } } +workflow.onError { + if (workflow.errorReport.contains("Process requirement exceeds available memory")) { + println("🛑 Default resources exceed availability 🛑 ") + println("💡 See here on how to configure pipeline: https://nf-co.re/docs/usage/configuration#tuning-workflow-resources 💡") + } +} /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From bd07e3ac65e840f6061f7efb802735fa36dc78bd Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Mon, 5 Feb 2024 23:42:12 +0000 Subject: [PATCH 37/49] Started updating nf-test snapshots --- .github/workflows/ci.yml | 2 +- .gitignore | 1 + nextflow.config | 3 --- nf-test.config | 16 +++++--------- tests/nextflow.config | 22 +++++++++++++++++++ .../tests/test_adapterremoval.nf.test.snap | 16 +++++--------- workflows/mag/tests/test_nothing.nf.test.snap | 12 +++------- 7 files changed, 37 insertions(+), 35 deletions(-) create mode 100644 tests/nextflow.config diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec2de1f2..9034d1db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,7 @@ jobs: - name: Run nf-test run: | - nf-test test --verbose --tag ${{ matrix.tags }} --profile test,test_data,"${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap + nf-test test --verbose --tag ${{ matrix.tags }} --profile "${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap - uses: pcolby/tap-summary@v1 with: diff --git a/.gitignore b/.gitignore index 8d38c7c0..613fd0c9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ testing* *.pyc .nf-tests/ .nf-test.log +.nf-test/ diff --git a/nextflow.config b/nextflow.config index f81b08d9..3411aa6a 100644 --- a/nextflow.config +++ b/nextflow.config @@ -308,9 +308,6 @@ profiles { test { includeConfig 'conf/test.config' } } -// include test data config by default -includeConfig 'conf/test_data.config' - // Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile // Will not be used unless Apptainer / Docker / Podman / Singularity are enabled // Set to your registry if you have a mirror of containers diff --git a/nf-test.config b/nf-test.config index 2f9c518d..6f5e2c47 100644 --- a/nf-test.config +++ b/nf-test.config @@ -1,16 +1,10 @@ config { - // location for all nf-tests + // Location of nf-tests testsDir "." - // nf-test directory including temporary files for each test - workDir ".nf-tests" + // nf-test directory used to create temporary files for each test + workDir System.getenv("NFT_WORKDIR") ?: ".nf-test" - // location of library folder that is added automatically to the classpath - libDir "lib/" - - // location of an optional nextflow.config file specific for executing tests - configFile "nextflow.config" - - // run all test with the defined docker profile from the main nextflow.config - profile "" + // Location of an optional nextflow.config file specific for executing pipeline tests + configFile "tests/nextflow.config" } diff --git a/tests/nextflow.config b/tests/nextflow.config new file mode 100644 index 00000000..67b43474 --- /dev/null +++ b/tests/nextflow.config @@ -0,0 +1,22 @@ +// include test data config by default +includeConfig '../conf/test_data.config' + +// Impose sensible resource limits for testing +process { + withName: '.*' { + cpus = 2 + memory = 3.GB + time = 2.h + } +} + +// Impose same minimum Nextflow version as the pipeline for testing +manifest { + nextflowVersion = '!>=23.04.0' +} + +// Disable all Nextflow reporting options +timeline { enabled = false } +report { enabled = false } +trace { enabled = false } +dag { enabled = false } diff --git a/workflows/mag/tests/test_adapterremoval.nf.test.snap b/workflows/mag/tests/test_adapterremoval.nf.test.snap index a52e2142..6f1dfede 100644 --- a/workflows/mag/tests/test_adapterremoval.nf.test.snap +++ b/workflows/mag/tests/test_adapterremoval.nf.test.snap @@ -28,12 +28,6 @@ ], "1": [ - ], - "10": [ - "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", - "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", - "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", - "versions.yml:md5,eca9f4c6e135848b1247916040e52b99" ], "2": [ @@ -57,16 +51,16 @@ ], "9": [ - + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,eca9f4c6e135848b1247916040e52b99" ], "assemblies": [ ], "bin_summary": [ - ], - "cat": [ - ], "genomad": [ @@ -118,6 +112,6 @@ ] } ], - "timestamp": "2024-01-17T09:17:39.984696852" + "timestamp": "2024-02-05T23:39:32.916744986" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_nothing.nf.test.snap b/workflows/mag/tests/test_nothing.nf.test.snap index 81e6c78d..cf1874f9 100644 --- a/workflows/mag/tests/test_nothing.nf.test.snap +++ b/workflows/mag/tests/test_nothing.nf.test.snap @@ -28,10 +28,6 @@ ], "1": [ - ], - "10": [ - "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", - "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400" ], "2": [ @@ -55,16 +51,14 @@ ], "9": [ - + "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400" ], "assemblies": [ ], "bin_summary": [ - ], - "cat": [ - ], "genomad": [ @@ -114,6 +108,6 @@ ] } ], - "timestamp": "2024-01-17T08:28:52.988011944" + "timestamp": "2024-02-05T23:35:18.598081054" } } \ No newline at end of file From a9c8c08989e0db5e0d1406f503a84875692f9bae Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Thu, 8 Feb 2024 15:57:37 -0800 Subject: [PATCH 38/49] Updated several workflow nf-tests after pipeline version update --- conf/test.config | 3 + nf-test.config | 4 +- tests/nextflow.config | 22 ----- .../mag/tests/test_ancient_dna.nf.test.snap | 94 +++++++++---------- workflows/mag/tests/test_bbnorm.nf.test.snap | 24 +++-- .../mag/tests/test_binrefinement.nf.test.snap | 74 +++++++-------- .../mag/tests/test_busco_auto.nf.test.snap | 72 +++++++------- workflows/mag/tests/test_host_rm.nf.test.snap | 20 ++-- workflows/mag/tests/test_hybrid.nf.test.snap | 30 +++--- .../tests/test_hybrid_host_rm.nf.test.snap | 28 +++--- 10 files changed, 169 insertions(+), 202 deletions(-) delete mode 100644 tests/nextflow.config diff --git a/conf/test.config b/conf/test.config index c063074c..a58152d8 100644 --- a/conf/test.config +++ b/conf/test.config @@ -19,3 +19,6 @@ params { max_memory = '6.GB' max_time = '6.h' } + +// Load test_data config +includeConfig './test_data.config' diff --git a/nf-test.config b/nf-test.config index 6f5e2c47..3ba80fb3 100644 --- a/nf-test.config +++ b/nf-test.config @@ -5,6 +5,6 @@ config { // nf-test directory used to create temporary files for each test workDir System.getenv("NFT_WORKDIR") ?: ".nf-test" - // Location of an optional nextflow.config file specific for executing pipeline tests - configFile "tests/nextflow.config" + // location of an optional nextflow.config file specific for executing tests + configFile "nextflow.config" } diff --git a/tests/nextflow.config b/tests/nextflow.config deleted file mode 100644 index 67b43474..00000000 --- a/tests/nextflow.config +++ /dev/null @@ -1,22 +0,0 @@ -// include test data config by default -includeConfig '../conf/test_data.config' - -// Impose sensible resource limits for testing -process { - withName: '.*' { - cpus = 2 - memory = 3.GB - time = 2.h - } -} - -// Impose same minimum Nextflow version as the pipeline for testing -manifest { - nextflowVersion = '!>=23.04.0' -} - -// Disable all Nextflow reporting options -timeline { enabled = false } -report { enabled = false } -trace { enabled = false } -dag { enabled = false } diff --git a/workflows/mag/tests/test_ancient_dna.nf.test.snap b/workflows/mag/tests/test_ancient_dna.nf.test.snap index 0698a573..b045cd24 100644 --- a/workflows/mag/tests/test_ancient_dna.nf.test.snap +++ b/workflows/mag/tests/test_ancient_dna.nf.test.snap @@ -64,49 +64,6 @@ "SPAdes-test_minigut_sample2_scaffolds.fasta:md5,98a1c0bf47c301bbd838b32a54b7537b" ] ], - "10": [ - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", - "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", - "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", - "versions.yml:md5,3550550b622b6d5f6803ffed9742e08b", - "versions.yml:md5,368d757df0f57b49bce09965641cc71a", - "versions.yml:md5,47df50b983a036f55cb936247d920744", - "versions.yml:md5,586dc48e8e2cc8106cb0ee3b70ce0fef", - "versions.yml:md5,6d41fb977d31ce96d7ddbb4ea2dbe38b", - "versions.yml:md5,7017c8959eb098440346ca9a2310c460", - "versions.yml:md5,7ccffaba4e75be98b8faedb54ccea16f", - "versions.yml:md5,8187d1683f97b64a582620caba2b175a", - "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", - "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", - "versions.yml:md5,aee39cb21305f6d3fe88bf9863529706", - "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", - "versions.yml:md5,c777f00c6a48f3c0490d09b1aa8d81a6", - "versions.yml:md5,cc4a4972327dec07faf1f54ead653715", - "versions.yml:md5,d8d2321bec0c70de2f9750f258fc890d", - "versions.yml:md5,dc29f891cb297131cf940ba5580a6832", - "versions.yml:md5,df654dd4fdb4039c2d8cf0b6b49f4b55", - "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", - "versions.yml:md5,e9f8fe610b5abe68b28d77efc951e9e6" - ], "2": [ ], @@ -195,7 +152,47 @@ ], "9": [ - + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,3550550b622b6d5f6803ffed9742e08b", + "versions.yml:md5,368d757df0f57b49bce09965641cc71a", + "versions.yml:md5,47df50b983a036f55cb936247d920744", + "versions.yml:md5,586dc48e8e2cc8106cb0ee3b70ce0fef", + "versions.yml:md5,6d41fb977d31ce96d7ddbb4ea2dbe38b", + "versions.yml:md5,7017c8959eb098440346ca9a2310c460", + "versions.yml:md5,7ccffaba4e75be98b8faedb54ccea16f", + "versions.yml:md5,8187d1683f97b64a582620caba2b175a", + "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,aee39cb21305f6d3fe88bf9863529706", + "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", + "versions.yml:md5,c777f00c6a48f3c0490d09b1aa8d81a6", + "versions.yml:md5,cc4a4972327dec07faf1f54ead653715", + "versions.yml:md5,d8d2321bec0c70de2f9750f258fc890d", + "versions.yml:md5,dc29f891cb297131cf940ba5580a6832", + "versions.yml:md5,df654dd4fdb4039c2d8cf0b6b49f4b55", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", + "versions.yml:md5,e9f8fe610b5abe68b28d77efc951e9e6" ], "assemblies": [ [ @@ -237,9 +234,6 @@ ], "bin_summary": [ - ], - "cat": [ - ], "genomad": [ @@ -394,6 +388,10 @@ ] } ], - "timestamp": "2024-01-17T08:28:03.926093976" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-08T11:19:54.901540968" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_bbnorm.nf.test.snap b/workflows/mag/tests/test_bbnorm.nf.test.snap index f7b0a04f..2d7504c4 100644 --- a/workflows/mag/tests/test_bbnorm.nf.test.snap +++ b/workflows/mag/tests/test_bbnorm.nf.test.snap @@ -28,14 +28,6 @@ ], "1": [ - ], - "10": [ - "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", - "versions.yml:md5,67cbab915f2801633ea61219086a7735", - "versions.yml:md5,67cbab915f2801633ea61219086a7735", - "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", - "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", - "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0" ], "2": [ @@ -59,16 +51,18 @@ ], "9": [ - + "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", + "versions.yml:md5,67cbab915f2801633ea61219086a7735", + "versions.yml:md5,67cbab915f2801633ea61219086a7735", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0" ], "assemblies": [ ], "bin_summary": [ - ], - "cat": [ - ], "genomad": [ @@ -122,6 +116,10 @@ ] } ], - "timestamp": "2024-01-17T09:19:27.939943074" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-08T13:57:10.226099714" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_binrefinement.nf.test.snap b/workflows/mag/tests/test_binrefinement.nf.test.snap index 1ebe267b..6aa34add 100644 --- a/workflows/mag/tests/test_binrefinement.nf.test.snap +++ b/workflows/mag/tests/test_binrefinement.nf.test.snap @@ -60,39 +60,6 @@ "SPAdes-test_minigut_sample2_contigs.fasta:md5,0c3fdd6f534d5e21b105be45351ee655" ] ], - "10": [ - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", - "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", - "versions.yml:md5,3550550b622b6d5f6803ffed9742e08b", - "versions.yml:md5,586dc48e8e2cc8106cb0ee3b70ce0fef", - "versions.yml:md5,6d41fb977d31ce96d7ddbb4ea2dbe38b", - "versions.yml:md5,7017c8959eb098440346ca9a2310c460", - "versions.yml:md5,7ccffaba4e75be98b8faedb54ccea16f", - "versions.yml:md5,8187d1683f97b64a582620caba2b175a", - "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", - "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", - "versions.yml:md5,aee39cb21305f6d3fe88bf9863529706", - "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", - "versions.yml:md5,c777f00c6a48f3c0490d09b1aa8d81a6", - "versions.yml:md5,cc4a4972327dec07faf1f54ead653715", - "versions.yml:md5,d8d2321bec0c70de2f9750f258fc890d", - "versions.yml:md5,dc29f891cb297131cf940ba5580a6832", - "versions.yml:md5,df654dd4fdb4039c2d8cf0b6b49f4b55", - "versions.yml:md5,e9f8fe610b5abe68b28d77efc951e9e6", - "versions.yml:md5,ede79d94ca30c948c1707584760b230c", - "versions.yml:md5,ede79d94ca30c948c1707584760b230c", - "versions.yml:md5,ede79d94ca30c948c1707584760b230c", - "versions.yml:md5,ede79d94ca30c948c1707584760b230c" - ], "2": [ ], @@ -163,7 +130,37 @@ ], "9": [ - + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,3550550b622b6d5f6803ffed9742e08b", + "versions.yml:md5,586dc48e8e2cc8106cb0ee3b70ce0fef", + "versions.yml:md5,6d41fb977d31ce96d7ddbb4ea2dbe38b", + "versions.yml:md5,7017c8959eb098440346ca9a2310c460", + "versions.yml:md5,7ccffaba4e75be98b8faedb54ccea16f", + "versions.yml:md5,8187d1683f97b64a582620caba2b175a", + "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,aee39cb21305f6d3fe88bf9863529706", + "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", + "versions.yml:md5,c777f00c6a48f3c0490d09b1aa8d81a6", + "versions.yml:md5,cc4a4972327dec07faf1f54ead653715", + "versions.yml:md5,d8d2321bec0c70de2f9750f258fc890d", + "versions.yml:md5,dc29f891cb297131cf940ba5580a6832", + "versions.yml:md5,df654dd4fdb4039c2d8cf0b6b49f4b55", + "versions.yml:md5,e9f8fe610b5abe68b28d77efc951e9e6", + "versions.yml:md5,ede79d94ca30c948c1707584760b230c", + "versions.yml:md5,ede79d94ca30c948c1707584760b230c", + "versions.yml:md5,ede79d94ca30c948c1707584760b230c", + "versions.yml:md5,ede79d94ca30c948c1707584760b230c" ], "assemblies": [ [ @@ -201,9 +198,6 @@ ], "bin_summary": [ - ], - "cat": [ - ], "genomad": [ @@ -330,6 +324,10 @@ ] } ], - "timestamp": "2024-01-17T09:39:03.032859012" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-08T14:25:50.952306304" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_busco_auto.nf.test.snap b/workflows/mag/tests/test_busco_auto.nf.test.snap index b10490c2..03182252 100644 --- a/workflows/mag/tests/test_busco_auto.nf.test.snap +++ b/workflows/mag/tests/test_busco_auto.nf.test.snap @@ -46,7 +46,28 @@ "MEGAHIT-test_minigut_sample2.contigs.fa:md5,2c0b7977b39fb2db31ee9b3adf75f8c6" ] ], - "10": [ + "2": [ + + ], + "3": [ + + ], + "4": [ + + ], + "5": [ + + ], + "6": [ + + ], + "7": [ + + ], + "8": [ + + ], + "9": [ "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", @@ -67,30 +88,6 @@ "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0" - ], - "2": [ - - ], - "3": [ - - ], - "4": [ - - ], - "5": [ - - ], - "6": [ - "bin_summary.tsv:md5,59e4568ef471db7c5d1f1105b3325216" - ], - "7": [ - - ], - "8": [ - - ], - "9": [ - ], "assemblies": [ [ @@ -113,28 +110,25 @@ ] ], "bin_summary": [ - "bin_summary.tsv:md5,59e4568ef471db7c5d1f1105b3325216" - ], - "cat": [ - + ], "genomad": [ - + ], "metaeuk": [ - + ], "prodigal": [ - + ], "prokka": [ - + ], "refined_bins": [ - + ], "refined_unbins": [ - + ], "short_reads": [ [ @@ -184,6 +178,10 @@ ] } ], - "timestamp": "2024-01-17T09:53:53.262636096" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-08T15:03:14.092155168" } -} \ No newline at end of file +} diff --git a/workflows/mag/tests/test_host_rm.nf.test.snap b/workflows/mag/tests/test_host_rm.nf.test.snap index 3e64f68d..082ff0dc 100644 --- a/workflows/mag/tests/test_host_rm.nf.test.snap +++ b/workflows/mag/tests/test_host_rm.nf.test.snap @@ -28,12 +28,6 @@ ], "1": [ - ], - "10": [ - "versions.yml:md5,59050473c9f749b269ad7079fd4693df", - "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", - "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", - "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0" ], "2": [ @@ -57,16 +51,16 @@ ], "9": [ - + "versions.yml:md5,59050473c9f749b269ad7079fd4693df", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0" ], "assemblies": [ ], "bin_summary": [ - ], - "cat": [ - ], "genomad": [ @@ -118,6 +112,10 @@ ] } ], - "timestamp": "2024-01-17T09:55:27.641552946" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-08T10:07:30.70766162" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_hybrid.nf.test.snap b/workflows/mag/tests/test_hybrid.nf.test.snap index 643ef8e9..10399055 100644 --- a/workflows/mag/tests/test_hybrid.nf.test.snap +++ b/workflows/mag/tests/test_hybrid.nf.test.snap @@ -35,17 +35,6 @@ "SPAdesHybrid-group-0_scaffolds.fasta:md5,84c68c6ba721ad5858588a1487e05b6f" ] ], - "10": [ - "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", - "versions.yml:md5,1bd6bbec156534880e04e0320dbce063", - "versions.yml:md5,5de8983775d3ef64cdbec6f382852619", - "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", - "versions.yml:md5,b94aaba1559ac9deaf7339181c398405", - "versions.yml:md5,cc143ba1b806a335a9963a434c887eba", - "versions.yml:md5,d13508b80e270ac6f1543725b431a682", - "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", - "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0" - ], "2": [ ], @@ -68,7 +57,15 @@ ], "9": [ - + "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", + "versions.yml:md5,1bd6bbec156534880e04e0320dbce063", + "versions.yml:md5,5de8983775d3ef64cdbec6f382852619", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,b94aaba1559ac9deaf7339181c398405", + "versions.yml:md5,cc143ba1b806a335a9963a434c887eba", + "versions.yml:md5,d13508b80e270ac6f1543725b431a682", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0" ], "assemblies": [ [ @@ -92,9 +89,6 @@ ], "bin_summary": [ - ], - "cat": [ - ], "genomad": [ @@ -140,6 +134,10 @@ ] } ], - "timestamp": "2024-01-17T10:12:08.554896181" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-08T15:47:29.016078996" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_hybrid_host_rm.nf.test.snap b/workflows/mag/tests/test_hybrid_host_rm.nf.test.snap index a4f438fb..b8ad3a86 100644 --- a/workflows/mag/tests/test_hybrid_host_rm.nf.test.snap +++ b/workflows/mag/tests/test_hybrid_host_rm.nf.test.snap @@ -26,16 +26,6 @@ "SPAdesHybrid-minigut_scaffolds.fasta:md5,d3c5e27f011a08daea597f9d2650436d" ] ], - "10": [ - "versions.yml:md5,1bd6bbec156534880e04e0320dbce063", - "versions.yml:md5,59050473c9f749b269ad7079fd4693df", - "versions.yml:md5,5de8983775d3ef64cdbec6f382852619", - "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", - "versions.yml:md5,b94aaba1559ac9deaf7339181c398405", - "versions.yml:md5,cc143ba1b806a335a9963a434c887eba", - "versions.yml:md5,d13508b80e270ac6f1543725b431a682", - "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0" - ], "2": [ ], @@ -58,7 +48,14 @@ ], "9": [ - + "versions.yml:md5,1bd6bbec156534880e04e0320dbce063", + "versions.yml:md5,59050473c9f749b269ad7079fd4693df", + "versions.yml:md5,5de8983775d3ef64cdbec6f382852619", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,b94aaba1559ac9deaf7339181c398405", + "versions.yml:md5,cc143ba1b806a335a9963a434c887eba", + "versions.yml:md5,d13508b80e270ac6f1543725b431a682", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0" ], "assemblies": [ [ @@ -73,9 +70,6 @@ ], "bin_summary": [ - ], - "cat": [ - ], "genomad": [ @@ -120,6 +114,10 @@ ] } ], - "timestamp": "2024-01-17T10:03:17.006107632" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-08T15:28:58.822332791" } } \ No newline at end of file From 61d86aa1ae16717c8fbb75c10ab2d6304d71e40c Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Fri, 9 Feb 2024 00:00:42 +0000 Subject: [PATCH 39/49] Added test.config to CI for mem req --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9034d1db..a7e72278 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,7 @@ jobs: - name: Run nf-test run: | - nf-test test --verbose --tag ${{ matrix.tags }} --profile "${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap + nf-test test --verbose --tag ${{ matrix.tags }} --profile "${{ matrix.profile }}",test --junitxml=test.xml --tap=test.tap - uses: pcolby/tap-summary@v1 with: From 03e36317ff7d653bc1934ff6bd39a0770606747a Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Mon, 12 Feb 2024 09:04:45 -0800 Subject: [PATCH 40/49] Updated virus identification test --- workflows/mag/tests/main.nf.test.snap | 94 +++++++++---------- .../test_virus_identification.nf.test.snap | 48 +++++----- 2 files changed, 69 insertions(+), 73 deletions(-) diff --git a/workflows/mag/tests/main.nf.test.snap b/workflows/mag/tests/main.nf.test.snap index 7b3f8e57..70718aec 100644 --- a/workflows/mag/tests/main.nf.test.snap +++ b/workflows/mag/tests/main.nf.test.snap @@ -64,45 +64,6 @@ "SPAdes-test_minigut_sample2_scaffolds.fasta:md5,235505d69a2822d77ad70ce061b80efe" ] ], - "10": [ - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", - "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", - "versions.yml:md5,18eeebfdf8c302c29cbb4abc4325628c", - "versions.yml:md5,200880072147033d455fede5cf4499dc", - "versions.yml:md5,2acb240e473bc70de2e0bb8a392748ac", - "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", - "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", - "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", - "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", - "versions.yml:md5,3550550b622b6d5f6803ffed9742e08b", - "versions.yml:md5,368d757df0f57b49bce09965641cc71a", - "versions.yml:md5,519a472d661281a99e0335305cdd1d85", - "versions.yml:md5,59e59108b091ee58d131ac74e9bfac43", - "versions.yml:md5,5cf45e79d026b3593199fa035f2d72ec", - "versions.yml:md5,7017c8959eb098440346ca9a2310c460", - "versions.yml:md5,7067399ef1687d4d1ba0aab93dc67754", - "versions.yml:md5,8187d1683f97b64a582620caba2b175a", - "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", - "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", - "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,bb7110425028f7e7abcbf17152405079", - "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", - "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", - "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", - "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", - "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", - "versions.yml:md5,ea3029fd33017b2f39de14bce337c246" - ], "2": [ [ { @@ -151,12 +112,9 @@ ], "6": [ - "bin_summary.tsv:md5,cb825146f68ff4d1722fc7064ae9017b" - ], - "7": [ ], - "8": [ + "7": [ [ { "id": "MEGAHIT-MaxBin2-test_minigut.001", @@ -266,8 +224,47 @@ "SPAdes-MetaBAT2-test_minigut.2.faa:md5,1cc11d484e49ff51111b0e0bf7a2349b" ] ], - "9": [ + "8": [ + ], + "9": [ + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,0f67b0839fcef0334e0dcaf5568be9e0", + "versions.yml:md5,15111ae1ae8b8b570a2efddd7c747bd0", + "versions.yml:md5,18eeebfdf8c302c29cbb4abc4325628c", + "versions.yml:md5,200880072147033d455fede5cf4499dc", + "versions.yml:md5,2acb240e473bc70de2e0bb8a392748ac", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,2d4f5bd36e073ac683075e4bc52884be", + "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", + "versions.yml:md5,3550550b622b6d5f6803ffed9742e08b", + "versions.yml:md5,368d757df0f57b49bce09965641cc71a", + "versions.yml:md5,519a472d661281a99e0335305cdd1d85", + "versions.yml:md5,59e59108b091ee58d131ac74e9bfac43", + "versions.yml:md5,5cf45e79d026b3593199fa035f2d72ec", + "versions.yml:md5,7017c8959eb098440346ca9a2310c460", + "versions.yml:md5,7067399ef1687d4d1ba0aab93dc67754", + "versions.yml:md5,8187d1683f97b64a582620caba2b175a", + "versions.yml:md5,82c86a3130605a0de153c903b2cfc6d2", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,b4ee8f08c1efaac6dee44f113c7a2e0b", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,bb7110425028f7e7abcbf17152405079", + "versions.yml:md5,c384e0cacaa344670dcde728d78a524a", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,d654471556992ae3c82a7fa792c90dc0", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0", + "versions.yml:md5,ea3029fd33017b2f39de14bce337c246" ], "assemblies": [ [ @@ -308,9 +305,6 @@ ] ], "bin_summary": [ - "bin_summary.tsv:md5,cb825146f68ff4d1722fc7064ae9017b" - ], - "cat": [ ], "genomad": [ @@ -538,6 +532,10 @@ ] } ], - "timestamp": "2024-01-17T10:27:28.027473013" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-09T13:01:27.205986175" } } \ No newline at end of file diff --git a/workflows/mag/tests/test_virus_identification.nf.test.snap b/workflows/mag/tests/test_virus_identification.nf.test.snap index 6e68b049..8d077933 100644 --- a/workflows/mag/tests/test_virus_identification.nf.test.snap +++ b/workflows/mag/tests/test_virus_identification.nf.test.snap @@ -37,13 +37,8 @@ "MEGAHIT-group-0.contigs.fa:md5,8419d4d6c72ac1006324a9fee1394646" ] ], - "10": [ - "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", - "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", - "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0" - ], "2": [ - + ], "3": [ [ @@ -53,26 +48,28 @@ "single_end": false, "assembler": "MEGAHIT" }, - "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,95db91fff85f23a19e64ef7ea0f8d371" + "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,b08861c8a938b3eaf8c1d8e533935042" ] ], "4": [ - + ], "5": [ - + ], "6": [ - + ], "7": [ - + ], "8": [ - + ], "9": [ - + "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", + "versions.yml:md5,85a05e91a1084dbca180ff6ff7e67400", + "versions.yml:md5,e2ed4d9a5b5d63ae1a6eef0418d976f0" ], "assemblies": [ [ @@ -86,10 +83,7 @@ ] ], "bin_summary": [ - - ], - "cat": [ - + ], "genomad": [ [ @@ -99,23 +93,23 @@ "single_end": false, "assembler": "MEGAHIT" }, - "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,95db91fff85f23a19e64ef7ea0f8d371" + "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,b08861c8a938b3eaf8c1d8e533935042" ] ], "metaeuk": [ - + ], "prodigal": [ - + ], "prokka": [ - + ], "refined_bins": [ - + ], "refined_unbins": [ - + ], "short_reads": [ [ @@ -148,6 +142,10 @@ ] } ], - "timestamp": "2024-01-17T09:14:33.005055708" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-09T15:30:48.208307453" } -} +} \ No newline at end of file From 4f3068bd1dda740038976162853cab7fca320ecd Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Mon, 12 Feb 2024 09:24:04 -0800 Subject: [PATCH 41/49] Updated test config temporarily to test CI --- conf/test.config | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/conf/test.config b/conf/test.config index 28984d13..c063074c 100644 --- a/conf/test.config +++ b/conf/test.config @@ -18,17 +18,4 @@ params { max_cpus = 2 max_memory = '6.GB' max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' - centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" - kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" - skip_krona = true - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz" - busco_clean = true - skip_gtdbtk = true - gtdbtk_min_completeness = 0 - skip_concoct = true } From 1f4d36f199fca1f359cd7146dac64daa24a61b76 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Mon, 12 Feb 2024 10:32:56 -0800 Subject: [PATCH 42/49] Update virus_identification snap --- .../test_virus_identification.nf.test.snap | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/workflows/mag/tests/test_virus_identification.nf.test.snap b/workflows/mag/tests/test_virus_identification.nf.test.snap index 8d077933..1f1d2eee 100644 --- a/workflows/mag/tests/test_virus_identification.nf.test.snap +++ b/workflows/mag/tests/test_virus_identification.nf.test.snap @@ -38,7 +38,7 @@ ] ], "2": [ - + ], "3": [ [ @@ -48,23 +48,23 @@ "single_end": false, "assembler": "MEGAHIT" }, - "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,b08861c8a938b3eaf8c1d8e533935042" + "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,95db91fff85f23a19e64ef7ea0f8d371" ] ], "4": [ - + ], "5": [ - + ], "6": [ - + ], "7": [ - + ], "8": [ - + ], "9": [ "versions.yml:md5,3222fd64337a6c070682b63fc5e0cb3c", @@ -83,7 +83,7 @@ ] ], "bin_summary": [ - + ], "genomad": [ [ @@ -93,23 +93,23 @@ "single_end": false, "assembler": "MEGAHIT" }, - "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,b08861c8a938b3eaf8c1d8e533935042" + "MEGAHIT-group-0.contigs_virus_summary.tsv:md5,95db91fff85f23a19e64ef7ea0f8d371" ] ], "metaeuk": [ - + ], "prodigal": [ - + ], "prokka": [ - + ], "refined_bins": [ - + ], "refined_unbins": [ - + ], "short_reads": [ [ @@ -148,4 +148,4 @@ }, "timestamp": "2024-02-09T15:30:48.208307453" } -} \ No newline at end of file +} From 9ade6a3db6d3513b1db6ff8871b1f84e2ae19f67 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Mon, 12 Feb 2024 15:47:00 -0800 Subject: [PATCH 43/49] Include test_data for nf-tests --- conf/test.config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf/test.config b/conf/test.config index c063074c..6980f774 100644 --- a/conf/test.config +++ b/conf/test.config @@ -19,3 +19,6 @@ params { max_memory = '6.GB' max_time = '6.h' } + +// Load test_data.config +includeConfig './test_data.config' From 1bc461908e22313a23c4a382ed06c4730203fd1a Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Mon, 12 Feb 2024 15:52:55 -0800 Subject: [PATCH 44/49] Added nf-test config file --- .github/workflows/ci.yml | 2 +- nf-test.config | 2 +- tests/nextflow.config | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 tests/nextflow.config diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7e72278..9034d1db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,7 @@ jobs: - name: Run nf-test run: | - nf-test test --verbose --tag ${{ matrix.tags }} --profile "${{ matrix.profile }}",test --junitxml=test.xml --tap=test.tap + nf-test test --verbose --tag ${{ matrix.tags }} --profile "${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap - uses: pcolby/tap-summary@v1 with: diff --git a/nf-test.config b/nf-test.config index 3ba80fb3..a2f80c2e 100644 --- a/nf-test.config +++ b/nf-test.config @@ -6,5 +6,5 @@ config { workDir System.getenv("NFT_WORKDIR") ?: ".nf-test" // location of an optional nextflow.config file specific for executing tests - configFile "nextflow.config" + configFile "test/nextflow.config" } diff --git a/tests/nextflow.config b/tests/nextflow.config new file mode 100644 index 00000000..4dc4a547 --- /dev/null +++ b/tests/nextflow.config @@ -0,0 +1,34 @@ +params { + // Base directory for nf-core/modules test data + modules_testdata_base_path = 's3://ngi-igenomes/testdata/nf-core/modules/' + + // Base directory for nf-core/fetchngs test data + pipelines_testdata_base_path = 's3://ngi-igenomes/testdata/nf-core/pipelines/fetchngs/1.15.0/' +} + +// Impose sensible resource limits for testing +process { + withName: '.*' { + cpus = 2 + memory = 3.GB + time = 2.h + } +} + +// Impose same minimum Nextflow version as the pipeline for testing +manifest { + nextflowVersion = '!>=23.04.0' +} + +// Disable all Nextflow reporting options +timeline { enabled = false } +report { enabled = false } +trace { enabled = false } +dag { enabled = false } + +// Load test_data.config +includeConfig '../conf/test_data.config' + + +// Include base config for reproducibility functions +includeConfig '../nextflow.config' From 5561b5fb216a06594039fd25dfa8a448a0b8762a Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Mon, 12 Feb 2024 15:56:13 -0800 Subject: [PATCH 45/49] Fixed path to nf-test config --- nf-test.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf-test.config b/nf-test.config index a2f80c2e..1848e921 100644 --- a/nf-test.config +++ b/nf-test.config @@ -6,5 +6,5 @@ config { workDir System.getenv("NFT_WORKDIR") ?: ".nf-test" // location of an optional nextflow.config file specific for executing tests - configFile "test/nextflow.config" + configFile "tests/nextflow.config" } From 1bdbc497e033dc3790873bdf1f376a5196a1d037 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Wed, 14 Feb 2024 08:52:36 -0800 Subject: [PATCH 46/49] Added back config files and restructued nf-test config --- .github/workflows/ci.yml | 2 +- conf/{test_bbnorm.config => mag_test.config} | 20 +++----- ....config => mag_test_adapterremoval.config} | 23 +++++---- conf/mag_test_ancient_dna.config | 49 +++++++++++++++++++ ...sco_auto.config => mag_test_bbnorm.config} | 25 ++++++---- conf/mag_test_binrefinement.config | 47 ++++++++++++++++++ conf/mag_test_busco_auto.config | 43 ++++++++++++++++ ...host_rm.config => mag_test_host_rm.config} | 23 ++++++--- ...t_hybrid.config => mag_test_hybrid.config} | 22 +++++++-- ....config => mag_test_hybrid_host_rm.config} | 21 ++++++-- conf/mag_test_nothing.config | 41 ++++++++++++++++ ...g => mag_test_virus_identification.config} | 31 ++++++------ conf/test.config | 24 --------- conf/test_ancient_dna.config | 42 ---------------- conf/test_binrefinement.config | 38 -------------- conf/test_nothing.config | 44 ----------------- nextflow.config | 15 +++++- tests/nextflow.config | 15 ++---- workflows/mag/tests/main.nf.test | 41 ---------------- workflows/mag/tests/test.nf.test | 21 ++++++++ .../{main.nf.test.snap => test.nf.test.snap} | 0 .../mag/tests/test_adapterremoval.nf.test | 21 -------- workflows/mag/tests/test_ancient_dna.nf.test | 31 ------------ workflows/mag/tests/test_bbnorm.nf.test | 22 --------- .../mag/tests/test_binrefinement.nf.test | 29 ----------- workflows/mag/tests/test_busco_auto.nf.test | 25 ---------- workflows/mag/tests/test_host_rm.nf.test | 24 --------- workflows/mag/tests/test_hybrid.nf.test | 24 --------- .../mag/tests/test_hybrid_host_rm.nf.test | 24 --------- workflows/mag/tests/test_nothing.nf.test | 22 --------- .../tests/test_virus_identification.nf.test | 25 ---------- 31 files changed, 321 insertions(+), 513 deletions(-) rename conf/{test_bbnorm.config => mag_test.config} (74%) rename conf/{test_adapterremoval.config => mag_test_adapterremoval.config} (63%) create mode 100644 conf/mag_test_ancient_dna.config rename conf/{test_busco_auto.config => mag_test_bbnorm.config} (63%) create mode 100644 conf/mag_test_binrefinement.config create mode 100644 conf/mag_test_busco_auto.config rename conf/{test_host_rm.config => mag_test_host_rm.config} (64%) rename conf/{test_hybrid.config => mag_test_hybrid.config} (59%) rename conf/{test_hybrid_host_rm.config => mag_test_hybrid_host_rm.config} (67%) create mode 100644 conf/mag_test_nothing.config rename conf/{test_virus_identification.config => mag_test_virus_identification.config} (80%) delete mode 100644 conf/test.config delete mode 100644 conf/test_ancient_dna.config delete mode 100644 conf/test_binrefinement.config delete mode 100644 conf/test_nothing.config delete mode 100644 workflows/mag/tests/main.nf.test create mode 100644 workflows/mag/tests/test.nf.test rename workflows/mag/tests/{main.nf.test.snap => test.nf.test.snap} (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9034d1db..92bb2ad6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,7 @@ jobs: - name: Run nf-test run: | - nf-test test --verbose --tag ${{ matrix.tags }} --profile "${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap + nf-test test --verbose --tag ${{ matrix.tags }} --profile "${{ matrix.profile }}",${{ matrix.tags }} --junitxml=test.xml --tap=test.tap - uses: pcolby/tap-summary@v1 with: diff --git a/conf/test_bbnorm.config b/conf/mag_test.config similarity index 74% rename from conf/test_bbnorm.config rename to conf/mag_test.config index a31f6b7b..2c6974a9 100644 --- a/conf/test_bbnorm.config +++ b/conf/mag_test.config @@ -19,23 +19,19 @@ params { max_memory = '6.GB' max_time = '6.h' - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - keep_phix = true - skip_clipping = true - skip_prokka = true - skip_prodigal = true - skip_quast = true - skip_binning = true + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" skip_krona = true - min_length_unbinned_contigs = 1 + megahit_fix_cpu_1 = true + spades_fix_cpus = 1 + bowtie2_fix_cpu_1 = true + maxbin2_fix_cpu_1 = true + binning_map_mode = 'own' + min_length_unbinned_contigs = 1000000 max_unbinned_contigs = 2 busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz" busco_clean = true skip_gtdbtk = true - gtdbtk_min_completeness = 0 - bbnorm = true - coassemble_group = true + skip_concoct = true } diff --git a/conf/test_adapterremoval.config b/conf/mag_test_adapterremoval.config similarity index 63% rename from conf/test_adapterremoval.config rename to conf/mag_test_adapterremoval.config index ca9fed10..0d9f6a66 100644 --- a/conf/test_adapterremoval.config +++ b/conf/mag_test_adapterremoval.config @@ -21,16 +21,19 @@ params { // Input data input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.euk.csv' - centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" - kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" - metaeuk_db = "https://github.com/nf-core/test-datasets/raw/modules/data/proteomics/database/yeast_UPS.fasta" + clip_tool = 'adapterremoval' + keep_phix = true + centrifuge_db = null + kraken2_db = null skip_krona = true - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz" + skip_megahit = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true skip_gtdbtk = true - gtdbtk_min_completeness = 0 - clip_tool = 'adapterremoval' - skip_concoct = true - bin_domain_classification = true + skip_prokka = true + skip_metaeuk = true } diff --git a/conf/mag_test_ancient_dna.config b/conf/mag_test_ancient_dna.config new file mode 100644 index 00000000..f859d8d2 --- /dev/null +++ b/conf/mag_test_ancient_dna.config @@ -0,0 +1,49 @@ +/* +======================================================================================== + Nextflow config file for running minimal tests +======================================================================================== + Defines input files and everything required to run a fast and simple pipeline test. + + Use as follows: + nextflow run nf-core/mag -profile test_ancient_dna, --outdir + +---------------------------------------------------------------------------------------- +*/ + +params { + config_profile_name = 'Ancient DNA test profile ' + config_profile_description = 'Minimal test dataset to check pipeline function for ancient DNA step' + + // Limit resources so that this can run on GitHub Actions + max_cpus = 2 + max_memory = '6.GB' + max_time = '6.h' + + // Input data + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + skip_clipping = true + keep_phix = true + kraken2_db = null + centrifuge_db = null + skip_krona = true + megahit_fix_cpu_1 = true + spades_fix_cpus = 1 + skip_spadeshybrid = true + ancient_dna = true + skip_quast = true + skip_prodigal = true + bowtie2_fix_cpu_1 = true + binning_map_mode = 'own' + maxbin2_fix_cpu_1 = true + bcftools_view_high_variant_quality = 0 + bcftools_view_medium_variant_quality = 0 + bcftools_view_minimal_allelesupport = 3 + refine_bins_dastool = true + refine_bins_dastool_threshold = 0 + min_length_unbinned_contigs = 1 + max_unbinned_contigs = 2 + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true +} diff --git a/conf/test_busco_auto.config b/conf/mag_test_bbnorm.config similarity index 63% rename from conf/test_busco_auto.config rename to conf/mag_test_bbnorm.config index 48f6b7b5..c3fb8518 100644 --- a/conf/test_busco_auto.config +++ b/conf/mag_test_bbnorm.config @@ -1,11 +1,11 @@ /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Nextflow config file for running minimal tests -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Defines input files and everything required to run a fast and simple pipeline test. Use as follows: - nextflow run nf-core/mag -profile test_busco_auto, --outdir + nextflow run nf-core/mag -profile test, --outdir ---------------------------------------------------------------------------------------- */ @@ -21,13 +21,20 @@ params { // Input data input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + bbnorm = true + keep_phix = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_megahit = true skip_spades = true - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true skip_gtdbtk = true - gtdbtk_min_completeness = 0 skip_prokka = true - skip_prodigal = true - skip_quast = true - skip_concoct = true + skip_metaeuk = true } diff --git a/conf/mag_test_binrefinement.config b/conf/mag_test_binrefinement.config new file mode 100644 index 00000000..4f6df31a --- /dev/null +++ b/conf/mag_test_binrefinement.config @@ -0,0 +1,47 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for running minimal tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Defines input files and everything required to run a fast and simple pipeline test. + + Use as follows: + nextflow run nf-core/mag -profile test_binrefinement, --outdir + +---------------------------------------------------------------------------------------- +*/ + +params { + config_profile_name = 'Test profile' + config_profile_description = 'Minimal test dataset to check pipeline function' + + // Limit resources so that this can run on GitHub Actions + max_cpus = 2 + max_memory = '6.GB' + max_time = '6.h' + + // Input data + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + assembly_input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/assembly_samplesheet.csv' + bbnorm = true + keep_phix = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_quast = true + skip_prodigal = true + bowtie2_fix_cpu_1 = true + maxbin2_fix_cpu_1 = true + binning_map_mode = 'own' + min_length_unbinned_contigs = 1 + max_unbinned_contigs = 2 + refine_bins_dastool = true + refine_bins_dastool_threshold = 0 + // TODO not using 'both' until #489 merged + postbinning_input = 'refined_bins_only' + skip_busco = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true +} diff --git a/conf/mag_test_busco_auto.config b/conf/mag_test_busco_auto.config new file mode 100644 index 00000000..9fda8cf5 --- /dev/null +++ b/conf/mag_test_busco_auto.config @@ -0,0 +1,43 @@ +/* +======================================================================================== + Nextflow config file for running minimal tests +======================================================================================== + Defines input files and everything required to run a fast and simple pipeline test. + + Use as follows: + nextflow run nf-core/mag -profile test_busco_auto, --outdir + +---------------------------------------------------------------------------------------- +*/ + +params { + config_profile_name = 'Test profile' + config_profile_description = 'Minimal test dataset to check pipeline function' + + // Limit resources so that this can run on GitHub Actions + max_cpus = 2 + max_memory = '6.GB' + max_time = '6.h' + + // Input data + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' + keep_phix = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + megahit_fix_cpu_1 = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + bowtie2_fix_cpu_1 = true + maxbin2_fix_cpu_1 = true + binning_map_mode = 'own' + min_length_unbinned_contigs = 1 + max_unbinned_contigs = 2 + skip_concoct = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true +} diff --git a/conf/test_host_rm.config b/conf/mag_test_host_rm.config similarity index 64% rename from conf/test_host_rm.config rename to conf/mag_test_host_rm.config index afd4e687..906c68a7 100644 --- a/conf/test_host_rm.config +++ b/conf/mag_test_host_rm.config @@ -20,12 +20,23 @@ params { max_time = '6.h' // Input data - host_fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/host_reference/genome.hg38.chr21_10000bp_region.fa" input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.host_rm.csv' - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz" + keep_phix = true + host_fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/host_reference/genome.hg38.chr21_10000bp_region.fa" + bowtie2_fix_cpu_1 = true + skip_clipping = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_megahit = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true skip_gtdbtk = true - gtdbtk_min_completeness = 0 - skip_concoct = true + skip_prokka = true + skip_metaeuk = true } diff --git a/conf/test_hybrid.config b/conf/mag_test_hybrid.config similarity index 59% rename from conf/test_hybrid.config rename to conf/mag_test_hybrid.config index a9f7ee07..d0fb0bcc 100644 --- a/conf/test_hybrid.config +++ b/conf/mag_test_hybrid.config @@ -21,10 +21,22 @@ params { // Input data input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.hybrid.csv' - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz" + keep_phix = true + skip_clipping = false + skip_adapter_trimming = false + centrifuge_db = null + kraken2_db = null + skip_krona = true + coassemble_group = true + megahit_fix_cpu_1 = true + skip_spades = true + skip_spadeshybrid = false + spadeshybrid_fix_cpus = 2 + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true skip_gtdbtk = true - gtdbtk_min_completeness = 0 - skip_concoct = true + skip_prokka = true + skip_metaeuk = true } diff --git a/conf/test_hybrid_host_rm.config b/conf/mag_test_hybrid_host_rm.config similarity index 67% rename from conf/test_hybrid_host_rm.config rename to conf/mag_test_hybrid_host_rm.config index 531a89d3..05b2f8b4 100644 --- a/conf/test_hybrid_host_rm.config +++ b/conf/mag_test_hybrid_host_rm.config @@ -20,12 +20,23 @@ params { max_time = '6.h' // Input data - host_fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/host_reference/genome.hg38.chr21_10000bp_region.fa" input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.hybrid_host_rm.csv' - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 + host_fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/host_reference/genome.hg38.chr21_10000bp_region.fa" + bowtie2_fix_cpu_1 = true + keep_phix = true + skip_clipping = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_megahit = true + skip_spades = true + skip_spadeshybrid = false + spadeshybrid_fix_cpus = 2 + skip_quast = true + skip_prodigal = true + skip_binning = true skip_binqc = true - skip_concoct = true skip_gtdbtk = true - gtdbtk_min_completeness = 0 + skip_prokka = true + skip_metaeuk = true } diff --git a/conf/mag_test_nothing.config b/conf/mag_test_nothing.config new file mode 100644 index 00000000..8904d2f6 --- /dev/null +++ b/conf/mag_test_nothing.config @@ -0,0 +1,41 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for running minimal tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Runs input data but skipping all possible steps to allow for a fast testing + profile for input checks etc. + + Use as follows: + nextflow run nf-core/mag -profile test_nothing, --outdir + +---------------------------------------------------------------------------------------- +*/ + +params { + config_profile_name = 'Test profile' + config_profile_description = 'Minimal test dataset to check pipeline function' + + // Limit resources so that this can run on GitHub Actions + max_cpus = 2 + max_memory = '6.GB' + max_time = '6.h' + + // Input data + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' + keep_phix = true + skip_clipping = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true + skip_megahit = true + skip_spades = true + skip_spadeshybrid = true + skip_quast = true + skip_prodigal = true + skip_binning = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true +} diff --git a/conf/test_virus_identification.config b/conf/mag_test_virus_identification.config similarity index 80% rename from conf/test_virus_identification.config rename to conf/mag_test_virus_identification.config index dba55db9..176549a3 100644 --- a/conf/test_virus_identification.config +++ b/conf/mag_test_virus_identification.config @@ -19,25 +19,24 @@ params { max_memory = '6.GB' max_time = '6.h' - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - run_virus_identification = true - genomad_splits = 7 - - // For computational efficiency - reads_minlength = 150 + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' + keep_phix = true + skip_clipping = true + skip_adapter_trimming = true + centrifuge_db = null + kraken2_db = null + skip_krona = true coassemble_group = true - skip_gtdbtk = true - gtdbtk_min_completeness = 0 - skip_binning = true - skip_prokka = true + megahit_fix_cpu_1 = true skip_spades = true skip_spadeshybrid = true skip_quast = true + run_virus_identification = true + genomad_splits = 7 skip_prodigal = true - skip_krona = true - skip_adapter_trimming = true - skip_metabat2 = true - skip_maxbin2 = true - skip_busco = true + skip_binning = true + skip_binqc = true + skip_gtdbtk = true + skip_prokka = true + skip_metaeuk = true } diff --git a/conf/test.config b/conf/test.config deleted file mode 100644 index 6980f774..00000000 --- a/conf/test.config +++ /dev/null @@ -1,24 +0,0 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Nextflow config file for running minimal tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run nf-core/mag -profile test, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Test profile' - config_profile_description = 'Minimal test dataset to check pipeline function' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' -} - -// Load test_data.config -includeConfig './test_data.config' diff --git a/conf/test_ancient_dna.config b/conf/test_ancient_dna.config deleted file mode 100644 index 5e935321..00000000 --- a/conf/test_ancient_dna.config +++ /dev/null @@ -1,42 +0,0 @@ -/* -======================================================================================== - Nextflow config file for running minimal tests -======================================================================================== - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run nf-core/mag -profile test_ancient_dna, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Ancient DNA test profile ' - config_profile_description = 'Minimal test dataset to check pipeline function for ancient DNA step' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" - kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" - skip_krona = true - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz" - skip_gtdbtk = true - gtdbtk_min_completeness = 0 - ancient_dna = true - binning_map_mode = 'own' - skip_spades = false - skip_spadeshybrid = true - bcftools_view_high_variant_quality = 0 - bcftools_view_medium_variant_quality = 0 - bcftools_view_minimal_allelesupport = 3 - refine_bins_dastool = true - refine_bins_dastool_threshold = 0 - skip_concoct = true -} diff --git a/conf/test_binrefinement.config b/conf/test_binrefinement.config deleted file mode 100644 index 54144244..00000000 --- a/conf/test_binrefinement.config +++ /dev/null @@ -1,38 +0,0 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Nextflow config file for running minimal tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run nf-core/mag -profile test_binrefinement, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Test profile' - config_profile_description = 'Minimal test dataset to check pipeline function' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - assembly_input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/assembly_samplesheet.csv' - centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" - kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" - skip_krona = true - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz" - skip_gtdbtk = true - gtdbtk_min_completeness = 0 - refine_bins_dastool = true - refine_bins_dastool_threshold = 0 - // TODO not using 'both' until #489 merged - postbinning_input = 'refined_bins_only' - busco_clean = true -} diff --git a/conf/test_nothing.config b/conf/test_nothing.config deleted file mode 100644 index cd3f6311..00000000 --- a/conf/test_nothing.config +++ /dev/null @@ -1,44 +0,0 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Nextflow config file for running minimal tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Runs input data but skipping all possible steps to allow for a fast testing - profile for input checks etc. - - Use as follows: - nextflow run nf-core/mag -profile test_nothing, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Test profile' - config_profile_description = 'Minimal test dataset to check pipeline function' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - centrifuge_db = null - kraken2_db = null - skip_krona = true - skip_clipping = true - skip_adapter_trimming = true - skip_spades = true - skip_spadeshybrid = true - skip_megahit = true - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_metabat2 = true - skip_maxbin2 = true - skip_concoct = true - skip_prokka = true - skip_binqc = true - skip_gtdbtk = true - gtdbtk_min_completeness = 0 - skip_concoct = true -} diff --git a/nextflow.config b/nextflow.config index 2f48ed66..dae04616 100644 --- a/nextflow.config +++ b/nextflow.config @@ -305,7 +305,20 @@ profiles { executor.cpus = 4 executor.memory = 8.GB } - test { includeConfig 'conf/test.config' } + mag_test { includeConfig 'conf/mag_test.config' } + mag_test_full { includeConfig 'conf/mag_test_full.config' } + mag_test_host_rm { includeConfig 'conf/mag_test_host_rm.config' } + mag_test_hybrid { includeConfig 'conf/mag_test_hybrid.config' } + mag_test_hybrid_host_rm { includeConfig 'conf/mag_test_hybrid_host_rm.config' } + mag_test_busco_auto { includeConfig 'conf/mag_test_busco_auto.config' } + mag_test_ancient_dna { includeConfig 'conf/mag_test_ancient_dna.config' } + mag_test_adapterremoval { includeConfig 'conf/mag_test_adapterremoval.config' } + mag_test_binning_entry { includeConfig 'conf/mag_test_binning_entry.config' } + mag_test_binrefinement { includeConfig 'conf/mag_test_binrefinement.config' } + mag_test_no_clipping { includeConfig 'conf/mag_test_no_clipping.config' } + mag_test_bbnorm { includeConfig 'conf/mag_test_bbnorm.config' } + mag_test_nothing { includeConfig 'conf/mag_test_nothing.config' } + mag_test_virus_identification { includeConfig 'conf/mag_test_virus_identification.config' } } // Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile diff --git a/tests/nextflow.config b/tests/nextflow.config index 4dc4a547..6c0fbcbf 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -4,15 +4,10 @@ params { // Base directory for nf-core/fetchngs test data pipelines_testdata_base_path = 's3://ngi-igenomes/testdata/nf-core/pipelines/fetchngs/1.15.0/' -} -// Impose sensible resource limits for testing -process { - withName: '.*' { - cpus = 2 - memory = 3.GB - time = 2.h - } + max_cpus = 2 + max_memory = '6.GB' + max_time = '6.h' } // Impose same minimum Nextflow version as the pipeline for testing @@ -28,7 +23,3 @@ dag { enabled = false } // Load test_data.config includeConfig '../conf/test_data.config' - - -// Include base config for reproducibility functions -includeConfig '../nextflow.config' diff --git a/workflows/mag/tests/main.nf.test b/workflows/mag/tests/main.nf.test deleted file mode 100644 index 66a50f50..00000000 --- a/workflows/mag/tests/main.nf.test +++ /dev/null @@ -1,41 +0,0 @@ -nextflow_workflow { - - name "Test workflow: MAG" - script "../main.nf" - workflow "MAG" - tag "workflows" - tag "mag" - tag "mag_test" - - test("Default paramters") { - - when { - params { - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' - centrifuge_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_cf.tar.gz" - kraken2_db = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/test_data/minigut_kraken.tgz" - skip_krona = true - megahit_fix_cpu_1 = true - spades_fix_cpus = 1 - bowtie2_fix_cpu_1 = true - maxbin2_fix_cpu_1 = true - binning_map_mode = 'own' - min_length_unbinned_contigs = 1000000 - max_unbinned_contigs = 2 - busco_db = "https://busco-data.ezlab.org/v5/data/lineages/bacteria_odb10.2024-01-08.tar.gz" - busco_clean = true - skip_gtdbtk = true - skip_concoct = true - } - } - - then { - assertAll( - { assert workflow.success }, - { assert snapshot( - workflow.out - ).match() } - ) - } - } -} diff --git a/workflows/mag/tests/test.nf.test b/workflows/mag/tests/test.nf.test new file mode 100644 index 00000000..b90b09b2 --- /dev/null +++ b/workflows/mag/tests/test.nf.test @@ -0,0 +1,21 @@ +nextflow_workflow { + + name "Test workflow: MAG" + script "../main.nf" + workflow "MAG" + tag "workflows" + tag "mag" + tag "mag_test" + + test("Default paramters") { + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out + ).match() } + ) + } + } +} diff --git a/workflows/mag/tests/main.nf.test.snap b/workflows/mag/tests/test.nf.test.snap similarity index 100% rename from workflows/mag/tests/main.nf.test.snap rename to workflows/mag/tests/test.nf.test.snap diff --git a/workflows/mag/tests/test_adapterremoval.nf.test b/workflows/mag/tests/test_adapterremoval.nf.test index 781b0e3c..9d05d884 100644 --- a/workflows/mag/tests/test_adapterremoval.nf.test +++ b/workflows/mag/tests/test_adapterremoval.nf.test @@ -9,27 +9,6 @@ nextflow_workflow { test("Parameters: clip_tool = 'adapterremoval'") { - when { - params { - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.euk.csv' - clip_tool = 'adapterremoval' - keep_phix = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - skip_megahit = true - skip_spades = true - skip_spadeshybrid = true - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true - } - } - then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_ancient_dna.nf.test b/workflows/mag/tests/test_ancient_dna.nf.test index 6b46a24a..b0023c14 100644 --- a/workflows/mag/tests/test_ancient_dna.nf.test +++ b/workflows/mag/tests/test_ancient_dna.nf.test @@ -9,37 +9,6 @@ nextflow_workflow { test("Parameters: ancient_dna = true") { - when { - params { - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - skip_clipping = true - keep_phix = true - kraken2_db = null - centrifuge_db = null - skip_krona = true - megahit_fix_cpu_1 = true - spades_fix_cpus = 1 - skip_spadeshybrid = true - ancient_dna = true - skip_quast = true - skip_prodigal = true - bowtie2_fix_cpu_1 = true - binning_map_mode = 'own' - maxbin2_fix_cpu_1 = true - bcftools_view_high_variant_quality = 0 - bcftools_view_medium_variant_quality = 0 - bcftools_view_minimal_allelesupport = 3 - refine_bins_dastool = true - refine_bins_dastool_threshold = 0 - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true - } - } - then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_bbnorm.nf.test b/workflows/mag/tests/test_bbnorm.nf.test index 2b80cd40..5a7dd6ba 100644 --- a/workflows/mag/tests/test_bbnorm.nf.test +++ b/workflows/mag/tests/test_bbnorm.nf.test @@ -9,28 +9,6 @@ nextflow_workflow { test("Parameters: bbnorm = true") { - when { - params { - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - bbnorm = true - keep_phix = true - skip_adapter_trimming = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - skip_megahit = true - skip_spades = true - skip_spadeshybrid = true - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true - } - } - then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_binrefinement.nf.test b/workflows/mag/tests/test_binrefinement.nf.test index 9883b286..277b06e5 100644 --- a/workflows/mag/tests/test_binrefinement.nf.test +++ b/workflows/mag/tests/test_binrefinement.nf.test @@ -9,35 +9,6 @@ nextflow_workflow { test("Parameters: refine_bins_dastool = true") { - when { - params { - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - assembly_input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/assembly_samplesheet.csv' - bbnorm = true - keep_phix = true - skip_adapter_trimming = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - skip_quast = true - skip_prodigal = true - bowtie2_fix_cpu_1 = true - maxbin2_fix_cpu_1 = true - binning_map_mode = 'own' - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - refine_bins_dastool = true - refine_bins_dastool_threshold = 0 - // TODO not using 'both' until #489 merged - postbinning_input = 'refined_bins_only' - skip_busco = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true - } - } - then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_busco_auto.nf.test b/workflows/mag/tests/test_busco_auto.nf.test index 5e7aaf98..5fb41543 100644 --- a/workflows/mag/tests/test_busco_auto.nf.test +++ b/workflows/mag/tests/test_busco_auto.nf.test @@ -9,31 +9,6 @@ nextflow_workflow { test("Parameters: skip_busco = false") { - when { - params { - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.csv' - keep_phix = true - skip_adapter_trimming = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - megahit_fix_cpu_1 = true - skip_spades = true - skip_spadeshybrid = true - skip_quast = true - skip_prodigal = true - bowtie2_fix_cpu_1 = true - maxbin2_fix_cpu_1 = true - binning_map_mode = 'own' - min_length_unbinned_contigs = 1 - max_unbinned_contigs = 2 - skip_concoct = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true - } - } - then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_host_rm.nf.test b/workflows/mag/tests/test_host_rm.nf.test index 478a490d..83ff9862 100644 --- a/workflows/mag/tests/test_host_rm.nf.test +++ b/workflows/mag/tests/test_host_rm.nf.test @@ -9,30 +9,6 @@ nextflow_workflow { test("Parameters: host_fasta != null") { - when { - params { - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.host_rm.csv' - keep_phix = true - host_fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/host_reference/genome.hg38.chr21_10000bp_region.fa" - bowtie2_fix_cpu_1 = true - skip_clipping = true - skip_adapter_trimming = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - skip_megahit = true - skip_spades = true - skip_spadeshybrid = true - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true - } - } - then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_hybrid.nf.test b/workflows/mag/tests/test_hybrid.nf.test index 66c89d7c..424bd4af 100644 --- a/workflows/mag/tests/test_hybrid.nf.test +++ b/workflows/mag/tests/test_hybrid.nf.test @@ -9,30 +9,6 @@ nextflow_workflow { test("Parameters: long reads included") { - when { - params { - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.hybrid.csv' - keep_phix = true - skip_clipping = false - skip_adapter_trimming = false - centrifuge_db = null - kraken2_db = null - skip_krona = true - coassemble_group = true - megahit_fix_cpu_1 = true - skip_spades = true - skip_spadeshybrid = false - spadeshybrid_fix_cpus = 2 - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true - } - } - then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_hybrid_host_rm.nf.test b/workflows/mag/tests/test_hybrid_host_rm.nf.test index 92da398c..ffa38654 100644 --- a/workflows/mag/tests/test_hybrid_host_rm.nf.test +++ b/workflows/mag/tests/test_hybrid_host_rm.nf.test @@ -9,30 +9,6 @@ nextflow_workflow { test("Parameters: long reads & host removal included") { - when { - params { - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.hybrid_host_rm.csv' - host_fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/mag/host_reference/genome.hg38.chr21_10000bp_region.fa" - bowtie2_fix_cpu_1 = true - keep_phix = true - skip_clipping = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - skip_megahit = true - skip_spades = true - skip_spadeshybrid = false - spadeshybrid_fix_cpus = 2 - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true - } - } - then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_nothing.nf.test b/workflows/mag/tests/test_nothing.nf.test index cbeafb06..7b94efbe 100644 --- a/workflows/mag/tests/test_nothing.nf.test +++ b/workflows/mag/tests/test_nothing.nf.test @@ -9,28 +9,6 @@ nextflow_workflow { test("Parameters: test_nothing") { - when { - params { - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' - keep_phix = true - skip_clipping = true - skip_adapter_trimming = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - skip_megahit = true - skip_spades = true - skip_spadeshybrid = true - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true - } - } - then { assertAll( { assert workflow.success }, diff --git a/workflows/mag/tests/test_virus_identification.nf.test b/workflows/mag/tests/test_virus_identification.nf.test index 54b72b0c..657394e7 100644 --- a/workflows/mag/tests/test_virus_identification.nf.test +++ b/workflows/mag/tests/test_virus_identification.nf.test @@ -9,31 +9,6 @@ nextflow_workflow { test("Parameters: run_virus_identification = true") { - when { - params { - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' - keep_phix = true - skip_clipping = true - skip_adapter_trimming = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - coassemble_group = true - megahit_fix_cpu_1 = true - skip_spades = true - skip_spadeshybrid = true - skip_quast = true - run_virus_identification = true - genomad_splits = 7 - skip_prodigal = true - skip_binning = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true - skip_metaeuk = true - } - } - then { assertAll( { assert workflow.success }, From 6080045cb730b73660dcf029493a0f7837fe48e8 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Wed, 14 Feb 2024 09:39:49 -0800 Subject: [PATCH 47/49] Added config file to each nf-test file --- .github/workflows/ci.yml | 2 +- workflows/mag/tests/test.nf.test | 1 + workflows/mag/tests/test_adapterremoval.nf.test | 1 + workflows/mag/tests/test_ancient_dna.nf.test | 1 + workflows/mag/tests/test_bbnorm.nf.test | 1 + workflows/mag/tests/test_binrefinement.nf.test | 1 + workflows/mag/tests/test_busco_auto.nf.test | 1 + workflows/mag/tests/test_host_rm.nf.test | 1 + workflows/mag/tests/test_hybrid.nf.test | 1 + workflows/mag/tests/test_hybrid_host_rm.nf.test | 1 + workflows/mag/tests/test_nothing.nf.test | 1 + workflows/mag/tests/test_virus_identification.nf.test | 1 + 12 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92bb2ad6..9034d1db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,7 @@ jobs: - name: Run nf-test run: | - nf-test test --verbose --tag ${{ matrix.tags }} --profile "${{ matrix.profile }}",${{ matrix.tags }} --junitxml=test.xml --tap=test.tap + nf-test test --verbose --tag ${{ matrix.tags }} --profile "${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap - uses: pcolby/tap-summary@v1 with: diff --git a/workflows/mag/tests/test.nf.test b/workflows/mag/tests/test.nf.test index b90b09b2..b9a201f8 100644 --- a/workflows/mag/tests/test.nf.test +++ b/workflows/mag/tests/test.nf.test @@ -6,6 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test" + config "../../../conf/mag_test.config" test("Default paramters") { diff --git a/workflows/mag/tests/test_adapterremoval.nf.test b/workflows/mag/tests/test_adapterremoval.nf.test index 9d05d884..087f9945 100644 --- a/workflows/mag/tests/test_adapterremoval.nf.test +++ b/workflows/mag/tests/test_adapterremoval.nf.test @@ -6,6 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_adapterremoval" + config "../../../conf/mag_test_adapterremoval.config" test("Parameters: clip_tool = 'adapterremoval'") { diff --git a/workflows/mag/tests/test_ancient_dna.nf.test b/workflows/mag/tests/test_ancient_dna.nf.test index b0023c14..433305e4 100644 --- a/workflows/mag/tests/test_ancient_dna.nf.test +++ b/workflows/mag/tests/test_ancient_dna.nf.test @@ -6,6 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_ancient_dna" + config "../../../conf/mag_test_ancient_dna.config" test("Parameters: ancient_dna = true") { diff --git a/workflows/mag/tests/test_bbnorm.nf.test b/workflows/mag/tests/test_bbnorm.nf.test index 5a7dd6ba..c1aa50be 100644 --- a/workflows/mag/tests/test_bbnorm.nf.test +++ b/workflows/mag/tests/test_bbnorm.nf.test @@ -6,6 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_bbnorm" + config "../../../conf/mag_test_bbnorm.config" test("Parameters: bbnorm = true") { diff --git a/workflows/mag/tests/test_binrefinement.nf.test b/workflows/mag/tests/test_binrefinement.nf.test index 277b06e5..074f9902 100644 --- a/workflows/mag/tests/test_binrefinement.nf.test +++ b/workflows/mag/tests/test_binrefinement.nf.test @@ -6,6 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_binrefinement" + config "../../../conf/mag_test_binrefinement.config" test("Parameters: refine_bins_dastool = true") { diff --git a/workflows/mag/tests/test_busco_auto.nf.test b/workflows/mag/tests/test_busco_auto.nf.test index 5fb41543..6c752bb3 100644 --- a/workflows/mag/tests/test_busco_auto.nf.test +++ b/workflows/mag/tests/test_busco_auto.nf.test @@ -6,6 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_busco_auto" + config "../../../conf/mag_test_binrefinement.config" test("Parameters: skip_busco = false") { diff --git a/workflows/mag/tests/test_host_rm.nf.test b/workflows/mag/tests/test_host_rm.nf.test index 83ff9862..68ed2ae5 100644 --- a/workflows/mag/tests/test_host_rm.nf.test +++ b/workflows/mag/tests/test_host_rm.nf.test @@ -6,6 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_host_rm" + config "../../../conf/mag_test_host_rm.config" test("Parameters: host_fasta != null") { diff --git a/workflows/mag/tests/test_hybrid.nf.test b/workflows/mag/tests/test_hybrid.nf.test index 424bd4af..397e7535 100644 --- a/workflows/mag/tests/test_hybrid.nf.test +++ b/workflows/mag/tests/test_hybrid.nf.test @@ -6,6 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_hybrid" + config "../../../conf/mag_test_hybrid.config" test("Parameters: long reads included") { diff --git a/workflows/mag/tests/test_hybrid_host_rm.nf.test b/workflows/mag/tests/test_hybrid_host_rm.nf.test index ffa38654..e305c551 100644 --- a/workflows/mag/tests/test_hybrid_host_rm.nf.test +++ b/workflows/mag/tests/test_hybrid_host_rm.nf.test @@ -6,6 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_hybrid_host_rm" + config "../../../conf/mag_test_hybrid_host_rm.config" test("Parameters: long reads & host removal included") { diff --git a/workflows/mag/tests/test_nothing.nf.test b/workflows/mag/tests/test_nothing.nf.test index 7b94efbe..d4aac149 100644 --- a/workflows/mag/tests/test_nothing.nf.test +++ b/workflows/mag/tests/test_nothing.nf.test @@ -6,6 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_nothing" + config "../../../conf/mag_test_nothing.config" test("Parameters: test_nothing") { diff --git a/workflows/mag/tests/test_virus_identification.nf.test b/workflows/mag/tests/test_virus_identification.nf.test index 657394e7..5b2bbf8d 100644 --- a/workflows/mag/tests/test_virus_identification.nf.test +++ b/workflows/mag/tests/test_virus_identification.nf.test @@ -6,6 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_virus_identification" + config "../../../conf/mag_test_virus_identification.config" test("Parameters: run_virus_identification = true") { From e625dee7d94cdf40b84faf4b456ed5884f0b5504 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Wed, 14 Feb 2024 10:44:02 -0800 Subject: [PATCH 48/49] Renamed test configs and included profiles in nf-tests --- .github/workflows/ci.yml | 2 +- conf/{mag_test.config => test.config} | 0 ...oval.config => test_adapterremoval.config} | 0 ...ent_dna.config => test_ancient_dna.config} | 0 ..._test_bbnorm.config => test_bbnorm.config} | 0 ...ement.config => test_binrefinement.config} | 0 ...sco_auto.config => test_busco_auto.config} | 0 ...est_host_rm.config => test_host_rm.config} | 0 ..._test_hybrid.config => test_hybrid.config} | 0 ...t_rm.config => test_hybrid_host_rm.config} | 0 ...est_nothing.config => test_nothing.config} | 0 conf/test_samplesheet.config | 41 ------------------- ...onfig => test_virus_identification.config} | 0 modules/nf-core/genomad/download/main.nf | 2 + nextflow.config | 28 ++++++------- workflows/mag/tests/test.nf.test | 2 +- .../mag/tests/test_adapterremoval.nf.test | 2 +- workflows/mag/tests/test_ancient_dna.nf.test | 2 +- workflows/mag/tests/test_bbnorm.nf.test | 2 +- .../mag/tests/test_binrefinement.nf.test | 2 +- workflows/mag/tests/test_busco_auto.nf.test | 2 +- workflows/mag/tests/test_host_rm.nf.test | 2 +- workflows/mag/tests/test_hybrid.nf.test | 2 +- .../mag/tests/test_hybrid_host_rm.nf.test | 2 +- workflows/mag/tests/test_nothing.nf.test | 2 +- .../tests/test_virus_identification.nf.test | 2 +- 26 files changed, 28 insertions(+), 67 deletions(-) rename conf/{mag_test.config => test.config} (100%) rename conf/{mag_test_adapterremoval.config => test_adapterremoval.config} (100%) rename conf/{mag_test_ancient_dna.config => test_ancient_dna.config} (100%) rename conf/{mag_test_bbnorm.config => test_bbnorm.config} (100%) rename conf/{mag_test_binrefinement.config => test_binrefinement.config} (100%) rename conf/{mag_test_busco_auto.config => test_busco_auto.config} (100%) rename conf/{mag_test_host_rm.config => test_host_rm.config} (100%) rename conf/{mag_test_hybrid.config => test_hybrid.config} (100%) rename conf/{mag_test_hybrid_host_rm.config => test_hybrid_host_rm.config} (100%) rename conf/{mag_test_nothing.config => test_nothing.config} (100%) delete mode 100644 conf/test_samplesheet.config rename conf/{mag_test_virus_identification.config => test_virus_identification.config} (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9034d1db..18e9251c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,7 @@ jobs: - name: Run nf-test run: | - nf-test test --verbose --tag ${{ matrix.tags }} --profile "${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap + nf-test test --verbose --tag ${{ matrix.tags }} --profile +"${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap - uses: pcolby/tap-summary@v1 with: diff --git a/conf/mag_test.config b/conf/test.config similarity index 100% rename from conf/mag_test.config rename to conf/test.config diff --git a/conf/mag_test_adapterremoval.config b/conf/test_adapterremoval.config similarity index 100% rename from conf/mag_test_adapterremoval.config rename to conf/test_adapterremoval.config diff --git a/conf/mag_test_ancient_dna.config b/conf/test_ancient_dna.config similarity index 100% rename from conf/mag_test_ancient_dna.config rename to conf/test_ancient_dna.config diff --git a/conf/mag_test_bbnorm.config b/conf/test_bbnorm.config similarity index 100% rename from conf/mag_test_bbnorm.config rename to conf/test_bbnorm.config diff --git a/conf/mag_test_binrefinement.config b/conf/test_binrefinement.config similarity index 100% rename from conf/mag_test_binrefinement.config rename to conf/test_binrefinement.config diff --git a/conf/mag_test_busco_auto.config b/conf/test_busco_auto.config similarity index 100% rename from conf/mag_test_busco_auto.config rename to conf/test_busco_auto.config diff --git a/conf/mag_test_host_rm.config b/conf/test_host_rm.config similarity index 100% rename from conf/mag_test_host_rm.config rename to conf/test_host_rm.config diff --git a/conf/mag_test_hybrid.config b/conf/test_hybrid.config similarity index 100% rename from conf/mag_test_hybrid.config rename to conf/test_hybrid.config diff --git a/conf/mag_test_hybrid_host_rm.config b/conf/test_hybrid_host_rm.config similarity index 100% rename from conf/mag_test_hybrid_host_rm.config rename to conf/test_hybrid_host_rm.config diff --git a/conf/mag_test_nothing.config b/conf/test_nothing.config similarity index 100% rename from conf/mag_test_nothing.config rename to conf/test_nothing.config diff --git a/conf/test_samplesheet.config b/conf/test_samplesheet.config deleted file mode 100644 index d36c87fe..00000000 --- a/conf/test_samplesheet.config +++ /dev/null @@ -1,41 +0,0 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Nextflow config file for running minimal tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run nf-core/mag -profile test, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Test profile' - config_profile_description = 'Minimal test dataset to check pipeline function' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/mag/samplesheets/samplesheet.multirun.csv' - generate_downstream_samplesheet = 'phageannotator' - skip_clipping = true - skip_adapter_trimming = true - keep_phix = true - centrifuge_db = null - kraken2_db = null - skip_krona = true - coassemble_group = true - megahit_fix_cpu_1 = true - skip_spadeshybrid = true - skip_spades = true - skip_quast = true - skip_prodigal = true - skip_binning = true - skip_binqc = true - skip_gtdbtk = true - skip_prokka = true -} diff --git a/conf/mag_test_virus_identification.config b/conf/test_virus_identification.config similarity index 100% rename from conf/mag_test_virus_identification.config rename to conf/test_virus_identification.config diff --git a/modules/nf-core/genomad/download/main.nf b/modules/nf-core/genomad/download/main.nf index a2ac6ecb..632d494e 100644 --- a/modules/nf-core/genomad/download/main.nf +++ b/modules/nf-core/genomad/download/main.nf @@ -16,6 +16,8 @@ process GENOMAD_DOWNLOAD { script: def args = task.ext.args ?: '' """ + https_proxy=http://klone-dip1-A-ib:3128 + export https_proxy genomad \\ download-database . diff --git a/nextflow.config b/nextflow.config index dae04616..6941d2ef 100644 --- a/nextflow.config +++ b/nextflow.config @@ -305,20 +305,20 @@ profiles { executor.cpus = 4 executor.memory = 8.GB } - mag_test { includeConfig 'conf/mag_test.config' } - mag_test_full { includeConfig 'conf/mag_test_full.config' } - mag_test_host_rm { includeConfig 'conf/mag_test_host_rm.config' } - mag_test_hybrid { includeConfig 'conf/mag_test_hybrid.config' } - mag_test_hybrid_host_rm { includeConfig 'conf/mag_test_hybrid_host_rm.config' } - mag_test_busco_auto { includeConfig 'conf/mag_test_busco_auto.config' } - mag_test_ancient_dna { includeConfig 'conf/mag_test_ancient_dna.config' } - mag_test_adapterremoval { includeConfig 'conf/mag_test_adapterremoval.config' } - mag_test_binning_entry { includeConfig 'conf/mag_test_binning_entry.config' } - mag_test_binrefinement { includeConfig 'conf/mag_test_binrefinement.config' } - mag_test_no_clipping { includeConfig 'conf/mag_test_no_clipping.config' } - mag_test_bbnorm { includeConfig 'conf/mag_test_bbnorm.config' } - mag_test_nothing { includeConfig 'conf/mag_test_nothing.config' } - mag_test_virus_identification { includeConfig 'conf/mag_test_virus_identification.config' } + test { includeConfig 'conf/test.config' } + test_full { includeConfig 'conf/test_full.config' } + test_host_rm { includeConfig 'conf/test_host_rm.config' } + test_hybrid { includeConfig 'conf/test_hybrid.config' } + test_hybrid_host_rm { includeConfig 'conf/test_hybrid_host_rm.config' } + test_busco_auto { includeConfig 'conf/test_busco_auto.config' } + test_ancient_dna { includeConfig 'conf/test_ancient_dna.config' } + test_adapterremoval { includeConfig 'conf/test_adapterremoval.config' } + test_binning_entry { includeConfig 'conf/test_binning_entry.config' } + test_binrefinement { includeConfig 'conf/test_binrefinement.config' } + test_no_clipping { includeConfig 'conf/test_no_clipping.config' } + test_bbnorm { includeConfig 'conf/test_bbnorm.config' } + test_nothing { includeConfig 'conf/test_nothing.config' } + test_virus_identification { includeConfig 'conf/test_virus_identification.config' } } // Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile diff --git a/workflows/mag/tests/test.nf.test b/workflows/mag/tests/test.nf.test index b9a201f8..3c5b28ff 100644 --- a/workflows/mag/tests/test.nf.test +++ b/workflows/mag/tests/test.nf.test @@ -6,7 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test" - config "../../../conf/mag_test.config" + profile "test" test("Default paramters") { diff --git a/workflows/mag/tests/test_adapterremoval.nf.test b/workflows/mag/tests/test_adapterremoval.nf.test index 087f9945..68fd8a2e 100644 --- a/workflows/mag/tests/test_adapterremoval.nf.test +++ b/workflows/mag/tests/test_adapterremoval.nf.test @@ -6,7 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_adapterremoval" - config "../../../conf/mag_test_adapterremoval.config" + profile "test_adapterremoval" test("Parameters: clip_tool = 'adapterremoval'") { diff --git a/workflows/mag/tests/test_ancient_dna.nf.test b/workflows/mag/tests/test_ancient_dna.nf.test index 433305e4..8120d57f 100644 --- a/workflows/mag/tests/test_ancient_dna.nf.test +++ b/workflows/mag/tests/test_ancient_dna.nf.test @@ -6,7 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_ancient_dna" - config "../../../conf/mag_test_ancient_dna.config" + profile "test_ancient_dna" test("Parameters: ancient_dna = true") { diff --git a/workflows/mag/tests/test_bbnorm.nf.test b/workflows/mag/tests/test_bbnorm.nf.test index c1aa50be..2dce5682 100644 --- a/workflows/mag/tests/test_bbnorm.nf.test +++ b/workflows/mag/tests/test_bbnorm.nf.test @@ -6,7 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_bbnorm" - config "../../../conf/mag_test_bbnorm.config" + profile "test_bbnorm" test("Parameters: bbnorm = true") { diff --git a/workflows/mag/tests/test_binrefinement.nf.test b/workflows/mag/tests/test_binrefinement.nf.test index 074f9902..fc08a808 100644 --- a/workflows/mag/tests/test_binrefinement.nf.test +++ b/workflows/mag/tests/test_binrefinement.nf.test @@ -6,7 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_binrefinement" - config "../../../conf/mag_test_binrefinement.config" + profile "test_binrefinement" test("Parameters: refine_bins_dastool = true") { diff --git a/workflows/mag/tests/test_busco_auto.nf.test b/workflows/mag/tests/test_busco_auto.nf.test index 6c752bb3..f1e0162b 100644 --- a/workflows/mag/tests/test_busco_auto.nf.test +++ b/workflows/mag/tests/test_busco_auto.nf.test @@ -6,7 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_busco_auto" - config "../../../conf/mag_test_binrefinement.config" + profile "test_busco_auto" test("Parameters: skip_busco = false") { diff --git a/workflows/mag/tests/test_host_rm.nf.test b/workflows/mag/tests/test_host_rm.nf.test index 68ed2ae5..5174204b 100644 --- a/workflows/mag/tests/test_host_rm.nf.test +++ b/workflows/mag/tests/test_host_rm.nf.test @@ -6,7 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_host_rm" - config "../../../conf/mag_test_host_rm.config" + profile "test_host_rm" test("Parameters: host_fasta != null") { diff --git a/workflows/mag/tests/test_hybrid.nf.test b/workflows/mag/tests/test_hybrid.nf.test index 397e7535..4666b6ca 100644 --- a/workflows/mag/tests/test_hybrid.nf.test +++ b/workflows/mag/tests/test_hybrid.nf.test @@ -6,7 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_hybrid" - config "../../../conf/mag_test_hybrid.config" + profile "test_hybrid" test("Parameters: long reads included") { diff --git a/workflows/mag/tests/test_hybrid_host_rm.nf.test b/workflows/mag/tests/test_hybrid_host_rm.nf.test index e305c551..d642a8c4 100644 --- a/workflows/mag/tests/test_hybrid_host_rm.nf.test +++ b/workflows/mag/tests/test_hybrid_host_rm.nf.test @@ -6,7 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_hybrid_host_rm" - config "../../../conf/mag_test_hybrid_host_rm.config" + profile "test_hybrid_host_rm" test("Parameters: long reads & host removal included") { diff --git a/workflows/mag/tests/test_nothing.nf.test b/workflows/mag/tests/test_nothing.nf.test index d4aac149..fd50c27f 100644 --- a/workflows/mag/tests/test_nothing.nf.test +++ b/workflows/mag/tests/test_nothing.nf.test @@ -6,7 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_nothing" - config "../../../conf/mag_test_nothing.config" + profile "test_nothing" test("Parameters: test_nothing") { diff --git a/workflows/mag/tests/test_virus_identification.nf.test b/workflows/mag/tests/test_virus_identification.nf.test index 5b2bbf8d..bfc1c054 100644 --- a/workflows/mag/tests/test_virus_identification.nf.test +++ b/workflows/mag/tests/test_virus_identification.nf.test @@ -6,7 +6,7 @@ nextflow_workflow { tag "workflows" tag "mag" tag "mag_test_virus_identification" - config "../../../conf/mag_test_virus_identification.config" + profile "test_virus_identification" test("Parameters: run_virus_identification = true") { From 41c682a4a91d82bf7b59de9880bc539ae18bdc55 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Wed, 14 Feb 2024 11:06:07 -0800 Subject: [PATCH 49/49] Removed local proxy code --- modules/nf-core/genomad/download/main.nf | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/nf-core/genomad/download/main.nf b/modules/nf-core/genomad/download/main.nf index 632d494e..a2ac6ecb 100644 --- a/modules/nf-core/genomad/download/main.nf +++ b/modules/nf-core/genomad/download/main.nf @@ -16,8 +16,6 @@ process GENOMAD_DOWNLOAD { script: def args = task.ext.args ?: '' """ - https_proxy=http://klone-dip1-A-ib:3128 - export https_proxy genomad \\ download-database .
Process Name \\", + " \\ Software Version
CUSTOM_DUMPSOFTWAREVERSIONSpython3.11.7
yaml5.4.1
TOOL1tool10.11.9
TOOL2tool21.9
WorkflowNextflow
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls