From 68ab301209184ce1e04631d3b24527d150f6e5a5 Mon Sep 17 00:00:00 2001 From: Jover Lee Date: Fri, 10 Jan 2025 13:47:40 -0800 Subject: [PATCH 1/2] Limit concurrent fauna connections to 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The latest upload workflow¹ seemed to overwhelm the rethinkdb server so limiting concurrent fauna connections to see if it would prevent the OOM error. Use of Snakemake's `workflow.global_resources` inspired by the ncov use to limit concurrent deploy jobs.² Doing this instead of limiting the overall number of Snakemake jobs with `-j` because there are other rules in the workflow that can run without connections to fauna. ¹ ² --- workflow/snakemake_rules/download_from_fauna.smk | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/workflow/snakemake_rules/download_from_fauna.smk b/workflow/snakemake_rules/download_from_fauna.smk index 64e83501..6bd8912a 100644 --- a/workflow/snakemake_rules/download_from_fauna.smk +++ b/workflow/snakemake_rules/download_from_fauna.smk @@ -9,6 +9,12 @@ titers = "data/{lineage}/{center}_{passage}_{assay}_titers.tsv" ''' +# Limit the number of concurrent fauna connections so that we are less likely +# to overwhelm the rethinkdb server. +# Inspired by the ncov's limit on concurrent deploy jobs +# +workflow.global_resources.setdefault("concurrent_fauna", 2) + # fields that will be canonicized by augur parse (upper/lower casing etc) path_to_fauna = '../fauna' @@ -47,6 +53,8 @@ rule download_sequences: sequences = "data/{lineage}/raw_{segment}.fasta" params: fasta_fields = config["fauna_fasta_fields"], + resources: + concurrent_fauna = 1 conda: "../envs/nextstrain.yaml" benchmark: "benchmarks/download_sequences_{lineage}_{segment}.txt" @@ -71,6 +79,8 @@ rule download_titers: dbs = _get_tdb_databases, assays = _get_tdb_assays, virus_passage_category=_get_virus_passage_category, + resources: + concurrent_fauna = 1 conda: "../envs/nextstrain.yaml" benchmark: "benchmarks/download_titers_{lineage}_{center}_{passage}_{assay}.txt" From 123716a48d60eb7c00e447a08434d874881fd813 Mon Sep 17 00:00:00 2001 From: Jover Lee Date: Fri, 10 Jan 2025 16:21:38 -0800 Subject: [PATCH 2/2] Remove `message` directive from download_sequences --- workflow/snakemake_rules/download_from_fauna.smk | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/snakemake_rules/download_from_fauna.smk b/workflow/snakemake_rules/download_from_fauna.smk index 6bd8912a..b6218505 100644 --- a/workflow/snakemake_rules/download_from_fauna.smk +++ b/workflow/snakemake_rules/download_from_fauna.smk @@ -48,7 +48,6 @@ def _get_virus_passage_category(wildcards): return "" rule download_sequences: - message: "Downloading sequences from fauna" output: sequences = "data/{lineage}/raw_{segment}.fasta" params: