From 411af3aced6a1775bc35d1a5a45cea23f3737604 Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Fri, 9 Dec 2022 15:07:16 -0500 Subject: [PATCH 1/6] set default GPU to K80->P4 in BEAST task; parameterize beagle_order --- pipes/WDL/tasks/tasks_interhost.wdl | 31 +++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/pipes/WDL/tasks/tasks_interhost.wdl b/pipes/WDL/tasks/tasks_interhost.wdl index 7e51f28bc..99e4bd529 100644 --- a/pipes/WDL/tasks/tasks_interhost.wdl +++ b/pipes/WDL/tasks/tasks_interhost.wdl @@ -96,6 +96,7 @@ task multi_align_mafft { task beast { input { File beauti_xml + String beagle_order="1,2,3,4" String? accelerator_type Int? accelerator_count @@ -106,6 +107,21 @@ task beast { String docker = "quay.io/broadinstitute/beast-beagle-cuda:1.10.5pre" } + meta { + description: "Execute GPU-accelerated BEAST. For tips on performance, see https://beast.community/performance#gpu" + } + parameter_meta { + beagle_order: { + description: "The order of CPU(0) and GPU(1+) resources used to process partitioned data." + } + accelerator_type: { + description: "The model of GPU to use. For availability and pricing on GCP, see https://cloud.google.com/compute/gpus-pricing#gpus" + } + accelerator_count: { + description: "The number of GPUs of the specified type to use." + } + } + Int disk_size = 300 Int boot_disk = 50 Int disk_size_az = disk_size + boot_disk @@ -119,10 +135,13 @@ task beast { bash -c "sleep 60; nvidia-smi" & beast \ -beagle_multipartition off \ - -beagle_GPU -beagle_cuda -beagle_SSE \ - -beagle_double -beagle_scaling always \ - -beagle_order 1,2,3,4 \ - ${beauti_xml} + -beagle_GPU \ + -beagle_cuda \ + -beagle_SSE \ + -beagle_double \ + -beagle_scaling always \ + ~{'-beagle_order ' + beagle_order} \ + ~{beauti_xml} } output { @@ -143,9 +162,9 @@ task beast { gpu: true # dxWDL dx_timeout: "40H" # dxWDL dx_instance_type: "mem1_ssd1_gpu2_x8" # dxWDL - acceleratorType: select_first([accelerator_type, "nvidia-tesla-k80"]) # GCP PAPIv2 + acceleratorType: select_first([accelerator_type, "nvidia-tesla-p4"]) # GCP PAPIv2 acceleratorCount: select_first([accelerator_count, 4]) # GCP PAPIv2 - gpuType: select_first([gpu_type, "nvidia-tesla-k80"]) # Terra + gpuType: select_first([gpu_type, "nvidia-tesla-p4"]) # Terra gpuCount: select_first([gpu_count, 4]) # Terra nvidiaDriverVersion: "410.79" } From 4eb59f05ced2d4dd496098fdb56c1823c96325ab Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Fri, 9 Dec 2022 18:18:12 -0500 Subject: [PATCH 2/6] expose bool to set single/double precision for beagle; set beagle_order based on GPU count if not otherwise specified expose bool to set single/double precision for beagle; generate+set beagle_order string based on GPU count if not otherwise specified --- pipes/WDL/tasks/tasks_interhost.wdl | 37 ++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/pipes/WDL/tasks/tasks_interhost.wdl b/pipes/WDL/tasks/tasks_interhost.wdl index 99e4bd529..672587a17 100644 --- a/pipes/WDL/tasks/tasks_interhost.wdl +++ b/pipes/WDL/tasks/tasks_interhost.wdl @@ -96,7 +96,9 @@ task multi_align_mafft { task beast { input { File beauti_xml - String beagle_order="1,2,3,4" + + Boolean beagle_double_precision=true + String? beagle_order String? accelerator_type Int? accelerator_count @@ -111,34 +113,51 @@ task beast { description: "Execute GPU-accelerated BEAST. For tips on performance, see https://beast.community/performance#gpu" } parameter_meta { + beagle_double_precision: { + description: "If beagle_double_precision=true, use double-precision calculation (perhaps set to false to gain execution speed if MCMC chain convergence is possible using single-precision calculation)." + } beagle_order: { description: "The order of CPU(0) and GPU(1+) resources used to process partitioned data." } + accelerator_type: { - description: "The model of GPU to use. For availability and pricing on GCP, see https://cloud.google.com/compute/gpus-pricing#gpus" + description: "[GCP] The model of GPU to use. For availability and pricing on GCP, see https://cloud.google.com/compute/gpus-pricing#gpus" } accelerator_count: { - description: "The number of GPUs of the specified type to use." + description: "[GCP] The number of GPUs of the specified type to use." + } + gpu_type: { + description: "[Terra] The model of GPU to use. For availability and pricing on GCP, see https://support.terra.bio/hc/en-us/articles/4403006001947-Getting-started-with-GPUs-in-a-Cloud-Environment" + } + gpu_count: { + description: "[Terra] The number of GPUs of the specified type to use." } } - Int disk_size = 300 - Int boot_disk = 50 + Int disk_size = 300 + Int boot_disk = 50 Int disk_size_az = disk_size + boot_disk - # TO DO: parameterize gpuType and gpuCount + # platform-agnostic number of GPUs we're actually using + Int gpu_count_used = ${default=${default=1 accelerator_count} gpu_count} command { set -e beast -beagle_info nvidia-smi + + # if beagle_order is not specified by the user, + # create an appropriate string based on the gpu count + beagle_order="~{default=$(echo $(echo {1..~{gpu_count_used}}| tr ' ' ',')) beagle_order}" + echo "beagle_order: $beagle_order" + bash -c "sleep 60; nvidia-smi" & beast \ -beagle_multipartition off \ -beagle_GPU \ -beagle_cuda \ -beagle_SSE \ - -beagle_double \ + ~{true="-beagle_double" false="-beagle_single" beagle_double_precision} \ -beagle_scaling always \ ~{'-beagle_order ' + beagle_order} \ ~{beauti_xml} @@ -163,9 +182,9 @@ task beast { dx_timeout: "40H" # dxWDL dx_instance_type: "mem1_ssd1_gpu2_x8" # dxWDL acceleratorType: select_first([accelerator_type, "nvidia-tesla-p4"]) # GCP PAPIv2 - acceleratorCount: select_first([accelerator_count, 4]) # GCP PAPIv2 + acceleratorCount: select_first([accelerator_count, 1]) # GCP PAPIv2 gpuType: select_first([gpu_type, "nvidia-tesla-p4"]) # Terra - gpuCount: select_first([gpu_count, 4]) # Terra + gpuCount: select_first([gpu_count, 1]) # Terra nvidiaDriverVersion: "410.79" } } From 92133c3c04a2b5d528202247df21e25eceef17a1 Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Fri, 9 Dec 2022 18:23:13 -0500 Subject: [PATCH 3/6] use select_first() outside command block rather than nested WDL substitutions --- pipes/WDL/tasks/tasks_interhost.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipes/WDL/tasks/tasks_interhost.wdl b/pipes/WDL/tasks/tasks_interhost.wdl index 672587a17..7bcf2577a 100644 --- a/pipes/WDL/tasks/tasks_interhost.wdl +++ b/pipes/WDL/tasks/tasks_interhost.wdl @@ -139,7 +139,7 @@ task beast { Int disk_size_az = disk_size + boot_disk # platform-agnostic number of GPUs we're actually using - Int gpu_count_used = ${default=${default=1 accelerator_count} gpu_count} + Int gpu_count_used = select_first([accelerator_count, gpu_count, 1]) command { set -e From e6cace1a1b12be70ff867fbba2dce3ef2fa16da1 Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Fri, 9 Dec 2022 18:27:07 -0500 Subject: [PATCH 4/6] refactor for more clear WDL vs shell scope --- pipes/WDL/tasks/tasks_interhost.wdl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipes/WDL/tasks/tasks_interhost.wdl b/pipes/WDL/tasks/tasks_interhost.wdl index 7bcf2577a..bbef93063 100644 --- a/pipes/WDL/tasks/tasks_interhost.wdl +++ b/pipes/WDL/tasks/tasks_interhost.wdl @@ -148,7 +148,8 @@ task beast { # if beagle_order is not specified by the user, # create an appropriate string based on the gpu count - beagle_order="~{default=$(echo $(echo {1..~{gpu_count_used}}| tr ' ' ',')) beagle_order}" + default_beagle_order=$(echo $(echo {1..~{gpu_count_used}}| tr ' ' ',')) + beagle_order=~{default="$default_beagle_order" beagle_order} echo "beagle_order: $beagle_order" bash -c "sleep 60; nvidia-smi" & From faf280ec532555c6a1823da23ead4e8ae0e272d5 Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Fri, 9 Dec 2022 18:33:44 -0500 Subject: [PATCH 5/6] WDL check changes --- pipes/WDL/tasks/tasks_interhost.wdl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipes/WDL/tasks/tasks_interhost.wdl b/pipes/WDL/tasks/tasks_interhost.wdl index bbef93063..930411b8b 100644 --- a/pipes/WDL/tasks/tasks_interhost.wdl +++ b/pipes/WDL/tasks/tasks_interhost.wdl @@ -148,7 +148,8 @@ task beast { # if beagle_order is not specified by the user, # create an appropriate string based on the gpu count - default_beagle_order=$(echo $(echo {1..~{gpu_count_used}}| tr ' ' ',')) + + default_beagle_order="$(echo $(echo {1..~{gpu_count_used}} | tr ' ' ','))" beagle_order=~{default="$default_beagle_order" beagle_order} echo "beagle_order: $beagle_order" From 8bce17ef510b2a935cdf867347741bdc86195294 Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Fri, 9 Dec 2022 18:45:11 -0500 Subject: [PATCH 6/6] omit bash {1..N} range nonsense for building beagle order string, since we don't need to worry about macOS and its quirky BSD interpretation of 'seq' -s --- pipes/WDL/tasks/tasks_interhost.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipes/WDL/tasks/tasks_interhost.wdl b/pipes/WDL/tasks/tasks_interhost.wdl index 930411b8b..36ed8f796 100644 --- a/pipes/WDL/tasks/tasks_interhost.wdl +++ b/pipes/WDL/tasks/tasks_interhost.wdl @@ -149,7 +149,7 @@ task beast { # if beagle_order is not specified by the user, # create an appropriate string based on the gpu count - default_beagle_order="$(echo $(echo {1..~{gpu_count_used}} | tr ' ' ','))" + default_beagle_order="$(seq -s, ~{gpu_count_used})" beagle_order=~{default="$default_beagle_order" beagle_order} echo "beagle_order: $beagle_order"