Merge branch 'master' into ct-lofreq-all-pos

broadinstitute · Dec 15, 2023 · 1a88c00 · 1a88c00
2 parents 91e7df9 + d659825
commit 1a88c00
Show file tree

Hide file tree

Showing 103 changed files with 4,713 additions and 2,126 deletions.
diff --git a/.dockstore.yml b/.dockstore.yml
@@ -50,6 +50,11 @@ workflows:
     primaryDescriptorPath: /pipes/WDL/workflows/augur_from_msa.wdl
     testParameterFiles:
       - empty.json
+  - name: augur_from_msa_with_subsampler
+    subclass: WDL
+    primaryDescriptorPath: /pipes/WDL/workflows/augur_from_msa_with_subsampler.wdl
+    testParameterFiles:
+      - empty.json
   - name: bams_multiqc
     subclass: WDL
     primaryDescriptorPath: /pipes/WDL/workflows/bams_multiqc.wdl
@@ -139,6 +144,11 @@ workflows:
     primaryDescriptorPath: /pipes/WDL/workflows/downsample.wdl
     testParameterFiles:
       - empty.json
+  - name: dump_gcloud_env_info
+    subclass: WDL
+    primaryDescriptorPath: /pipes/WDL/workflows/dump_gcloud_env_info.wdl
+    testParameterFiles:
+      - empty.json
   - name: fastq_to_ubam
     subclass: WDL
     primaryDescriptorPath: /pipes/WDL/workflows/fastq_to_ubam.wdl
@@ -234,11 +244,21 @@ workflows:
     primaryDescriptorPath: /pipes/WDL/workflows/merge_vcfs.wdl
     testParameterFiles:
       - empty.json
+  - name: metagenomic_denovo
+    subclass: WDL
+    primaryDescriptorPath: /pipes/WDL/workflows/metagenomic_denovo.wdl
+    testParameterFiles:
+      - empty.json
   - name: multiqc_only
     subclass: WDL
     primaryDescriptorPath: /pipes/WDL/workflows/multiqc_only.wdl
     testParameterFiles:
       - empty.json
+  - name: nextclade_single
+    subclass: WDL
+    primaryDescriptorPath: /pipes/WDL/workflows/nextclade_single.wdl
+    testParameterFiles:
+      - empty.json
   - name: sarscov2_batch_relineage
     subclass: WDL
     primaryDescriptorPath: /pipes/WDL/workflows/sarscov2_batch_relineage.wdl
@@ -273,111 +293,17 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /pipes/WDL/workflows/sarscov2_lineages.wdl
     testParameterFiles:
-  - name: isnvs_merge_to_vcf
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/isnvs_merge_to_vcf.wdl
-    testParameterFiles:
-      - empty.json
-  - name: isnvs_one_sample
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/isnvs_one_sample.wdl
-    testParameterFiles:
-      - empty.json
-  - name: kraken2_build
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/kraken2_build.wdl
-    testParameterFiles:
-      - empty.json
-  - name: mafft_and_snp
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/mafft_and_snp.wdl
-    testParameterFiles:
-      - empty.json
-  - name: mafft_and_snp_annotated
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/mafft_and_snp_annotated.wdl
-    testParameterFiles:
-      - empty.json
-  - name: mafft_and_trim
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/mafft_and_trim.wdl
-    testParameterFiles:
-      - empty.json
-  - name: mafft
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/mafft.wdl
-    testParameterFiles:
-      - empty.json
-  - name: merge_bams
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/merge_bams.wdl
-    testParameterFiles:
-      - empty.json
-  - name: merge_metagenomics
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/merge_metagenomics.wdl
-    testParameterFiles:
-      - empty.json
-  - name: merge_tar_chunks
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/merge_tar_chunks.wdl
-    testParameterFiles:
-      - empty.json
-  - name: merge_vcfs_and_annotate
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/merge_vcfs_and_annotate.wdl
-    testParameterFiles:
-      - empty.json
-  - name: merge_vcfs
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/merge_vcfs.wdl
-    testParameterFiles:
-      - empty.json
-  - name: multiqc_only
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/multiqc_only.wdl
-    testParameterFiles:
-      - empty.json
-  - name: sarscov2_batch_relineage
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/sarscov2_batch_relineage.wdl
-    testParameterFiles:
-      - empty.json
-  - name: sarscov2_biosample_load
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/sarscov2_biosample_load.wdl
-    testParameterFiles:
-      - empty.json
-  - name: sarscov2_data_release
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/sarscov2_data_release.wdl
-    testParameterFiles:
-      - empty.json
-  - name: sarscov2_genbank
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/sarscov2_genbank.wdl
-    testParameterFiles:
-      - empty.json
-  - name: sarscov2_genbank_ingest
+    - /test/input/WDL/test_inputs-sarscov2_lineages-local.json
+  - name: calc_bam_read_depths
     subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/sarscov2_genbank_ingest.wdl
+    primaryDescriptorPath: /pipes/WDL/workflows/calc_bam_read_depths.wdl
     testParameterFiles:
       - empty.json
   - name: sarscov2_gisaid_ingest
     subclass: WDL
     primaryDescriptorPath: /pipes/WDL/workflows/sarscov2_gisaid_ingest.wdl
     testParameterFiles:
       - empty.json
-  - name: sarscov2_illumina_full
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/sarscov2_illumina_full.wdl
-    testParameterFiles:
-      - empty.json
-  - name: sarscov2_lineages
-    subclass: WDL
-    primaryDescriptorPath: /pipes/WDL/workflows/sarscov2_lineages.wdl
-    testParameterFiles:
-    - /test/input/WDL/test_inputs-sarscov2_lineages-local.json
   - name: sarscov2_nextclade_multi
     subclass: WDL
     primaryDescriptorPath: /pipes/WDL/workflows/sarscov2_nextclade_multi.wdl
@@ -408,6 +334,11 @@ workflows:
     primaryDescriptorPath: /pipes/WDL/workflows/scaffold_and_refine.wdl
     testParameterFiles:
       - empty.json
+  - name: subsample_by_casecounts
+    subclass: WDL
+    primaryDescriptorPath: /pipes/WDL/workflows/subsample_by_casecounts.wdl
+    testParameterFiles:
+      - empty.json
   - name: subsample_by_metadata
     subclass: WDL
     primaryDescriptorPath: /pipes/WDL/workflows/subsample_by_metadata.wdl
@@ -433,3 +364,25 @@ workflows:
     primaryDescriptorPath: /pipes/WDL/workflows/trimal.wdl
     testParameterFiles:
       - empty.json
+  - name: classify_qiime2_multi
+    subclass: WDL
+    primaryDescriptorPath: /pipes/WDL/workflows/classify_qiime2_multi.wdl
+    testParameterFiles:
+      - empty.json
+  - name: qiime_import_bam
+    subclass: WDL
+    primaryDescriptorPath: /pipes/WDL/workflows/bam_to_qiime.wdl
+    testParameterFiles:
+      - empty.json
+
+  - name: create_enterics_qc_viz
+    subclass: WDL
+    primaryDescriptorPath: /pipes/WDL/workflows/create_enterics_qc_viz.wdl
+    testParameterFiles:
+      - empty.json
+
+  - name: create_enterics_qc_viz_general
+    subclass: WDL
+    primaryDescriptorPath: /pipes/WDL/workflows/create_enterics_qc_viz_general.wdl
+    testParameterFiles:
+      - empty.json
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -2,14 +2,13 @@ name: "viral-pipelines CI"
 
 on:
   push:
+  merge_group:
   pull_request:
     branches:
       - master
   release:
     types:
       - created
-  schedule:
-    - cron: '0 18 * * 1' # weekly at 18:00 on Mondays
 
 env:
   PYTHONIOENCODING: UTF8
@@ -36,9 +35,9 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       # fetch git tags (tagged releases) because 
-      # actions/checkout@v2 does either a full checkout or a shallow checkout without tags
+      # actions/checkout@v3 does either a full checkout or a shallow checkout without tags
       - name: fetch tags
         run: git fetch --prune --unshallow --tags      
       - name: Programmatic environment setup
@@ -68,7 +67,7 @@ jobs:
         run: |
           env
       - name: install python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.8'
       - name: install system dependencies
@@ -89,9 +88,9 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       # fetch git tags (tagged releases) because 
-      # actions/checkout@v2 does either a full checkout or a shallow checkout without tags
+      # actions/checkout@v3 does either a full checkout or a shallow checkout without tags
       - name: fetch tags
         run: git fetch --prune --unshallow --tags
       - name: Programmatic environment setup
@@ -117,7 +116,7 @@ jobs:
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH"
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV
       - name: install java
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v3
         with:
           distribution: 'adopt'
           java-version: '11'
@@ -133,15 +132,15 @@ jobs:
           github_actions_ci/validate-wdl-womtool.sh
 
   ## note: this test_docs job does not actually produce the output on readthedocs
-  ## readthedocs does its own build trigger. this travis job exists simply to alert us
-  ## of build failures of the docs because otherwise we would never know.
+  ## readthedocs does its own build trigger. this job exists simply to 
+  ## attempt a build and alert us of documentation build failures because otherwise we would never know.
   test_docs:
     runs-on: ubuntu-20.04
     steps:
       - name: checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       # fetch git tags (tagged releases) because 
-      # actions/checkout@v2 does either a full checkout or a shallow checkout without tags
+      # actions/checkout@v3 does either a full checkout or a shallow checkout without tags
       - name: fetch tags
         run: git fetch --prune --unshallow --tags
       - name: Programmatic environment setup
@@ -167,7 +166,7 @@ jobs:
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH"
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV
       - name: install python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with: 
           python-version: '3.8'
       - name: install docs dependencies
@@ -184,9 +183,9 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       # fetch git tags (tagged releases) because 
-      # actions/checkout@v2 does either a full checkout or a shallow checkout without tags
+      # actions/checkout@v3 does either a full checkout or a shallow checkout without tags
       - name: fetch tags
         run: git fetch --prune --unshallow --tags
       - name: Programmatic environment setup
@@ -212,7 +211,7 @@ jobs:
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH"
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV
       - name: install java
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v3
         with:
           distribution: 'adopt'
           java-version: '11'
@@ -232,9 +231,9 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       # fetch git tags (tagged releases) because 
-      # actions/checkout@v2 does either a full checkout or a shallow checkout without tags
+      # actions/checkout@v3 does either a full checkout or a shallow checkout without tags
       - name: fetch tags
         run: git fetch --prune --unshallow --tags
       - name: Programmatic environment setup
@@ -260,7 +259,7 @@ jobs:
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH"
           echo "GITHUB_ACTIONS_BRANCH=$GITHUB_ACTIONS_BRANCH" >> $GITHUB_ENV
       - name: install python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.8'
       - name: install system dependencies
@@ -279,21 +278,21 @@ jobs:
 
   deploy_dnanexus:
     #if: ${{ github.event.ref == 'dnanexus' }}
-    if: github.event_name == 'release' || (github.base_ref == 'master' && github.event_name == 'pull_request' && github.event.pull_request.merged == true)
+    if: github.event_name == 'release' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dnanexus'
     needs:
       - validate_wdl_womtool
       - validate_wdl_miniwdl
     runs-on: ubuntu-20.04
     env:
-      # DX_API_TOKEN (for DNAnexus builds) -- token #3, expires 12/31/2024
+      # DX_API_TOKEN (for DNAnexus builds) -- GitHub Actions token #1, expires 2025-01-17
       # stored here: https://github.com/broadinstitute/viral-pipelines/settings/secrets/actions
       DX_API_TOKEN: ${{ secrets.DX_API_TOKEN }}
       DX_PROJECT: project-F8PQ6380xf5bK0Qk0YPjB17P
     steps:
       - name: checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       # fetch git tags (tagged releases) because 
-      # actions/checkout@v2 does either a full checkout or a shallow checkout without tags
+      # actions/checkout@v3 does either a full checkout or a shallow checkout without tags
       - name: fetch tags
         run: git fetch --prune --unshallow --tags
       - name: Programmatic environment setup
@@ -324,11 +323,11 @@ jobs:
           echo "${{ github.event.action }}"
           echo "${{ github.event.pull_request.merged }}"
       - name: install python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with: 
           python-version: '3.8'
       - name: install java
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v3
         with:
           distribution: 'adopt'
           java-version: '11'
@@ -344,4 +343,4 @@ jobs:
         run: |
           github_actions_ci/relative-wdl-paths.sh
           github_actions_ci/build-dx.sh
-          github_actions_ci/tests-dx.sh
+          github_actions_ci/tests-dx.sh
diff --git a/.gitignore b/.gitignore
@@ -77,3 +77,5 @@ tools/conda-tools/
 
 # miniWDL
 default.profraw
+
+.vscode/
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -7,6 +7,7 @@ version: 2
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
+  builder: html
   configuration: docs/conf.py
 
 # Build documentation with MkDocs
@@ -18,6 +19,6 @@ formats: all
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
-  version: 3.6
+  version: 3.8
   install:
-    - requirements: docs/requirements.txt
+    - requirements: docs/requirements.txt
Original file line number	Diff line number	Diff line change
Expand Up		@@ -77,3 +77,5 @@ tools/conda-tools/

		# miniWDL
		default.profraw

		.vscode/