Initial commit

.github/workflows/cd.yml

@@ -0,0 +1,79 @@
+# Continuous integration and delivery of the main branch.
+
+name: CI/CD
+on:
+  push:
+    branches:
+    - main
+
+jobs:
+  # Release on Github and Docker
+  release:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.9]
+
+    steps:
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Checkout github repo
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        $CONDA/bin/conda install -y -c bioconda womtool 
+        $CONDA/bin/conda config --add channels conda-forge
+        $CONDA/bin/conda install -y -c conda-forge miniwdl shellcheck
+        echo "$CONDA/bin" >> $GITHUB_PATH
+
+    - name: Test with tox
+      run: tox
+
+    - name: Write commit message
+      run: git log --format=%B -n 1 ${{ github.event.after }} > release_notes.txt
+
+    - name: Github Bumpversion Action
+      id: version-bump
+      uses: jasonamyers/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        DEFAULT_BUMP: "patch"
+
+    - name: Push changes
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        tags: true
+
+    - name: Create Github release
+      id: create_release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: v${{ steps.version-bump.outputs.new_ver }}
+        release_name: 'v${{ steps.version-bump.outputs.new_ver }}'
+        body_path: "release_notes.txt"
+        draft: false
+        prerelease: false
+
+    - id: 'auth'
+      name: 'Authenticate to Google Cloud'
+      uses: 'google-github-actions/auth@v0'
+      with:
+        credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+
+    - name: Build Docker images
+      run: |-
+        gcloud config set account [email protected]
+        gcloud auth configure-docker -q us-central1-docker.pkg.dev
+        ./scripts/build_docker_images.sh ${{ steps.version-bump.outputs.new_ver }}
+

.github/workflows/ci_push.yml

@@ -0,0 +1,61 @@
+# Continuous integration test of any branch.
+
+name: CI
+on:
+  push:
+    branches-ignore:
+    - main
+
+jobs:
+  # Run Python tests on Github runner
+  run_tox_test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.9]
+
+    steps:
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Checkout github repo
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        $CONDA/bin/conda install -y -c bioconda womtool 
+        $CONDA/bin/conda config --add channels conda-forge
+        $CONDA/bin/conda install -y -c conda-forge miniwdl shellcheck
+        echo "$CONDA/bin" >> $GITHUB_PATH
+
+    - name: Test with tox
+      run: tox
+
+  # Build Docker image
+  build_docker_images:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout github repo
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - id: 'auth'
+      name: 'Authenticate to Google Cloud'
+      uses: 'google-github-actions/auth@v0'
+      with:
+        credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+
+    - name: Build Docker images
+      run: |-
+        gcloud config set account [email protected]
+        gcloud auth configure-docker -q us-central1-docker.pkg.dev
+        ./scripts/build_docker_images.sh $(git rev-parse --abbrev-ref HEAD)
+

DEVELOP.md

@@ -0,0 +1,82 @@
+# AoU LR Playground
+Tools to make working with AoU LR data easier.
+
+Current version: 0.0.8
+
+## Development
+
+To do development in this codebase, the python3 development package must
+be installed.
+
+After installation the development environment can be set up by
+the following commands:
+
+    python3 -mvenv venv
+    . venv/bin/activate
+    pip install --upgrade pip
+    pip install -r requirements.txt
+    pip install -e .
+
+### Linting files
+
+    # run all linting commands
+    tox -e lint
+
+    # reformat all project files
+    black src tests setup.py
+
+    # sort imports in project files
+    isort -rc src tests setup.py
+
+    # check pep8 against all project files
+    flake8 src tests setup.py
+
+    # lint python code for common errors and codestyle issues
+    pylint src
+
+### Tests
+
+    # run all linting and test
+    tox
+
+    # run only (fast) unit tests
+    tox -e unit
+
+    # run only integration tests
+    tox -e integration
+
+    # run only a single test
+    # (in this case, the integration tests for `annotate`)
+    tox -e singletest -- tests/integration/test_annotate.py::test_annotate
+
+    # run only linting
+    tox -e lint
+
+Note: If you run into "module not found" errors when running tox for testing, verify the modules are listed in test-requirements.txt and delete the .tox folder to force tox to refresh dependencies.
+
+## Releasing
+
+Docker images are released automatically upon pushes to the main branch (generally by merging branches into main).  Automatic releases consist of the following steps:
+
+### Versioning
+
+We use `bumpversion` to maintain version numbers.  It will automatically create a new tag each time it is run.
+*DO NOT MANUALLY EDIT ANY VERSION NUMBERS.*
+
+Our versions are specified by a 3 number semantic version system (https://semver.org/):
+
+	major.minor.patch
+
+By default, pushes to main will increment the patch number.  Major and minor version numbers can be incremented through special keywords in commit messages.  From the [automated-version-bump](https://github.com/marketplace/actions/automated-version-bump) Github action documentation:
+
+> Based on the commit messages, increment the version from the latest release.
+> * If the string "BREAKING CHANGE", "major" or the Attention pattern refactor!: drop support for Node 6 is found anywhere in any of the commit messages or descriptions the major version will be incremented.
+> * If a commit message begins with the string "feat" or includes "minor" then the minor version will be increased. This works for most common commit metadata for feature additions: "feat: new API" and "feature: new API".
+> * If a commit message contains the word "pre-alpha" or "pre-beta" or "pre-rc" then the pre-release version will be increased (for example specifying pre-alpha: 1.6.0-alpha.1 -> 1.6.0-alpha.2 or, specifying pre-beta: 1.6.0-alpha.1 -> 1.6.0-beta.0)
+All other changes will increment the patch version.
+
+Versions will always be bumped from the *main branch* _after_ merging in any PRs for that version.
+
+### Docker images
+
+This repository includes a Docker folder that contains subdirectories with `Dockerfile`. New images will be built automatically, named after the subdirectory in which they appear, and pushed to [Google's Artifact Registry](https://console.cloud.google.com/artifacts/docker/broad-dsp-lrma/us-central1/aou-lr) . 

LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2023, Kiran V Garimella
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.rst

@@ -0,0 +1,33 @@
+AoU LR
+""""""
+
+|GitHub release| |Generic badge|
+
+.. |GitHub release| image:: https://img.shields.io/github/release/kvg/aou-lr.svg
+   :target: https://github.com/kvg/longbow/aou-lr
+
+.. |Generic badge| image:: https://img.shields.io/badge/Docker-v0.0.8-blue.svg
+   :target: https://console.cloud.google.com/artifacts/docker/broad-dsp-lrma/us-central1/aou-lr
+
+Analysis workflows for AoU long read data.
+
+
+Quick start
+-----------
+
+::
+
+    # Clone repo
+    git clone https://github.com/kvg/aou-lr.git
+
+
+Getting help
+------------
+
+If you encounter bugs or have questions/comments/concerns, please file an issue on our `Github page <https://github.com/kvg/aou-lr/issues>`_.
+
+
+Developers' guide
+-----------------
+
+For information on contributing to development, visit our `developer documentation <DEVELOP.md>`_.

docker/aou-lr-sv/Dockerfile

@@ -0,0 +1,17 @@
+FROM continuumio/miniconda3
+
+MAINTAINER Kiran V Garimella
+
+# copy other resources
+COPY ./environment.yml /
+
+# install conda packages
+RUN conda env create -f /environment.yml && conda clean -a
+ENV PATH=/opt/conda/envs/aou-lr-sv/bin/:${PATH}
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/conda/envs/aou-lr-sv/lib/
+
+RUN apt-get -y update \
+	&& apt-get -y install git make cmake protobuf-compiler gcc g++ zlib1g-dev libcurl4-openssl-dev libbz2-dev tree python3-pip liblzma-dev wget curl \
+	&& apt-get clean
+
+RUN echo "source activate aou-lr-sv" > ~/.bashrc

docker/aou-lr-sv/environment.yml

@@ -0,0 +1,8 @@
+name: aou-lr-sv
+channels:
+  - bioconda
+  - conda-forge
+  - defaults
+dependencies:
+  - minimap2
+prefix: /opt/conda/envs/aou-lr-sv

inputs/HelloWorkflow.json

@@ -0,0 +1,3 @@
+{
+    "HelloWorkflow.greeting": "Hi there!"
+}

requirements.txt

@@ -0,0 +1,6 @@
+google-cloud-storage
+numpy
+pysam
+pytest
+pytest-console-scripts
+tox

resources/workflow_options/default.json

@@ -0,0 +1,5 @@
+{
+  "write_to_cache": true,
+  "read_from_cache": true,
+  "workflow_failure_mode": "ContinueWhilePossible",
+}

resources/workflow_options/fresh.run.json

@@ -0,0 +1,5 @@
+{
+  "write_to_cache": false,
+  "read_from_cache": false,
+  "workflow_failure_mode": "ContinueWhilePossible",
+}

run_hello_workflow.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+RESOURCES="resources/workflow_options/default.json"
+CA="cromshell-alpha -q --hide_logo -t 30 submit -d ./wdl"
+
+$CA wdl/pipelines/HelloWorkflow.wdl inputs/HelloWorkflow.json

scripts/build_docker_images.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+DOCKER_REPO="aou-lr"
+LABEL=$1
+
+for DOCKER_FILE in $(find docker -name Dockerfile)
+do
+    DIR_NAME=$(dirname $DOCKER_FILE)
+    DOCKER_NAME=$(basename $DIR_NAME)
+
+    TAG="us-central1-docker.pkg.dev/broad-dsp-lrma/$DOCKER_REPO/$DOCKER_NAME:$LABEL"
+
+    if docker manifest inspect $TAG > /dev/null; then
+        docker pull $TAG
+    fi
+
+    docker build -t $TAG $DIR_NAME && docker push $TAG
+done