From d363dd563c264c61a05235838cfb7a793afca648 Mon Sep 17 00:00:00 2001 From: Alexander Baumann Date: Fri, 22 Mar 2024 11:41:01 -0400 Subject: [PATCH 01/13] removing full requirements from setup.py and only installing in github action tests and docker --- .github/workflows/python-package.yml | 1 + setup.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index bbf7a60c5..de5409c7b 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -34,6 +34,7 @@ jobs: python -m pip install --upgrade pip # Install the ml4h Python package. pip install . + pip install -r ../../docker/vm_boot_images/config/tensorflow-requirements.txt - name: Test with pytest run: | pytest tests -m "not slow" diff --git a/setup.py b/setup.py index d2c923049..7431dd8fa 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,6 @@ here = pathlib.Path(__file__).parent.resolve() # Get the requirements from the requirements file -requirements = (here / 'docker/vm_boot_images/config/tensorflow-requirements.txt').read_text(encoding='utf-8') long_description = (here / 'README.md').read_text(encoding='utf-8') setup( name='ml4h', @@ -13,7 +12,6 @@ long_description_content_type='text/markdown', url='https://github.com/broadinstitute/ml4h', python_requires='>=3.6', - #install_requires=["ml4ht", "tensorflow", "pytest", "numcodecs"], # requirements - install_requires=requirements, + install_requires=["ml4ht", "tensorflow", "pytest", "numcodecs"], # requirements packages=find_packages(), ) From c4b01142d3ace633c3abcab713281052330e1fe2 Mon Sep 17 00:00:00 2001 From: abaumann Date: Fri, 22 Mar 2024 12:55:28 -0400 Subject: [PATCH 02/13] fixing path to requirements file --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index de5409c7b..8d94fbdab 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -34,7 +34,7 @@ jobs: python -m pip install --upgrade pip # Install the ml4h Python package. pip install . - pip install -r ../../docker/vm_boot_images/config/tensorflow-requirements.txt + pip install -r docker/vm_boot_images/config/tensorflow-requirements.txt - name: Test with pytest run: | pytest tests -m "not slow" From 521d3d1c3c11631f22972fdf9d2eb4a5b02da434 Mon Sep 17 00:00:00 2001 From: abaumann Date: Fri, 22 Mar 2024 16:41:06 -0400 Subject: [PATCH 03/13] added github action for pushing to pypi upon github release --- .github/workflows/publish-to-pypi.yml | 123 ++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 .github/workflows/publish-to-pypi.yml diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml new file mode 100644 index 000000000..3abf00889 --- /dev/null +++ b/.github/workflows/publish-to-pypi.yml @@ -0,0 +1,123 @@ +name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI + +on: push + +jobs: + build: + name: Build distribution 📦 + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + # Install the ml4h Python package. + pip install . + pip install -r docker/vm_boot_images/config/tensorflow-requirements.txt + - name: Install pypa/build + run: >- + python -m pip install build --user + - name: Build a binary wheel and a source tarball + run: python -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v3 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: >- + Publish Python 🐍 distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/ml4h + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + github-release: + name: >- + Sign the Python 🐍 distribution 📦 with Sigstore + and upload them to GitHub Release + needs: + - publish-to-pypi + runs-on: ubuntu-latest + + permissions: + contents: write # IMPORTANT: mandatory for making GitHub Releases + id-token: write # IMPORTANT: mandatory for sigstore + + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v1.2.3 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: Create GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + gh release create + '${{ github.ref_name }}' + --repo '${{ github.repository }}' + --notes "" + - name: Upload artifact signatures to GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + # Upload to GitHub Release using the `gh` CLI. + # `dist/` contains the built packages, and the + # sigstore-produced signatures and certificates. + run: >- + gh release upload + '${{ github.ref_name }}' dist/** + --repo '${{ github.repository }}' + + publish-to-testpypi: + name: Publish Python 🐍 distribution 📦 to TestPyPI + needs: + - build + runs-on: ubuntu-latest + + environment: + name: testpypi + url: https://test.pypi.org/p/ml4h + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ \ No newline at end of file From 252858987a049286bfe5dc4a99cecc6e5c10a8a6 Mon Sep 17 00:00:00 2001 From: abaumann Date: Fri, 22 Mar 2024 16:48:10 -0400 Subject: [PATCH 04/13] removing releasing to testpypi --- .github/workflows/publish-to-pypi.yml | 28 ++------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 3abf00889..31a085998 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -1,4 +1,4 @@ -name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI +name: Publish Python 🐍 distribution 📦 to PyPI on: push @@ -96,28 +96,4 @@ jobs: run: >- gh release upload '${{ github.ref_name }}' dist/** - --repo '${{ github.repository }}' - - publish-to-testpypi: - name: Publish Python 🐍 distribution 📦 to TestPyPI - needs: - - build - runs-on: ubuntu-latest - - environment: - name: testpypi - url: https://test.pypi.org/p/ml4h - - permissions: - id-token: write # IMPORTANT: mandatory for trusted publishing - - steps: - - name: Download all the dists - uses: actions/download-artifact@v3 - with: - name: python-package-distributions - path: dist/ - - name: Publish distribution 📦 to TestPyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: https://test.pypi.org/legacy/ \ No newline at end of file + --repo '${{ github.repository }}' \ No newline at end of file From b0c43cf6558cb1ed00615a241093ccd32c685afe Mon Sep 17 00:00:00 2001 From: abaumann Date: Wed, 27 Mar 2024 14:42:15 -0400 Subject: [PATCH 05/13] added gh action for pushing docker to gcr and ghcr --- .github/workflows/publish-to-gcr-ghcr.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/publish-to-gcr-ghcr.yml diff --git a/.github/workflows/publish-to-gcr-ghcr.yml b/.github/workflows/publish-to-gcr-ghcr.yml new file mode 100644 index 000000000..036a1f9cf --- /dev/null +++ b/.github/workflows/publish-to-gcr-ghcr.yml @@ -0,0 +1,23 @@ +name: Push to GCR/GHCR GitHub Action +on: push + +jobs: + build-and-push-to-gcr-service-account: + name: Build & push to GCR/GHCR + if: startsWith(github.ref, 'refs/tags/') # only publish to registry on tag pushes + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v2 + with: + credentials_json: '${{ secrets.B64_GCLOUD_SERVICE_ACCOUNT_JSON }}' + - name: Building and pushing the image + uses: ./ + run: | + docker login ghcr.io -u abaumann -p ${github_token} + ./docker/vm_boot_images/build.sh -P + ./docker/vm_boot_images/build.sh -c -P + with: + github_token: ${{ secrets.GHCR_TOKEN }} \ No newline at end of file From f32539419422c832a5f5ed92ef177cd7af872280 Mon Sep 17 00:00:00 2001 From: abaumann Date: Wed, 27 Mar 2024 15:06:50 -0400 Subject: [PATCH 06/13] github actions cannot use both uses and run, removing the uses --- .github/workflows/publish-to-gcr-ghcr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/publish-to-gcr-ghcr.yml b/.github/workflows/publish-to-gcr-ghcr.yml index 036a1f9cf..08987d5be 100644 --- a/.github/workflows/publish-to-gcr-ghcr.yml +++ b/.github/workflows/publish-to-gcr-ghcr.yml @@ -14,7 +14,6 @@ jobs: with: credentials_json: '${{ secrets.B64_GCLOUD_SERVICE_ACCOUNT_JSON }}' - name: Building and pushing the image - uses: ./ run: | docker login ghcr.io -u abaumann -p ${github_token} ./docker/vm_boot_images/build.sh -P From 65d4eef2777dacd4ec292112b0afc885b702f3c8 Mon Sep 17 00:00:00 2001 From: abaumann Date: Wed, 27 Mar 2024 15:15:43 -0400 Subject: [PATCH 07/13] fixing gh action to remove with and use env for a run --- .github/workflows/publish-to-gcr-ghcr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-to-gcr-ghcr.yml b/.github/workflows/publish-to-gcr-ghcr.yml index 08987d5be..16b26d666 100644 --- a/.github/workflows/publish-to-gcr-ghcr.yml +++ b/.github/workflows/publish-to-gcr-ghcr.yml @@ -18,5 +18,5 @@ jobs: docker login ghcr.io -u abaumann -p ${github_token} ./docker/vm_boot_images/build.sh -P ./docker/vm_boot_images/build.sh -c -P - with: + env: github_token: ${{ secrets.GHCR_TOKEN }} \ No newline at end of file From 34dfd070b1bc787c8041bb2597543cfc7f60dde0 Mon Sep 17 00:00:00 2001 From: abaumann Date: Wed, 27 Mar 2024 16:31:25 -0400 Subject: [PATCH 08/13] changed hardcoded username for docker login to gh secret --- .github/workflows/publish-to-gcr-ghcr.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-to-gcr-ghcr.yml b/.github/workflows/publish-to-gcr-ghcr.yml index 16b26d666..13fc510ec 100644 --- a/.github/workflows/publish-to-gcr-ghcr.yml +++ b/.github/workflows/publish-to-gcr-ghcr.yml @@ -15,8 +15,9 @@ jobs: credentials_json: '${{ secrets.B64_GCLOUD_SERVICE_ACCOUNT_JSON }}' - name: Building and pushing the image run: | - docker login ghcr.io -u abaumann -p ${github_token} + docker login ghcr.io -u ${github_username} -p ${github_token} ./docker/vm_boot_images/build.sh -P ./docker/vm_boot_images/build.sh -c -P env: - github_token: ${{ secrets.GHCR_TOKEN }} \ No newline at end of file + github_token: ${{ secrets.GHCR_TOKEN }} + github_username: ${{ secrets.GHCR_USERNAME }} \ No newline at end of file From 6a747a9d1e18c9d2a4ceacfcc14a6835361c96c9 Mon Sep 17 00:00:00 2001 From: abaumann Date: Thu, 28 Mar 2024 12:21:51 -0400 Subject: [PATCH 09/13] temporarily changing version to test out gh actions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7431dd8fa..1d348d435 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ long_description = (here / 'README.md').read_text(encoding='utf-8') setup( name='ml4h', - version='0.0.7', + version='0.0.7.1', #'0.0.7', description='Machine Learning for Health python package', long_description=long_description, # Optional long_description_content_type='text/markdown', From 86c18c376799eb931df68b916fd226c0ac2f6480 Mon Sep 17 00:00:00 2001 From: abaumann Date: Thu, 28 Mar 2024 13:37:32 -0400 Subject: [PATCH 10/13] Changing trigger to push: tags: to try to auto kick off on tagging a release --- .github/workflows/publish-to-gcr-ghcr.yml | 6 ++++-- .github/workflows/publish-to-pypi.yml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-to-gcr-ghcr.yml b/.github/workflows/publish-to-gcr-ghcr.yml index 13fc510ec..8c9d77ef0 100644 --- a/.github/workflows/publish-to-gcr-ghcr.yml +++ b/.github/workflows/publish-to-gcr-ghcr.yml @@ -1,10 +1,12 @@ name: Push to GCR/GHCR GitHub Action -on: push +on: + push: + tags: + - '*' # Push events to every tag not containing / jobs: build-and-push-to-gcr-service-account: name: Build & push to GCR/GHCR - if: startsWith(github.ref, 'refs/tags/') # only publish to registry on tag pushes runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 31a085998..129364bf7 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -1,6 +1,9 @@ name: Publish Python 🐍 distribution 📦 to PyPI -on: push +on: + push: + tags: + - '*' # Push events to every tag not containing / jobs: build: @@ -36,7 +39,6 @@ jobs: publish-to-pypi: name: >- Publish Python 🐍 distribution 📦 to PyPI - if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes needs: - build runs-on: ubuntu-latest From fd68673a8c6b598900ed56fc5a6f85135a82caa1 Mon Sep 17 00:00:00 2001 From: abaumann Date: Thu, 28 Mar 2024 13:40:46 -0400 Subject: [PATCH 11/13] Changing trigger to be on push: tags: to try to get this auto deployed --- .github/workflows/publish-to-gcr-ghcr.yml | 2 +- .github/workflows/publish-to-pypi.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-to-gcr-ghcr.yml b/.github/workflows/publish-to-gcr-ghcr.yml index 8c9d77ef0..38e5ad0b7 100644 --- a/.github/workflows/publish-to-gcr-ghcr.yml +++ b/.github/workflows/publish-to-gcr-ghcr.yml @@ -2,7 +2,7 @@ name: Push to GCR/GHCR GitHub Action on: push: tags: - - '*' # Push events to every tag not containing / + - '*' # Push events to every tag not containing / jobs: build-and-push-to-gcr-service-account: diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 129364bf7..7703ec95a 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -3,7 +3,7 @@ name: Publish Python 🐍 distribution 📦 to PyPI on: push: tags: - - '*' # Push events to every tag not containing / + - '*' # Push events to every tag not containing / jobs: build: From 905f451084e71a9a1d3bebb3263f37e46e88f067 Mon Sep 17 00:00:00 2001 From: abaumann Date: Thu, 28 Mar 2024 14:16:54 -0400 Subject: [PATCH 12/13] removing the creation of a github release from the action - we are already making a release from the UI which triggers this, no need --- .github/workflows/publish-to-pypi.yml | 45 +-------------------------- 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 7703ec95a..b1ae867fc 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -55,47 +55,4 @@ jobs: name: python-package-distributions path: dist/ - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - - github-release: - name: >- - Sign the Python 🐍 distribution 📦 with Sigstore - and upload them to GitHub Release - needs: - - publish-to-pypi - runs-on: ubuntu-latest - - permissions: - contents: write # IMPORTANT: mandatory for making GitHub Releases - id-token: write # IMPORTANT: mandatory for sigstore - - steps: - - name: Download all the dists - uses: actions/download-artifact@v3 - with: - name: python-package-distributions - path: dist/ - - name: Sign the dists with Sigstore - uses: sigstore/gh-action-sigstore-python@v1.2.3 - with: - inputs: >- - ./dist/*.tar.gz - ./dist/*.whl - - name: Create GitHub Release - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - gh release create - '${{ github.ref_name }}' - --repo '${{ github.repository }}' - --notes "" - - name: Upload artifact signatures to GitHub Release - env: - GITHUB_TOKEN: ${{ github.token }} - # Upload to GitHub Release using the `gh` CLI. - # `dist/` contains the built packages, and the - # sigstore-produced signatures and certificates. - run: >- - gh release upload - '${{ github.ref_name }}' dist/** - --repo '${{ github.repository }}' \ No newline at end of file + uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file From 3345ce0abfbf6b231dabde4e74c99af2890932d3 Mon Sep 17 00:00:00 2001 From: abaumann Date: Thu, 28 Mar 2024 14:25:17 -0400 Subject: [PATCH 13/13] adding gcloud auth for docker to support pushing to gcr --- .github/workflows/publish-to-gcr-ghcr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-to-gcr-ghcr.yml b/.github/workflows/publish-to-gcr-ghcr.yml index 9fded8560..2af2c8548 100644 --- a/.github/workflows/publish-to-gcr-ghcr.yml +++ b/.github/workflows/publish-to-gcr-ghcr.yml @@ -19,6 +19,7 @@ jobs: - name: Building and pushing the image run: | docker login ghcr.io -u ${github_username} -p ${github_token} + yes | gcloud auth configure-docker gcr.io ./docker/vm_boot_images/build.sh -P ./docker/vm_boot_images/build.sh -c -P env: