Sync tools and sync workflows and create catalogue #656
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync tools and sync workflows and create catalogue | |
| on: | |
| workflow_dispatch: | |
| pull_request_review: | |
| types: | |
| - "submitted" | |
| branches: | |
| - main | |
| jobs: | |
| setup: | |
| # First check | |
| # 1a. It's a PR that's in an approved state | |
| # AND | |
| # 1b. It's not a PR to 'Add in images for release' | |
| # 2. OR it's been run manually by using the workflow_dispatch command | |
| if: | | |
| ( | |
| ( | |
| github.event.review.state == 'approved' && | |
| github.event.pull_request.draft == false | |
| ) && | |
| ! startsWith( github.event.pull_request.title, '[GitHub Actions]' ) | |
| ) || | |
| github.event_name == 'workflow_dispatch' | |
| name: setup-sync-tools | |
| concurrency: git_commits | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| # # DEBUG | |
| # - uses: hmarr/debug-action@v2 | |
| # Set to fail | |
| - name: Update bash settings | |
| id: update_bash_settings | |
| run: | | |
| set -euo pipefail | |
| # Install jq (for querying branch name) | |
| - name: Install Jq | |
| id: install_jq | |
| run: | | |
| sudo apt-get update -y | |
| sudo apt-get install jq -y | |
| # Get branch name from event path | |
| - name: Get Branch Name | |
| id: get_branch_name | |
| run: | | |
| # Get reference | |
| ref="$( \ | |
| jq --raw-output \ | |
| ' | |
| # Get head of the pull request | |
| if .pull_request?.head?.ref? != null then | |
| .pull_request.head.ref | |
| # If not try the base reference | |
| elif .pull_request?.base?.ref? != null then | |
| .pull_request.base.ref | |
| # Could this maybe not a PR? | |
| # Try legacy | |
| elif .ref? != null then | |
| .ref | |
| elif .base?.ref? != null then | |
| .base.ref | |
| else | |
| null | |
| end | |
| ' \ | |
| < "${GITHUB_EVENT_PATH}" \ | |
| )" | |
| # Could not get reference commit id | |
| if [[ "${ref}" == "null" ]]; then | |
| echo "Error! Could not get reference commit id" | |
| cat "${GITHUB_EVENT_PATH}" | |
| exit 1 | |
| fi | |
| # Set output | |
| echo "branch_name=${ref%refs/heads/}" >> "${GITHUB_OUTPUT}" | |
| # Standard checkout step | |
| - name: Checkout code | |
| id: git_checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| # Otherwise we're just in a detached head state | |
| ref: ${{ steps.get_branch_name.outputs.branch_name }} | |
| # Get git commit ID | |
| - name: Get git commit ID | |
| id: get_git_commit_id | |
| run: echo "git_commit_id=$(git log --format="%H" -n1 | cut -c1-7)" >> "${GITHUB_OUTPUT}" | |
| # Get outputs | |
| outputs: | |
| git_commit_id: ${{ steps.get_git_commit_id.outputs.git_commit_id }} | |
| branch_name: ${{ steps.get_branch_name.outputs.branch_name }} | |
| # Get access tokens | |
| get_umccr_development_workflows_ica_access_token: | |
| name: Get ICA Access Token (umccr-development) | |
| uses: ./.github/workflows/get_ica_access_token.yml | |
| needs: | |
| - setup | |
| permissions: | |
| id-token: write | |
| contents: read | |
| with: | |
| project_id: "0df0356d-3637-48a5-80d1-a924642a6556" # development_workflows | |
| role_to_assume_arn: arn:aws:iam::843407916570:role/ica-credentials-dev-stack-IcaSecretsicacredentialsd-9oauPhOTTfHb | |
| secret_arn: arn:aws:secretsmanager:ap-southeast-2:843407916570:secret:IcaSecretsWorkflow-yUaicA | |
| aws_region: ${{ vars.AWS_REGION }} | |
| branch_name: ${{ needs.setup.outputs.branch_name }} | |
| secrets: | |
| symmetrical_encryption_key: ${{ secrets.SYMMETRICAL_ENCRYPTION_KEY }} | |
| get_umccr_collab_illumina_dev_workflows_ica_access_token: | |
| name: Get ICA Access Token (umccr-collab-illumina-dev) | |
| uses: ./.github/workflows/get_ica_access_token.yml | |
| needs: | |
| - setup | |
| permissions: | |
| id-token: write | |
| contents: read | |
| with: | |
| # Uses same secret as dev | |
| project_id: "dddd6c29-24d3-49f4-91c0-7e818b3c0a21" # collab-illumina-dev_workflows | |
| role_to_assume_arn: arn:aws:iam::843407916570:role/ica-credentials-dev-stack-IcaSecretsicacredentialsd-9oauPhOTTfHb | |
| secret_arn: arn:aws:secretsmanager:ap-southeast-2:843407916570:secret:IcaSecretsWorkflow-yUaicA | |
| aws_region: ${{ vars.AWS_REGION }} | |
| branch_name: ${{ needs.setup.outputs.branch_name }} | |
| secrets: | |
| symmetrical_encryption_key: ${{ secrets.SYMMETRICAL_ENCRYPTION_KEY }} | |
| get_umccr_production_workflows_ica_access_token: | |
| name: Get ICA Access Token (umccr-production) | |
| uses: ./.github/workflows/get_ica_access_token.yml | |
| needs: | |
| - setup | |
| permissions: | |
| id-token: write | |
| contents: read | |
| with: | |
| project_id: "fdd48e11-cdcc-46a9-b5ac-dee3a4c5f19d" # production_workflows | |
| role_to_assume_arn: arn:aws:iam::472057503814:role/ica-credentials-prod-stac-IcaSecretsicacredentialsp-pFZQUb2Bvys7 | |
| secret_arn: arn:aws:secretsmanager:ap-southeast-2:472057503814:secret:IcaSecretsWorkflow-zq2Abw | |
| aws_region: ${{ vars.AWS_REGION }} | |
| branch_name: ${{ needs.setup.outputs.branch_name }} | |
| secrets: | |
| symmetrical_encryption_key: ${{ secrets.SYMMETRICAL_ENCRYPTION_KEY }} | |
| # Now sync | |
| sync_tools_and_workflows_and_create_catalogue: | |
| name: sync tools and workflows and create catalogue | |
| concurrency: git_commits | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| needs: | |
| - setup | |
| - get_umccr_development_workflows_ica_access_token | |
| - get_umccr_collab_illumina_dev_workflows_ica_access_token | |
| - get_umccr_production_workflows_ica_access_token | |
| steps: | |
| # Standard checkout step | |
| - name: Checkout code | |
| id: git_checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| # For changed files we need to look back in time a little | |
| fetch-depth: 0 | |
| # Otherwise we're just in a detached head state | |
| ref: ${{ needs.setup.outputs.branch_name }} | |
| # Add masks to encrypted tokens | |
| - name: Add masks to encrypted tokens | |
| run: | | |
| echo "::add-mask::${{ needs.get_umccr_development_workflows_ica_access_token.outputs.ica_access_token_encrypted }}" | |
| echo "::add-mask::${{ needs.get_umccr_collab_illumina_dev_workflows_ica_access_token.outputs.ica_access_token_encrypted }}" | |
| echo "::add-mask::${{ needs.get_umccr_production_workflows_ica_access_token.outputs.ica_access_token_encrypted }}" | |
| # Decrypt tokens | |
| - id: decrypt_development_workflows_project_id_access_token | |
| name: Decrypt Development Access Token | |
| uses: ./.github/actions/decrypt_token | |
| with: | |
| encrypted_token: ${{ needs.get_umccr_development_workflows_ica_access_token.outputs.ica_access_token_encrypted }} | |
| decryption_key: ${{ secrets.SYMMETRICAL_ENCRYPTION_KEY }} | |
| - id: decrypt_collab_illumina_dev_workflows_project_id_access_token | |
| name: Decrypt Collab Illumina Dev Access Token | |
| uses: ./.github/actions/decrypt_token | |
| with: | |
| encrypted_token: ${{ needs.get_umccr_collab_illumina_dev_workflows_ica_access_token.outputs.ica_access_token_encrypted }} | |
| decryption_key: ${{ secrets.SYMMETRICAL_ENCRYPTION_KEY }} | |
| - id: decrypt_production_workflows_project_id_access_token | |
| name: Decrypt Production Access Token | |
| uses: ./.github/actions/decrypt_token | |
| with: | |
| encrypted_token: ${{ needs.get_umccr_production_workflows_ica_access_token.outputs.ica_access_token_encrypted }} | |
| decryption_key: ${{ secrets.SYMMETRICAL_ENCRYPTION_KEY }} | |
| # Check that the paths were interested in are actually of any significants | |
| - id: check_file_changes | |
| name: Check that this PR is relevant to workflows | |
| uses: tj-actions/changed-files@v41 | |
| with: | |
| files_yaml: | | |
| workflows: | |
| # Workflows, Expressions, CommandLineTools | |
| - "**/*.cwl" | |
| # Schemas | |
| - "schemas/**/*.yaml" | |
| # TypeScript Expressions | |
| - "**/*.cwljs" | |
| gh_actions: | |
| - ".github/actions/get_ica_access_token/action.yml" | |
| - ".github/workflows/sync-tools_and_sync-workflows.yml" | |
| - ".github/scripts/run_sync-tools_and_sync-workflows.sh" | |
| catalogs: | |
| - ".github/create_catalogue.sh" | |
| configurations: | |
| # Config files (say a merge conflict is resolved) | |
| - "config/project.yaml" | |
| - "config/tool.yaml" | |
| - "config/workflow.yaml" | |
| - "config/run.yaml" | |
| base_sha: ${{ github.event.pull_request.base.sha }} | |
| # Create secrets json | |
| - name: create secrets json | |
| id: create_secrets_json | |
| run: | | |
| secrets_json="$( \ | |
| jq --null-input --raw-output \ | |
| --arg development_workflows_access_token "${{ steps.decrypt_development_workflows_project_id_access_token.outputs.decrypted_token }}" \ | |
| --arg collab_illumina_dev_workflows_access_token "${{ steps.decrypt_collab_illumina_dev_workflows_project_id_access_token.outputs.decrypted_token }}" \ | |
| --arg production_workflows_access_token "${{ steps.decrypt_production_workflows_project_id_access_token.outputs.decrypted_token }}" \ | |
| ' | |
| { | |
| "development_workflows": { | |
| "ICA_ACCESS_TOKEN": $development_workflows_access_token | |
| }, | |
| "collab_illumina_dev_workflows": { | |
| "ICA_ACCESS_TOKEN": $collab_illumina_dev_workflows_access_token | |
| }, | |
| "production_workflows": { | |
| "ICA_ACCESS_TOKEN": $production_workflows_access_token | |
| } | |
| } | | |
| @base64 | |
| ' \ | |
| )" | |
| echo "::add-mask::${secrets_json}" | |
| echo "secrets_json=${secrets_json}" >> "${GITHUB_OUTPUT}" | |
| # Sync tools and workflows | |
| - name: sync-tools and sync-workflows | |
| if: | | |
| steps.check_file_changes.outputs.workflows_any_changed == 'true' || | |
| steps.check_file_changes.outputs.configurations_any_changed == 'true' || | |
| github.event_name == 'workflow_dispatch' | |
| run: | | |
| docker run \ | |
| --rm \ | |
| --user "$(id -u):$(id -g)" \ | |
| --volume "$PWD:$PWD" \ | |
| --workdir "$PWD" \ | |
| --env USER="$(id -u)" \ | |
| --env GIT_COMMIT_ID="${{ needs.setup.outputs.git_commit_id }}" \ | |
| --env SECRETS_JSON="${{ steps.create_secrets_json.outputs.secrets_json }}" \ | |
| --env ICA_BASE_URL="${{ vars.ICA_BASE_URL }}" \ | |
| ghcr.io/umccr/cwl-ica-cli:latest \ | |
| bash ".github/scripts/run_sync-tools_and_sync-workflows.sh" | |
| # Create Catalogue | |
| - name: create catalogue | |
| if: | | |
| steps.check_file_changes.outputs.workflows_any_changed == 'true' || | |
| steps.check_file_changes.outputs.configurations_any_changed == 'true' || | |
| steps.check_file_changes.outputs.catalogs_any_changed == 'true' || | |
| github.event_name == 'workflow_dispatch' | |
| run: | | |
| docker run \ | |
| --rm \ | |
| --user "$(id -u):$(id -g)" \ | |
| --volume "$PWD:$PWD" \ | |
| --workdir "$PWD" \ | |
| --env USER="$(id -nu)" \ | |
| --env GITHUB_SERVER_URL="${GITHUB_SERVER_URL}" \ | |
| --env GITHUB_REPOSITORY="${GITHUB_REPOSITORY}" \ | |
| ghcr.io/umccr/cwl-ica-cli:latest \ | |
| bash ".github/scripts/create_catalogue.sh" | |
| # Commit config files | |
| - id: commit_and_push_config_and_catalogue | |
| name: Commit catalogue files | |
| uses: EndBug/[email protected] | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| add: "[ 'config/', 'cwl-ica-catalogue.md', '.github/catalogue/' ]" | |
| default_author: github_actions | |
| message: "Updating cwl ica config and catalogue files - (Autogenerated github actions commit)" | |
| push: true | |
| pull: ${{ format('--rebase --autostash origin {0}', needs.setup.outputs.branch_name) }} | |