New Model: Test/test/1.0.0 #48
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: aws_runner | |
| on: | |
| issues: | |
| types: [ assigned, labeled ] | |
| jobs: | |
| start-runner: | |
| name: Start EC2 runner | |
| if: ${{ startsWith(github.event.issue.title, 'New Model:') && github.event.label.name != 'failed' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.start-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_KEY_SECRET }} | |
| aws-region: ${{ vars.AWS_REGION }} | |
| - name: Start EC2 runner | |
| id: start-ec2-runner | |
| uses: machulav/ec2-github-runner@v2 | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.GH_TOKEN }} | |
| ec2-image-id: ${{ vars.AWS_IMAGE_ID }} | |
| ec2-instance-type: ${{ vars.AWS_INSTANCE_TYPE }} | |
| subnet-id: ${{ vars.AWS_SUBNET }} | |
| security-group-id: ${{ vars.AWS_SECURITY_GROUP }} | |
| create_new_branch: | |
| needs: [start-runner] | |
| runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner | |
| steps: | |
| # - name: Cleanup disk space for large docker images | |
| # run: | | |
| # sudo rm -rf /usr/share/dotnet | |
| # sudo rm -rf /opt/ghc | |
| # sudo rm -rf "/usr/local/share/boost" | |
| # sudo rm -rf /opt/hostedtoolcache | |
| # sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| # - name: Free Disk Space (Ubuntu) | |
| # uses: jlumbroso/free-disk-space@main | |
| # with: | |
| # # this might remove tools that are actually needed, | |
| # # if set to "true" but frees about 6 GB | |
| # tool-cache: false | |
| # # all of these default to true, but feel free to set to | |
| # # "false" if necessary for your workflow | |
| # android: true | |
| # dotnet: true | |
| # haskell: true | |
| # large-packages: true | |
| # docker-images: false | |
| - name: Install singularity | |
| run: | | |
| wget https://github.com/sylabs/singularity/releases/download/v4.0.0/singularity-ce_4.0.0-focal_amd64.deb | |
| sudo apt install ./singularity-ce_4.0.0-focal_amd64.deb -y | |
| singularity version | |
| - name: Install Datalad | |
| id: datalad-testing | |
| run: | | |
| apt install unzip datalad python3-pip subversion tree -y | |
| python3 -m pip install datalad-installer datalad-osf | |
| python3 -m pip install pyyaml oyaml linkml | |
| datalad-installer --sudo ok git-annex -m datalad/git-annex:release | |
| sudo git config --global filter.annex.process "git-annex filter-process" | |
| - name: Configure git username and email | |
| run: | | |
| git config --system user.name "trained_models" | |
| git config --system user.email "trained_models" | |
| # This will automatically create a new branch from this issue, using custom config at /.github/issue-branch.yml 🟢 | |
| - name: Create Issue Branch | |
| id: branch | |
| uses: robvanderleek/create-issue-branch@main | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} | |
| - name: Clone repo and checkout branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.branch.outputs.branchName }} | |
| fetch-depth: 0 | |
| - name: Parse github issue body | |
| uses: stefanbuck/github-issue-parser@v3 | |
| id: issue-parser | |
| with: | |
| template-path: .github/ISSUE_TEMPLATE/addModel.yml # optional but recommended | |
| env: | |
| HOME: "/home/ubuntu" | |
| - name: Get the Path | |
| id: get_path | |
| run: echo "path=${{ steps.issue-parser.outputs.issueparser_path }}" >> $GITHUB_OUTPUT | |
| - name: Get the Weights | |
| id: get_weights | |
| run: echo "weights=${{ steps.issue-parser.outputs.issueparser_weights }}" >> $GITHUB_OUTPUT | |
| - name: Get docker folder | |
| id: get_docker | |
| run: echo "docker=${{ steps.issue-parser.outputs.issueparser_docker }}" >> $GITHUB_OUTPUT | |
| - name: Get Python Scripts | |
| id: python_scripts | |
| run: | | |
| echo "pythons<<EOF" >> $GITHUB_ENV | |
| echo "${{ steps.issue-parser.outputs.issueparser_python-scripts }}" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Get Model Info | |
| id: get_model_info | |
| run: | | |
| echo "model_info<<EOF" >> $GITHUB_ENV | |
| echo "${{ steps.issue-parser.outputs.issueparser_model_info }}" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Get Test Command | |
| id: get_test_command | |
| run: | | |
| echo "test_command=${{ steps.issue-parser.outputs.issueparser_testCommand }}" >> $GITHUB_OUTPUT | |
| - name: Get Sample Data | |
| id: get_data | |
| run: echo "sample_data=${{ steps.issue-parser.outputs.issueparser_sample-data }}" >> $GITHUB_OUTPUT | |
| - name: Get Config | |
| id: get_model_config | |
| run: | | |
| echo "model_config=${{ steps.issue-parser.outputs.issueparser_modelConfig }}" >> $GITHUB_OUTPUT | |
| # Get svn url for exporting docker 🟢 | |
| - name: Clone docker folder | |
| run: | | |
| url="${{ steps.get_docker.outputs.docker }}" | |
| svn_url=$(echo "$url" | sed -E 's|/tree/[^/]+|/trunk|; s|/blob/[^/]+|/trunk|') | |
| svn export --force $svn_url ./${{ steps.get_path.outputs.path }}/docker | |
| # Get svn url for exporting 🟢 | |
| - name: Generate Python SVN URLs | |
| id: generate_urls | |
| run: | | |
| echo "pythons2<<EOF" >> $GITHUB_ENV | |
| python3 ./.github/workflows/getPythonScripts.py >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| env: | |
| pythons: ${{ env.pythons }} | |
| # Export the urls/clone the scripts 🟢 | |
| - name: Clone python scripts | |
| run: | | |
| cd ./${{ steps.get_path.outputs.path }} | |
| svn_urls="${{ env.pythons2 }}" | |
| for svn_url in $svn_urls; do | |
| svn export --force $svn_url | |
| done | |
| # Get svn urls for exporting card and spec urls 🟢 | |
| - name: Generate Model Info SVN URLs | |
| id: generate_model_info_urls | |
| run: | | |
| echo "model_info2<<EOF" >> $GITHUB_ENV | |
| python3 ./.github/workflows/getModelInfo.py >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| env: | |
| model_info: ${{ env.model_info }} | |
| # Export the urls/clone the model info 🟢 | |
| - name: Clone model info | |
| run: | | |
| model_info="${{ env.model_info2 }}" | |
| for svn_url in $model_info; do | |
| svn export --force $svn_url ./${{ steps.get_path.outputs.path }} | |
| done | |
| # Clone config files | |
| - name: Config file clone | |
| # Only run if needs.create_new_branch.outputs.CONFIG is not empty | |
| if: ${{ steps.get_model_config.outputs.model_config != '' }} | |
| run: | | |
| mkdir ./${{ steps.get_path.outputs.path }}/config/ | |
| url="${{ steps.get_model_config.outputs.model_config }}" | |
| svn_url=$(echo "$url" | sed -E 's|/tree/[^/]+|/trunk|; s|/blob/[^/]+|/trunk|') | |
| svn export --force $svn_url ./${{ steps.get_path.outputs.path }}/config | |
| - name: Validate model card and spec files | |
| run: | | |
| echo "## Model Card and Spec Validation :white_check_mark:" >> $GITHUB_STEP_SUMMARY | |
| echo "Model Card and spec yaml files are being validated here with LinkML Schema" >> $GITHUB_STEP_SUMMARY | |
| python3 ./.github/workflows/create_model_card_and_spec.py | |
| env: | |
| model_path: ${{ steps.get_path.outputs.path }} | |
| - name: Update yaml file | |
| run: | | |
| model_name=$(echo "${{ steps.get_path.outputs.path }}" | awk -F '/' '{print $(NF-1)}') | |
| python3 ./.github/workflows/update_yaml_info.py ${{ steps.get_path.outputs.path }} $model_name | |
| - name: Set docker image name | |
| id: set_image_name | |
| run: | | |
| full_path="${{ steps.get_path.outputs.path }}" | |
| org_name=$(echo "$full_path" | awk -F '/' '{print $1}') | |
| model_name=$(echo "$full_path" | awk -F '/' '{print $2}') | |
| model_version=$(echo "$full_path" | awk -F '/' '{print $3}') | |
| echo "image_name=neuronets/$model_name" >> $GITHUB_OUTPUT | |
| echo "org_name=$org_name" >> $GITHUB_OUTPUT | |
| echo "model_name=$model_name" >> $GITHUB_OUTPUT | |
| echo "model_version=$model_version" >> $GITHUB_OUTPUT | |
| - name: Add weights | |
| env: | |
| OSF_TOKEN: ${{ secrets.OSF_TOKEN }} | |
| run: | | |
| datalad download-url -o --nosave "${{ steps.get_weights.outputs.weights }}" -O ./${{ steps.get_path.outputs.path }}/weights/ | |
| datalad download-url -o --nosave "${{ steps.get_data.outputs.sample_data }}" -O ./${{ steps.get_path.outputs.path }}/example-data/ | |
| # unzip files in place and delete after | |
| find . -name '*.zip' -not -path "./.git/*" -execdir unzip '{}' ';' -delete | |
| - name: print directory structure | |
| run: tree ${{ steps.set_image_name.outputs.org_name }} | |
| - name: Build Docker image | |
| run: | | |
| docker build -t ${{ steps.set_image_name.outputs.model_name }} ./${{ steps.get_path.outputs.path }}/docker | |
| - name: Convert docker image to singularity image | |
| run: | | |
| singularity build ../${{ steps.set_image_name.outputs.model_name }}.sif docker-daemon://${{ steps.set_image_name.outputs.model_name }}:latest | |
| - name: Collect Workflow Telemetry | |
| uses: runforesight/workflow-telemetry-action@v1 | |
| with: | |
| job_summary: true | |
| proc_trace_sys_enable: true | |
| proc_trace_table_show: true | |
| - name: Run test command in Singularity | |
| id: singularity_test | |
| run: | | |
| # Parent directory as a bind path in a env variable | |
| # singularity exec --nv ../${{ steps.set_image_name.outputs.model_name }}.sif ${{ steps.get_test_command.outputs.test_command }} | |
| singularity exec --nv ../${{ steps.set_image_name.outputs.model_name }}.sif pwd | |
| - name: Run test command in Docker | |
| id: docker_test | |
| run: | | |
| docker run --gpus all -v $(pwd):/output \ | |
| -w /output \ | |
| ${{ steps.set_image_name.outputs.model_name }} "pwd" | |
| # ${{ steps.set_image_name.outputs.model_name }} "${{ steps.get_test_command.outputs.test_command }}" | |
| # - name: Push Docker image | |
| # if: steps.docker_test.outcome == 'success' && steps.singularity_test.outcome == 'success' | |
| # run: | | |
| # docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }} | |
| # docker tag ${{ steps.set_image_name.outputs.model_name }} ${{ steps.set_image_name.outputs.image_name }}:${{ steps.set_image_name.outputs.model_version }} | |
| # docker push ${{ steps.set_image_name.outputs.image_name }}:${{ steps.set_image_name.outputs.model_version }} | |
| - name: Commit changes | |
| if: steps.docker_test.outcome == 'success' && steps.singularity_test.outcome == 'success' | |
| run: | | |
| datalad save -m "SUCCESS: Adding Model" ${{ steps.set_image_name.outputs.org_name }} | |
| datalad push -f all --to osf-annex1106-storage | |
| datalad push --to origin | |
| # git push origin ${{ steps.branch.outputs.branchName }} | |
| env: | |
| OSF_TOKEN: ${{ secrets.OSF_TOKEN }} | |
| stop-runner: | |
| name: Stop EC2 runner | |
| needs: [start-runner, create_new_branch] # required to wait when the main job is done | |
| runs-on: ubuntu-latest | |
| if: ${{ always() }} | |
| # required to stop the runner even if the error happened in the previous jobs | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_KEY_SECRET }} | |
| aws-region: ${{ vars.AWS_REGION }} | |
| - name: Stop EC2 runner | |
| uses: machulav/ec2-github-runner@v2 | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.GH_TOKEN }} | |
| label: ${{ needs.start-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} | |
| failed: | |
| runs-on: ubuntu-latest | |
| needs: [start-runner, create_new_branch] | |
| if: ${{ failure() }} | |
| steps: | |
| - name: Set labels | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: 'set-labels' | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| issue-number: ${{ github.event.issue.number }} | |
| labels: 'failed' | |
| - name: Create comment | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: 'create-comment' | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| issue-number: ${{ github.event.issue.number }} | |
| body: | | |
| [This is an automated message] Hello @${{ github.event.issue.user.login }}, | |
| 🔴 The folders/scripts you provided did not pass our tests. Please review the Action, and modify this issue's urls accordingly. | |
| When ready, simply append "Ready XX" in the issue title (where XX is a number incrementing with 01 each time a fix has been applied). | |
| emoji: '+1,eyes' | |
| success: | |
| needs: [start-runner, create_new_branch, stop-runner] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set labels | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: 'set-labels' | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| issue-number: ${{ github.event.issue.number }} | |
| labels: 'success' | |
| - name: Create comment | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: 'create-comment' | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| issue-number: ${{ github.event.issue.number }} | |
| body: | | |
| [This is an automated message] Hello @${{ github.event.issue.user.login }}. The workflow linked to adding your model finished successfully! Please double check that this issue's tag is "success." | |
| 🟢 A Draft PR should be linked to this issue. Now that your model passed the checks, feel free to change the status of the PR to "ready for review." | |
| ⭐ Thank you for adding a model to Nobrainer-Zoo! | |
| emoji: '+1,hooray,rocket' |