Garage CI #2797
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Garage CI | |
| env: | |
| GARAGE_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| PR_COMMIT_RANGE: origin/${{ github.base_ref }}...origin/${{ github.head_ref }} | |
| DOCKER_TAG: garage-ci-${{ github.run_id }} | |
| OWNER: "rlworkgroup" | |
| DOCKER_CACHE_REPO: "garage-ci" | |
| MJKEY: ${{ secrets.MJKEY }} | |
| CI_USER: rlworkgroupbot | |
| IS_PR_FROM_FORK: ${{ github.event.client_payload.pull_request.head.repo.fork }} | |
| on: | |
| schedule: | |
| - cron: '0 9 * * *' | |
| push: | |
| branches: | |
| - master | |
| # Triggers on release candidates from master | |
| tags: | |
| - 'v20[2-9][0-9].[0-9][0-9].[0-9]+rc[0-9]' | |
| pull_request: | |
| branches: | |
| - master | |
| repository_dispatch: | |
| types: [ ok-to-test-command ] | |
| jobs: | |
| comment_link_to_output_on_fork_pr: | |
| name: Post link to CI output for fork PRs | |
| runs-on: ubuntu-latest | |
| # can't use env.IS_PR_FROM_FORK here because env is not available here | |
| if: github.event.client_payload.pull_request.head.repo.fork | |
| steps: | |
| - name: Dump the client payload context | |
| env: | |
| PAYLOAD_CONTEXT: ${{ toJson(github.event.client_payload) }} | |
| run: echo "$PAYLOAD_CONTEXT" | |
| - name: Create URL to the run output | |
| id: vars | |
| run: echo ::set-output name=run-url::https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID | |
| - name: Create comment | |
| uses: peter-evans/create-or-update-comment@v1 | |
| with: | |
| token: ${{ secrets.CI_REGISTRY_TOKEN }} | |
| repository: ${{ github.event.client_payload.github.payload.repository.full_name }} | |
| issue-number: ${{ github.event.client_payload.github.payload.issue.number }} | |
| body: | | |
| [Command run output][1] for ${{ github.event.client_payload.pull_request.html_url }}/commits/${{ github.event.client_payload.pull_request.head.sha }} | |
| [1]: ${{ steps.vars.outputs.run-url }} | |
| build_docker_container: | |
| name: Build Docker Container | |
| runs-on: ubuntu-latest | |
| if: ${{ !(github.event.pull_request.head.repo.fork && github.event_name == 'pull_request') }} | |
| steps: | |
| - name: Docker info | |
| run: docker version | |
| - uses: actions/checkout@v2 | |
| if: ${{ !env.IS_PR_FROM_FORK }} | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/checkout@v2 | |
| if: env.IS_PR_FROM_FORK | |
| with: | |
| ref: ${{ github.event.client_payload.pull_request.merge_commit_sha }} | |
| fetch-depth: 0 | |
| - name: Login to GitHub Package Registry | |
| run: echo ${{ secrets.CI_REGISTRY_TOKEN }} | docker login docker.pkg.github.com -u ${CI_USER} --password-stdin | |
| - name: Build Docker container | |
| run: | | |
| DOCKER_BUILDKIT=1 docker build . \ | |
| -f docker/Dockerfile \ | |
| --target garage-dev \ | |
| -t "${DOCKER_TAG}" \ | |
| --build-arg GARAGE_GH_TOKEN \ | |
| --cache-from="rlworkgroup/garage" | |
| - name: Push to cache (GitHub Package Registry) | |
| run: | | |
| docker tag "${DOCKER_TAG}" "docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG}" | |
| docker push "docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG}" | |
| - name: Update check status for PRs from forks | |
| uses: LouisBrunner/[email protected] | |
| if: always() && env.IS_PR_FROM_FORK | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| name: Build Docker Container | |
| sha: ${{ github.event.client_payload.pull_request.head.sha }} | |
| conclusion: ${{ job.status }} | |
| check_pre_commit: | |
| name: Check pre-commit | |
| runs-on: ubuntu-latest | |
| needs: build_docker_container | |
| steps: | |
| - name: Login to GitHub Package Registry | |
| if: github.event_name == 'pull_request' || env.IS_PR_FROM_FORK | |
| run: echo ${{ secrets.CI_REGISTRY_TOKEN }} | docker login docker.pkg.github.com -u ${CI_USER} --password-stdin | |
| - name: Pull from cache (GitHub Package Registry) | |
| if: github.event_name == 'pull_request' || env.IS_PR_FROM_FORK | |
| run: docker pull "docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG}" | |
| - name: Tag docker image | |
| if: github.event_name == 'pull_request' || env.IS_PR_FROM_FORK | |
| run: docker tag docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG} ${DOCKER_TAG} | |
| - name: Export commit range for fork PRs | |
| if: env.IS_PR_FROM_FORK | |
| run: | | |
| echo "PR_COMMIT_RANGE=${{ github.event.client_payload.pull_request.base.sha }}...${{ github.event.client_payload.pull_request.head.sha }}" >> $GITHUB_ENV | |
| - name: Check pre-commit | |
| if: github.event_name == 'pull_request' || env.IS_PR_FROM_FORK | |
| run: | | |
| docker run \ | |
| -e PR_COMMIT_RANGE \ | |
| -e MJKEY \ | |
| --memory 6500m \ | |
| --memory-swap 6500m \ | |
| "${DOCKER_TAG}" scripts/ci/check_precommit.sh | |
| - name: Update check status for PRs from forks | |
| uses: LouisBrunner/[email protected] | |
| if: env.IS_PR_FROM_FORK && always() | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| name: Check pre-commit | |
| sha: ${{ github.event.client_payload.pull_request.head.sha }} | |
| conclusion: ${{ job.status }} | |
| doctest: | |
| name: Run Doctest | |
| runs-on: ubuntu-latest | |
| needs: check_pre_commit | |
| steps: | |
| - name: Login to GitHub Package Registry | |
| run: echo ${{ secrets.CI_REGISTRY_TOKEN }} | docker login docker.pkg.github.com -u ${CI_USER} --password-stdin | |
| - name: Pull from cache (GitHub Package Registry) | |
| run: docker pull "docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG}" | |
| - name: Tag docker image | |
| run: docker tag docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG} ${DOCKER_TAG} | |
| - name: Doctest | |
| run: | | |
| docker run \ | |
| -e MJKEY \ | |
| --memory 6500m \ | |
| --memory-swap 6500m \ | |
| "${DOCKER_TAG}" \ | |
| /bin/bash -c \ | |
| 'pip list && pushd docs && make doctest clean && popd; cat /tmp/*.log' | |
| - name: Update check status for PRs from forks | |
| uses: LouisBrunner/[email protected] | |
| if: env.IS_PR_FROM_FORK && always() | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| name: Run Doctest | |
| sha: ${{ github.event.client_payload.pull_request.head.sha }} | |
| conclusion: ${{ job.status }} | |
| normal_test: | |
| name: Normal Tests | |
| runs-on: ubuntu-latest | |
| needs: check_pre_commit | |
| steps: | |
| - name: Login to GitHub Package Registry | |
| run: echo ${{ secrets.CI_REGISTRY_TOKEN }} | docker login docker.pkg.github.com -u ${CI_USER} --password-stdin | |
| - name: Pull from cache (GitHub Package Registry) | |
| run: docker pull "docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG}" | |
| - name: Tag docker image | |
| run: docker tag docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG} ${DOCKER_TAG} | |
| - name: Normal tests | |
| run: | | |
| ci_env="$(bash <(curl -s https://codecov.io/env))" && | |
| docker run \ | |
| -e GITHUB_ACTIONS `# used by codecov` \ | |
| -e CODECOV_TOKEN \ | |
| $ci_env\ | |
| --memory 6500m \ | |
| --memory-swap 6500m \ | |
| "${DOCKER_TAG}" \ | |
| /bin/bash -c \ | |
| '[ ! -f ${MJKEY_PATH} ] || mv ${MJKEY_PATH} ${MJKEY_PATH}.bak && | |
| pytest --cov=garage --cov-report=xml --reruns 1 -m \ | |
| "not gpu and not nightly and not huge and not flaky and not large and not mujoco and not mujoco_long" --durations=20 && | |
| for i in {1..5}; do | |
| bash <(curl -s https://codecov.io/bash --retry 5) -Z && break | |
| if [ $i == 5 ]; then | |
| exit 0 | |
| else | |
| echo "Retry ${i}..." | |
| sleep 30 | |
| fi | |
| done' | |
| - name: Update check status for PRs from forks | |
| uses: LouisBrunner/[email protected] | |
| if: env.IS_PR_FROM_FORK && always() | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| name: Normal Tests | |
| sha: ${{ github.event.client_payload.pull_request.head.sha }} | |
| conclusion: ${{ job.status }} | |
| large_test: | |
| name: Large Tests | |
| runs-on: ubuntu-latest | |
| needs: check_pre_commit | |
| steps: | |
| - name: Login to GitHub Package Registry | |
| run: echo ${{ secrets.CI_REGISTRY_TOKEN }} | docker login docker.pkg.github.com -u ${CI_USER} --password-stdin | |
| - name: Pull from cache (GitHub Package Registry) | |
| run: docker pull "docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG}" | |
| - name: Tag docker image | |
| run: docker tag docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG} ${DOCKER_TAG} | |
| - name: Large tests | |
| run: | | |
| ci_env="$(bash <(curl -s https://codecov.io/env))" && | |
| docker run \ | |
| -e GITHUB_ACTIONS \ | |
| -e CODECOV_TOKEN \ | |
| $ci_env\ | |
| --memory 6500m \ | |
| --memory-swap 6500m \ | |
| "${DOCKER_TAG}" \ | |
| /bin/bash -c \ | |
| '[ ! -f ${MJKEY_PATH} ] || mv ${MJKEY_PATH} ${MJKEY_PATH}.bak && | |
| pytest --cov=garage --cov-report=xml --reruns 1 -m "large and not flaky" --durations=20 && | |
| for i in {1..5}; do | |
| bash <(curl -s https://codecov.io/bash --retry 5) -Z && break | |
| if [ $i == 5 ]; then | |
| exit 0 | |
| else | |
| echo "Retry ${i}..." | |
| sleep 30 | |
| fi | |
| done' | |
| - name: Update check status for PRs from forks | |
| uses: LouisBrunner/[email protected] | |
| if: env.IS_PR_FROM_FORK && always() | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| name: Large Tests | |
| sha: ${{ github.event.client_payload.pull_request.head.sha }} | |
| conclusion: ${{ job.status }} | |
| mujoco_test: | |
| name: MuJoCo-Based Tests | |
| runs-on: ubuntu-latest | |
| needs: check_pre_commit | |
| steps: | |
| - name: Login to GitHub Package Registry | |
| run: echo ${{ secrets.CI_REGISTRY_TOKEN }} | docker login docker.pkg.github.com -u ${CI_USER} --password-stdin | |
| - name: Pull from cache (GitHub Package Registry) | |
| run: docker pull "docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG}" | |
| - name: Tag docker image | |
| run: docker tag docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG} ${DOCKER_TAG} | |
| - name: MuJoCo tests | |
| run: | | |
| ci_env="$(bash <(curl -s https://codecov.io/env))" && | |
| docker run \ | |
| -e MJKEY \ | |
| -e GITHUB_ACTIONS \ | |
| -e CODECOV_TOKEN \ | |
| $ci_env\ | |
| --memory 6500m \ | |
| --memory-swap 6500m \ | |
| "${DOCKER_TAG}" \ | |
| /bin/bash -c \ | |
| 'pytest --cov=garage --cov-report=xml --reruns 1 -m "mujoco and not flaky" --durations=20 && | |
| for i in {1..5}; do | |
| bash <(curl -s https://codecov.io/bash --retry 5) -Z && break | |
| if [ $i == 5 ]; then | |
| exit 0 | |
| else | |
| echo "Retry ${i}..." | |
| sleep 30 | |
| fi | |
| done' | |
| - name: Update check status for PRs from forks | |
| uses: LouisBrunner/[email protected] | |
| if: env.IS_PR_FROM_FORK && always() | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| name: MuJoCo-Based Tests | |
| sha: ${{ github.event.client_payload.pull_request.head.sha }} | |
| conclusion: ${{ job.status }} | |
| mujoco_test_long: | |
| name: Large MuJoCo-Based Tests | |
| runs-on: ubuntu-latest | |
| needs: check_pre_commit | |
| steps: | |
| - name: Login to GitHub Package Registry | |
| run: echo ${{ secrets.CI_REGISTRY_TOKEN }} | docker login docker.pkg.github.com -u ${CI_USER} --password-stdin | |
| - name: Pull from cache (GitHub Package Registry) | |
| run: docker pull "docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG}" | |
| - name: Tag docker image | |
| run: docker tag docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG} ${DOCKER_TAG} | |
| - name: Large MuJoCo tests | |
| run: | | |
| ci_env="$(bash <(curl -s https://codecov.io/env))" && | |
| docker run \ | |
| -e MJKEY \ | |
| -e GITHUB_ACTIONS \ | |
| -e CODECOV_TOKEN \ | |
| $ci_env\ | |
| --memory 6500m \ | |
| --memory-swap 6500m \ | |
| "${DOCKER_TAG}" \ | |
| /bin/bash -c \ | |
| 'pytest --cov=garage --cov-report=xml --reruns 1 -m "mujoco_long and not flaky" --durations=20 && | |
| for i in {1..5}; do | |
| bash <(curl -s https://codecov.io/bash --retry 5) -Z && break | |
| if [ $i == 5 ]; then | |
| exit 0 | |
| else | |
| echo "Retry ${i}..." | |
| sleep 30 | |
| fi | |
| done' | |
| - name: Update check status for PRs from forks | |
| uses: LouisBrunner/[email protected] | |
| if: env.IS_PR_FROM_FORK && always() | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| name: Large MuJoCo-Based Tests | |
| sha: ${{ github.event.client_payload.pull_request.head.sha }} | |
| conclusion: ${{ job.status }} | |
| nightly_test: | |
| name: Nightly Tests | |
| runs-on: ubuntu-latest | |
| needs: build_docker_container | |
| if: github.event_name == 'schedule' | |
| steps: | |
| - name: Login to GitHub Package Registry | |
| run: echo ${{ secrets.CI_REGISTRY_TOKEN }} | docker login docker.pkg.github.com -u ${CI_USER} --password-stdin | |
| - name: Pull from cache (GitHub Package Registry) | |
| run: docker pull "docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG}" | |
| - name: Tag docker image | |
| run: docker tag docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG} ${DOCKER_TAG} | |
| - name: Nightly tests | |
| run: | | |
| docker run \ | |
| -e MJKEY \ | |
| --memory 6500m \ | |
| --memory-swap 6500m \ | |
| "${DOCKER_TAG}" pytest -v --reruns 1 -m nightly | |
| verify_envs_conda: | |
| name: Verify Conda Environment Installation | |
| runs-on: ubuntu-latest | |
| needs: build_docker_container | |
| if: github.event_name == 'schedule' | |
| steps: | |
| - name: Login to GitHub Package Registry | |
| run: echo ${{ secrets.CI_REGISTRY_TOKEN }} | docker login docker.pkg.github.com -u ${CI_USER} --password-stdin | |
| - name: Pull from cache (GitHub Package Registry) | |
| run: docker pull "docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG}" | |
| - name: Tag docker image | |
| run: docker tag docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG} ${DOCKER_TAG} | |
| - name: Verify Conda | |
| run: | | |
| docker run \ | |
| -e MJKEY \ | |
| --memory 6500m \ | |
| --memory-swap 6500m \ | |
| "${DOCKER_TAG}" \ | |
| /bin/bash -c \ | |
| 'CONDA_ROOT=$HOME/miniconda \ | |
| CONDA=${CONDA_ROOT}/bin/conda \ | |
| GARAGE_BIN=${CONDA_ROOT}/envs/garage-ci/bin; \ | |
| touch ${MJKEY_PATH} && \ | |
| wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \ | |
| bash miniconda.sh -b -p ${CONDA_ROOT} && \ | |
| hash -r && \ | |
| ${CONDA} config --set always_yes yes --set changeps1 no && \ | |
| ${CONDA} install -c anaconda setuptools && \ | |
| ${CONDA} update -q conda && \ | |
| ${CONDA} init && \ | |
| ${CONDA} info -a && \ | |
| ${CONDA} create -n garage-ci python=3.6 pip -y && \ | |
| ${GARAGE_BIN}/pip install --upgrade pip setuptools && \ | |
| ${GARAGE_BIN}/pip install dist/garage.tar.gz[all,dev] && \ | |
| ${GARAGE_BIN}/pylint --disable=all --enable=import-error garage' | |
| verify_envs_pipenv: | |
| name: Verify Pipenv Environment Installation | |
| runs-on: ubuntu-latest | |
| needs: build_docker_container | |
| # For unclear reasons this step takes over six hours on GitHub Actions. | |
| # To enable the CI to run, it is disabled for now. | |
| # if: github.event_name == 'schedule' | |
| if: false | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Login to GitHub Package Registry | |
| run: echo ${{ secrets.CI_REGISTRY_TOKEN }} | docker login docker.pkg.github.com -u ${CI_USER} --password-stdin | |
| - name: Pull from cache (GitHub Package Registry) | |
| run: docker pull "docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG}" | |
| - name: Tag docker image | |
| run: docker tag docker.pkg.github.com/${OWNER}/${DOCKER_CACHE_REPO}/${DOCKER_TAG} ${DOCKER_TAG} | |
| - name: Verify Pipenv | |
| run: | | |
| docker run \ | |
| -e MJKEY \ | |
| --memory 6500m \ | |
| --memory-swap 6500m \ | |
| "${DOCKER_TAG}" \ | |
| /bin/bash -c \ | |
| "export PATH=\$PATH_NO_VENV && \ | |
| export VIRTUAL_ENV= && \ | |
| export PIPENV_MAX_RETRIES=2 && \ | |
| touch \${MJKEY_PATH} && \ | |
| pip3 install --upgrade pip setuptools && \ | |
| pip3 install pipenv && \ | |
| pipenv --python=3.6 && \ | |
| pipenv install dist/garage.tar.gz[all,dev] && \ | |
| pipenv graph && \ | |
| # pylint will verify all imports work | |
| pipenv run pylint --disable=all --enable=import-error garage" | |
| deploy_to_pypi: | |
| name: Deploy to PyPI | |
| runs-on: ubuntu-latest | |
| needs: [normal_test, large_test, mujoco_test, mujoco_test_long, doctest] | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.6 | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: 3.6 | |
| - name: Update Version | |
| run: echo ${GITHUB_REF##*/} > VERSION | |
| - name: Setup Distribution | |
| run: >- | |
| python setup.py sdist | |
| - name: Deploy to PyPI | |
| uses: pypa/gh-action-pypi-publish@master | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_TOKEN }} | |
| delete_docker_container: | |
| name: Delete Docker Container | |
| runs-on: ubuntu-latest | |
| needs: [build_docker_container, normal_test, large_test, mujoco_test, mujoco_test_long, nightly_test, verify_envs_conda, verify_envs_pipenv] | |
| # There's currently no working version of this action available, so disable it and delete packages manually | |
| if: ${{ false && always() && needs.build_docker_container.result == 'success' }} | |
| steps: | |
| - uses: actions/delete-package-versions@v2 | |
| with: | |
| owner: ${{ env.OWNER }} | |
| repo: ${{ env.DOCKER_CACHE_REPO }} | |
| package-name: ${{ env.DOCKER_TAG }} | |
| token: ${{ secrets.CI_REGISTRY_TOKEN }} |