ci.yaml

name: CI

on:
  push:
    branches:
      - dev
      - master
  pull_request: ~

env:
  CACHE_VERSION: 1
  DEFAULT_PYTHON: "3.10"
  PRE_COMMIT_CACHE: ~/.cache/pre-commit
  DOCKER_BUILDKIT: 1
  COMPOSE_DOCKER_CLI_BUILD: 1
  BUILDKIT_PROGRESS: plain

jobs:
  prepare-python-venv:
    name: Prepare Python venv
    runs-on: ubuntu-22.04
    outputs:
      python-venv-cache-key: ${{ steps.python-venv-cache-key.outputs.key }}
    steps:
      # Create Python virtual env used for linters
      - name: Check out code from GitHub
        uses: actions/checkout@v3.5.2
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        id: python
        uses: actions/setup-python@v4.3.0
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
      # Create a virtual environment and cache it
      - name: Generate Python virtual environment restore key
        id: python-venv-cache-key
        env:
          cache-name: cache-python-venv
        run: >-
          echo "key=${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}-${{ env.CACHE_VERSION }}-${{ hashFiles('.pre-commit-config.yaml') }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements_test.txt') }}-${{ hashFiles('requirements_ci.txt') }}" >> $GITHUB_OUTPUT
      - name: Restore Python virtual environment
        id: cache-venv
        uses: actions/cache@v3.0.11
        with:
          path: venv
          key: ${{ steps.python-venv-cache-key.outputs.key }}
      - name: Create Python virtual environment
        if: steps.cache-venv.outputs.cache-hit != 'true'
        # Install pycoral from Google Coral repo since it is not on PyPI
        run: |
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends libgirepository1.0-dev
          python3 -m venv --system-site-packages venv
      - name: Install requirements into Python virtual environment
        if: steps.cache-venv.outputs.cache-hit != 'true'
        uses: ./.github/templates/run_in_venv
        with:
          command: |
            pip3 install --extra-index-url https://google-coral.github.io/py-repo/ -r requirements.txt -r requirements_test.txt -r requirements_ci.txt && \
            pre-commit install

  prepare-pre-commit:
    name: Prepare pre-commit
    runs-on: ubuntu-22.04
    needs: prepare-python-venv
    outputs:
      pre-commit-cache-key: ${{ steps.pre-commit-cache-key.outputs.key }}
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.5.2
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v4.3.0
        id: python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache@v3.0.11
        with:
          path: venv
          key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Generate pre-commit restore key
        id: pre-commit-cache-key
        env:
          cache-name: cache-pre-commit
        run: >-
          echo "key=${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}-${{ env.CACHE_VERSION }}-${{ hashFiles('.pre-commit-config.yaml') }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements_test.txt') }}-${{ hashFiles('requirements_ci.txt') }}" >> $GITHUB_OUTPUT
      # Install pre-commit hooks into the cached virtual environment
      - name: Restore pre-commit hooks
        id: cache-pre-commit
        uses: actions/cache@v3.0.11
        with:
          path: ${{ env.PRE_COMMIT_CACHE }}
          key: ${{ steps.pre-commit-cache-key.outputs.key }}
      - name: Create pre-commit hooks
        if: steps.cache-pre-commit.outputs.cache-hit != 'true'
        uses: ./.github/templates/run_in_venv
        with:
          command: |
            pre-commit install-hooks

  run-isort:
    name: Run isort
    runs-on: ubuntu-22.04
    needs: [prepare-python-venv, prepare-pre-commit]
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.5.2
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v4.3.0
        id: python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache@v3.0.11
        with:
          path: venv
          key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Restore pre-commit environment from cache
        id: cache-pre-commit
        uses: actions/cache@v3.0.11
        with:
          path: ${{ env.PRE_COMMIT_CACHE }}
          key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
      - name: Fail job if pre-commit cache restore failed
        if: steps.cache-pre-commit.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore pre-commit environment from cache"
          exit 1
      - name: Run isort
        uses: ./.github/templates/run_in_venv
        with:
          command: |
            pre-commit run --hook-stage manual isort --all-files --show-diff-on-failure

  run-black:
    name: Run black
    runs-on: ubuntu-22.04
    needs: [prepare-python-venv, prepare-pre-commit]
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.5.2
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v4.3.0
        id: python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache@v3.0.11
        with:
          path: venv
          key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Restore pre-commit environment from cache
        id: cache-pre-commit
        uses: actions/cache@v3.0.11
        with:
          path: ${{ env.PRE_COMMIT_CACHE }}
          key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
      - name: Fail job if pre-commit cache restore failed
        if: steps.cache-pre-commit.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore pre-commit environment from cache"
          exit 1
      - name: Run black
        uses: ./.github/templates/run_in_venv
        with:
          command: |
            pre-commit run --hook-stage manual black --all-files --show-diff-on-failure

  run-codespell:
    name: Run codespell
    runs-on: ubuntu-22.04
    needs: [prepare-python-venv, prepare-pre-commit]
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.5.2
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v4.3.0
        id: python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache@v3.0.11
        with:
          path: venv
          key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Restore pre-commit environment from cache
        id: cache-pre-commit
        uses: actions/cache@v3.0.11
        with:
          path: ${{ env.PRE_COMMIT_CACHE }}
          key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
      - name: Fail job if pre-commit cache restore failed
        if: steps.cache-pre-commit.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore pre-commit environment from cache"
          exit 1
      - name: Run codespell
        uses: ./.github/templates/run_in_venv
        with:
          command: |
            pre-commit run --hook-stage manual codespell --all-files --show-diff-on-failure

  run-mypy:
    name: Run mypy
    runs-on: ubuntu-22.04
    needs: [prepare-python-venv, prepare-pre-commit]
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.5.2
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v4.3.0
        id: python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache@v3.0.11
        with:
          path: venv
          key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Restore pre-commit environment from cache
        id: cache-pre-commit
        uses: actions/cache@v3.0.11
        with:
          path: ${{ env.PRE_COMMIT_CACHE }}
          key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
      - name: Fail job if pre-commit cache restore failed
        if: steps.cache-pre-commit.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore pre-commit environment from cache"
          exit 1
      - name: Run mypy
        uses: ./.github/templates/run_in_venv
        with:
          command: |
            pre-commit run --hook-stage manual mypy --all-files --show-diff-on-failure

  run-pylint:
    name: Run pylint
    runs-on: ubuntu-22.04
    needs: [prepare-python-venv, prepare-pre-commit]
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.5.2
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v4.3.0
        id: python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache@v3.0.11
        with:
          path: venv
          key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Restore pre-commit environment from cache
        id: cache-pre-commit
        uses: actions/cache@v3.0.11
        with:
          path: ${{ env.PRE_COMMIT_CACHE }}
          key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
      - name: Fail job if pre-commit cache restore failed
        if: steps.cache-pre-commit.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore pre-commit environment from cache"
          exit 1
      - name: Run pylint
        uses: ./.github/templates/run_in_venv
        with:
          command: |
            pylint --version
            pylint viseron

  run-flake8:
    name: Run flake8
    runs-on: ubuntu-22.04
    needs: [prepare-python-venv, prepare-pre-commit]
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.5.2
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v4.3.0
        id: python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache@v3.0.11
        with:
          path: venv
          key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Restore pre-commit environment from cache
        id: cache-pre-commit
        uses: actions/cache@v3.0.11
        with:
          path: ${{ env.PRE_COMMIT_CACHE }}
          key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
      - name: Fail job if pre-commit cache restore failed
        if: steps.cache-pre-commit.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore pre-commit environment from cache"
          exit 1
      - name: Run flake8
        uses: ./.github/templates/run_in_venv
        with:
          command: |
            pre-commit run --hook-stage manual flake8 --all-files --show-diff-on-failure

  run-pyupgrade:
    name: Run pyupgrade
    runs-on: ubuntu-22.04
    needs: [prepare-python-venv, prepare-pre-commit]
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.5.2
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v4.3.0
        id: python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache@v3.0.11
        with:
          path: venv
          key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Restore pre-commit environment from cache
        id: cache-pre-commit
        uses: actions/cache@v3.0.11
        with:
          path: ${{ env.PRE_COMMIT_CACHE }}
          key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
      - name: Fail job if pre-commit cache restore failed
        if: steps.cache-pre-commit.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore pre-commit environment from cache"
          exit 1
      - name: Run pyupgrade
        uses: ./.github/templates/run_in_venv
        with:
          command: |
            pre-commit run --hook-stage manual pyupgrade --all-files --show-diff-on-failure

  run-pytest:
    name: Run pytest
    runs-on: ubuntu-22.04
    needs: prepare-python-venv
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.5.2
        with:
          fetch-depth: 2
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v4.3.0
        id: python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache@v3.0.11
        with:
          path: venv
          key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Pull current Docker dev tag
        run: |
          docker-compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env pull amd64-viseron
          docker-compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env pull amd64-wheels
      - name: Re-build wheels
        run: |
          docker-compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env build --build-arg BUILDKIT_INLINE_CACHE=1 amd64-wheels
          docker-compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env build --build-arg BUILDKIT_INLINE_CACHE=1 amd64-viseron
      - name: Build pytest Docker image
        run: |
          docker-compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env build --build-arg BUILDKIT_INLINE_CACHE=1 amd64-viseron-tests
      - name: Run pytest
        run: |
          if docker-compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env up amd64-viseron-tests; then
            exit 0
          else
            exit 1
          fi
      - name: Copy .coverage to host
        run: |
          docker cp amd64-viseron-tests:/src/coverage.xml coverage.xml
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3

  run-generated-docs:
    name: Check generated docs
    runs-on: ubuntu-22.04
    needs: [prepare-python-venv, prepare-pre-commit]
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.5.2
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v4.3.0
        id: python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache@v3.0.11
        with:
          path: venv
          key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Restore pre-commit environment from cache
        id: cache-pre-commit
        uses: actions/cache@v3.0.11
        with:
          path: ${{ env.PRE_COMMIT_CACHE }}
          key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
      - name: Fail job if pre-commit cache restore failed
        if: steps.cache-pre-commit.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore pre-commit environment from cache"
          exit 1
      - name: Install libedgetpu1 for edgetpu docs
        run: |
          curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
          sudo add-apt-repository "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main"
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends libedgetpu1-std python3-gi python3-gst-1.0
      - name: Run script to check generated docs
        uses: ./.github/templates/run_in_venv
        with:
          command: |
            pre-commit run --hook-stage manual generate_docs --all-files --show-diff-on-failure
            pre-commit run --hook-stage manual check_config_json --all-files --show-diff-on-failure