integration.yaml

name: Spark Image Tests

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  pull_request:
  workflow_call:

jobs:
  build-rock:
    uses: ./.github/workflows/build.yaml
  integration:
    runs-on: ubuntu-latest
    timeout-minutes: 120
    needs: build-rock
    strategy:
      matrix:
        env: [integration]
        k8s_version: ["1.28-strict/stable","1.29-strict/stable", "1.30-strict/stable" ]
      fail-fast: false
    env:
      AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }}
      AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Get Artifact Name
        id: artifact
        run: |
          BASE_ARTIFACT=$(make help | grep 'Artifact: ')
          echo "base_artifact_name=${BASE_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
          JUPYTER_ARTIFACT=$(make help FLAVOUR=jupyter | grep 'Artifact: ')
          echo "jupyter_artifact_name=${JUPYTER_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
          KYUUBI_ARTIFACT=$(make help FLAVOUR=kyuubi | grep 'Artifact: ')
          echo "kyuubi_artifact_name=${KYUUBI_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
          GPU_ARTIFACT=$(make help FLAVOUR=spark-gpu | grep 'Artifact: ')
          echo "gpu_artifact_name=${GPU_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT

      - name: Install and configure microk8s
        run: |
          make microk8s-setup MICROK8S_CHANNEL=${{ matrix.k8s_version }}

      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          name: charmed-spark
          path: charmed-spark

      - name: Run tests
        run: |
          # Unpack Artifact
          mv charmed-spark/${{ steps.artifact.outputs.base_artifact_name }} .

          # Import artifact into microk8s to be used in integration tests
          sudo make microk8s-import PREFIX=test- REPOSITORY=ghcr.io/canonical/ \
            -o ${{ steps.artifact.outputs.base_artifact_name }}
          
          sg snap_microk8s -c "make tests"

      - name: Run tests (Jupyter)
        run: |
          # Unpack Artifact
          mv charmed-spark/${{ steps.artifact.outputs.jupyter_artifact_name }} .

          # Import artifact into docker with new tag
          sudo make microk8s-import \
            FLAVOUR=jupyter \
            TAG=$(yq .version images/charmed-spark/rockcraft.yaml) \
            REPOSITORY=ghcr.io/canonical/ \
            PREFIX=test- \
            -o ${{ steps.artifact.outputs.jupyter_artifact_name }}

          sg snap_microk8s -c "make tests FLAVOUR=jupyter"

      - name: Run tests (Kyuubi)
        env:
          AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }}
          AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }}
        run: |
          # Unpack Artifact
          mv charmed-spark/${{ steps.artifact.outputs.kyuubi_artifact_name }} .
          rmdir charmed-spark

          # Import artifact into docker with new tag
          sudo make microk8s-import \
            FLAVOUR=kyuubi \
            TAG=$(yq .version images/charmed-spark/rockcraft.yaml) \
            REPOSITORY=ghcr.io/canonical/ PREFIX=test- \
            -o ${{ steps.artifact.outputs.kyuubi_artifact_name }}

          sg snap_microk8s -c "make tests FLAVOUR=kyuubi"