From 256e1ab2b3c8735f52711f94a3dd799b48b93765 Mon Sep 17 00:00:00 2001 From: Paolo Sottovia Date: Wed, 14 Aug 2024 08:34:35 +0000 Subject: [PATCH] Initial steps for version 4.0.0-preview1 --- .github/workflows/build.yaml | 8 +++--- .github/workflows/integration.yaml | 38 ++++++++++++++--------------- .github/workflows/publish.yaml | 6 ++--- images/charmed-spark/rockcraft.yaml | 12 +++------ 4 files changed, 30 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5202d6cf..6245f64b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -44,8 +44,8 @@ jobs: - name: Build image (Jupyter) run: sudo make build FLAVOUR=jupyter - - name: Build image (Kyuubi) - run: sudo make build FLAVOUR=kyuubi + # - name: Build image (Kyuubi) + # run: sudo make build FLAVOUR=kyuubi - name: Get Artifact Name id: artifact @@ -54,8 +54,8 @@ jobs: echo "base_artifact_name=${BASE_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT JUPYTER_ARTIFACT=$(make help FLAVOUR=jupyter | grep 'Artifact: ') echo "jupyter_artifact_name=${JUPYTER_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT - KYUUBI_ARTIFACT=$(make help FLAVOUR=kyuubi | grep 'Artifact: ') - echo "kyuubi_artifact_name=${KYUUBI_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT + # KYUUBI_ARTIFACT=$(make help FLAVOUR=kyuubi | grep 'Artifact: ') + # echo "kyuubi_artifact_name=${KYUUBI_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT - name: Change artifact permissions run: sudo chmod a+r ${{ steps.artifact.outputs.base_artifact_name }} ${{ steps.artifact.outputs.jupyter_artifact_name }} ${{ steps.artifact.outputs.kyuubi_artifact_name }} diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index b193ce06..974a847a 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -34,10 +34,10 @@ jobs: echo "base_artifact_name=${BASE_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT JUPYTER_ARTIFACT=$(make help FLAVOUR=jupyter | grep 'Artifact: ') echo "jupyter_artifact_name=${JUPYTER_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT - KYUUBI_ARTIFACT=$(make help FLAVOUR=kyuubi | grep 'Artifact: ') - echo "kyuubi_artifact_name=${KYUUBI_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT - GPU_ARTIFACT=$(make help FLAVOUR=spark-gpu | grep 'Artifact: ') - echo "gpu_artifact_name=${GPU_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT + # KYUUBI_ARTIFACT=$(make help FLAVOUR=kyuubi | grep 'Artifact: ') + # echo "kyuubi_artifact_name=${KYUUBI_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT + # GPU_ARTIFACT=$(make help FLAVOUR=spark-gpu | grep 'Artifact: ') + # echo "gpu_artifact_name=${GPU_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT - name: Install and configure microk8s run: | @@ -75,20 +75,20 @@ jobs: sg microk8s -c "make tests FLAVOUR=jupyter" - - name: Run tests (Kyuubi) - env: - AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }} - AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }} - run: | - # Unpack Artifact - mv charmed-spark/${{ steps.artifact.outputs.kyuubi_artifact_name }} . - rmdir charmed-spark + # - name: Run tests (Kyuubi) + # env: + # AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }} + # AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }} + # run: | + # # Unpack Artifact + # mv charmed-spark/${{ steps.artifact.outputs.kyuubi_artifact_name }} . + # rmdir charmed-spark - # Import artifact into docker with new tag - sudo make microk8s-import \ - FLAVOUR=kyuubi \ - TAG=$(yq .version images/charmed-spark/rockcraft.yaml) \ - REPOSITORY=ghcr.io/canonical/ PREFIX=test- \ - -o ${{ steps.artifact.outputs.kyuubi_artifact_name }} + # # Import artifact into docker with new tag + # sudo make microk8s-import \ + # FLAVOUR=kyuubi \ + # TAG=$(yq .version images/charmed-spark/rockcraft.yaml) \ + # REPOSITORY=ghcr.io/canonical/ PREFIX=test- \ + # -o ${{ steps.artifact.outputs.kyuubi_artifact_name }} - sg microk8s -c "make tests FLAVOUR=kyuubi" + # sg microk8s -c "make tests FLAVOUR=kyuubi" diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 3d7bc8ad..fd2ac0f3 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -47,9 +47,9 @@ jobs: uses: ./.github/workflows/integration.yaml secrets: inherit - tests-gpu: - uses: ./.github/workflows/integration-gpu.yaml - secrets: inherit + # tests-gpu: + # uses: ./.github/workflows/integration-gpu.yaml + # secrets: inherit publish: needs: [tests, release_checks, tests-gpu] diff --git a/images/charmed-spark/rockcraft.yaml b/images/charmed-spark/rockcraft.yaml index 765c122c..496da6cf 100644 --- a/images/charmed-spark/rockcraft.yaml +++ b/images/charmed-spark/rockcraft.yaml @@ -10,7 +10,7 @@ description: | license: Apache-2.0 -version: "3.5.1" +version: "4.0.0-preview1" base: ubuntu@22.04 @@ -51,8 +51,8 @@ services: parts: spark: plugin: dump - source: https://github.com/canonical/central-uploader/releases/download/spark-3.5.1-ubuntu1/spark-3.5.1-ubuntu1-20240801193221-bin-k8s.tgz - source-checksum: sha512/4c9ec4dc3f67602ec9f83fa08486985647b8ab7eaf3748c3042a382ed367fc776b7623a067ceef408337cf5c2171a010da0ff063dcbf76c4b84674b661d2330b + source: https://github.com/canonical/central-uploader/releases/download/spark-4.0.0-preview1-ubuntu0/spark-4.0.0-preview1-ubuntu0-20240813100410-bin-k8s.tgz + source-checksum: sha512/9d506d28d356c33608bebaf53dd6b60705826f60764bb6ed60d1b5cf3f496d99bf45b6aaee5df6f7cce0e37dad04ab003f45e8219e15dc918a2cef78d8425c09 overlay-script: | sed -i 's/http:\/\/deb.\(.*\)/https:\/\/deb.\1/g' /etc/apt/sources.list apt-get update @@ -96,16 +96,12 @@ parts: mkdir -p $CRAFT_PART_INSTALL/opt/spark/jars cd $CRAFT_PART_INSTALL/opt/spark/jars - ICEBERG_SPARK_RUNTIME_VERSION='3.4_2.12' - ICEBERG_VERSION='1.4.3' SPARK_METRICS_VERSION='3.4-1.0.1' SERVLET_FILTERS_VERSION='0.0.1' - SHA1SUM_ICEBERG_JAR='48d553e4e5496f731b9e0e6adb5bc0fd040cb0df' SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR='493cf77133cbf03e96fb848121ce10ac16e6f907f595df637649b98b42118e57d6b6e1bdab71bfee3394eb369637c5b4f6b05dd8fa30a1ff6899e74069c972ce' SHA512SUM_SPARK_SERVLET_FILTER_JAR='ffeb809d58ef0151d513b09d4c2bfd5cc064b0b888ca45899687aed2f42bcb1ce9834be9709290dd70bd9df84049f02cbbff6c2d5ec3c136c278c93f167c8096' JARS=( - "https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/LIB_VERSION/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-LIB_VERSION.jar $ICEBERG_VERSION sha1sum $SHA1SUM_ICEBERG_JAR" "https://github.com/canonical/central-uploader/releases/download/spark-metrics-assembly-LIB_VERSION/spark-metrics-assembly-LIB_VERSION.jar $SPARK_METRICS_VERSION sha512sum $SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR" "https://github.com/canonical/central-uploader/releases/download/servlet-filters-LIB_VERSION/servlet-filters-LIB_VERSION.jar $SERVLET_FILTERS_VERSION sha512sum $SHA512SUM_SPARK_SERVLET_FILTER_JAR" ) @@ -139,7 +135,7 @@ parts: - python3-pip overlay-script: | mkdir -p $CRAFT_PART_INSTALL/opt/spark8t/python/dist - pip install --target=${CRAFT_PART_INSTALL}/opt/spark8t/python/dist https://github.com/canonical/spark-k8s-toolkit-py/releases/download/v0.0.7/spark8t-0.0.7-py3-none-any.whl + pip install --target=${CRAFT_PART_INSTALL}/opt/spark8t/python/dist https://github.com/canonical/spark-k8s-toolkit-py/releases/download/v0.0.10/spark8t-0.0.10-py3-none-any.whl rm usr/bin/pip* stage: - opt/spark8t/python/dist