diff --git a/.github/workflows/api-binary-compatibility.yml b/.github/workflows/api-binary-compatibility.yml deleted file mode 100644 index fbdb740e6e72..000000000000 --- a/.github/workflows/api-binary-compatibility.yml +++ /dev/null @@ -1,66 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "API Binary Compatibility Checks" -on: - push: - branches: - - 'main' - - '0.*' - - '1.*' - - '2.*' - tags: - - 'apache-iceberg-**' - pull_request: - paths: - - '.github/workflows/api-binary-compatibility.yml' - - '*.gradle' - - 'gradle*' - - 'api/**' - - '.palantir/revapi.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - revapi: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - with: - # fetch-depth of zero ensures that the tags are pulled in and we're not in a detached HEAD state - # revapi depends on the tags, specifically the tag from git describe, to find the relevant override - # in the .palantir/revapi.yml file - # - # See https://github.com/actions/checkout/issues/124 - fetch-depth: 0 - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: 11 - - run: | - echo "Using the old version tag, as per git describe, of $(git describe)"; - - run: ./gradlew revapi --rerun-tasks - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test logs - path: | - **/build/testlogs diff --git a/.github/workflows/delta-conversion-ci.yml b/.github/workflows/delta-conversion-ci.yml deleted file mode 100644 index 521d061f6552..000000000000 --- a/.github/workflows/delta-conversion-ci.yml +++ /dev/null @@ -1,127 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Delta Conversion CI" -on: - push: - branches: - - 'main' - - '0.*' - - '1.*' - - '2.*' - tags: - - 'apache-iceberg-**' - pull_request: - paths-ignore: - - '.github/ISSUE_TEMPLATE/**' - - '.github/workflows/api-binary-compatibility.yml' - - '.github/workflows/flink-ci.yml' - - '.github/workflows/hive-ci.yml' - - '.github/workflows/java-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' - - '.github/workflows/kafka-connect-ci.yml' - - '.github/workflows/labeler.yml' - - '.github/workflows/licence-check.yml' - - '.github/workflows/open-api.yml' - - '.github/workflows/publish-snapshot.yml' - - '.github/workflows/recurring-jmh-benchmarks.yml' - - '.github/workflows/site-ci.yml' - - '.github/workflows/spark-ci.yml' - - '.github/workflows/stale.yml' - - '.gitignore' - - '.asf.yml' - - 'dev/**' - - 'mr/**' - - 'hive3/**' - - 'hive3-orc-bundle/**' - - 'hive-runtime/**' - - 'flink/**' - - 'kafka-connect/**' - - 'docs/**' - - 'site/**' - - 'open-api/**' - - 'format/**' - - '.gitattributes' - - 'README.md' - - 'CONTRIBUTING.md' - - 'LICENSE' - - 'NOTICE' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - delta-conversion-scala-2-12-tests: - runs-on: ubuntu-22.04 - strategy: - matrix: - jvm: [11, 17, 21] - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: ${{ matrix.jvm }} - - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- - - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - run: ./gradlew -DsparkVersions=3.5 -DscalaVersion=2.12 -DhiveVersions= -DkafkaVersions= -DflinkVersions= :iceberg-delta-lake:check -Pquick=true -x javadoc - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test logs - path: | - **/build/testlogs - - delta-conversion-scala-2-13-tests: - runs-on: ubuntu-22.04 - strategy: - matrix: - jvm: [11, 17, 21] - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: ${{ matrix.jvm }} - - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- - - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - run: ./gradlew -DsparkVersions=3.5 -DscalaVersion=2.13 -DhiveVersions= -DkafkaVersions= -DflinkVersions= :iceberg-delta-lake:check -Pquick=true -x javadoc - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test logs - path: | - **/build/testlogs diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml deleted file mode 100644 index 22f4f008a215..000000000000 --- a/.github/workflows/flink-ci.yml +++ /dev/null @@ -1,101 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Flink CI" -on: - push: - branches: - - 'main' - - '0.*' - - '1.*' - - '2.*' - tags: - - 'apache-iceberg-**' - pull_request: - paths-ignore: - - '.github/ISSUE_TEMPLATE/**' - - '.github/workflows/api-binary-compatibility.yml' - - '.github/workflows/delta-conversion-ci.yml' - - '.github/workflows/hive-ci.yml' - - '.github/workflows/java-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' - - '.github/workflows/kafka-connect-ci.yml' - - '.github/workflows/labeler.yml' - - '.github/workflows/licence-check.yml' - - '.github/workflows/open-api.yml' - - '.github/workflows/publish-snapshot.yml' - - '.github/workflows/recurring-jmh-benchmarks.yml' - - '.github/workflows/site-ci.yml' - - '.github/workflows/spark-ci.yml' - - '.github/workflows/stale.yml' - - '.gitignore' - - '.asf.yml' - - 'dev/**' - - 'mr/**' - - 'hive3/**' - - 'hive3-orc-bundle/**' - - 'hive-runtime/**' - - 'kafka-connect/**' - - 'spark/**' - - 'docs/**' - - 'site/**' - - 'open-api/**' - - 'format/**' - - '.gitattributes' - - 'README.md' - - 'CONTRIBUTING.md' - - 'LICENSE' - - 'NOTICE' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - - # Test all flink versions with scala 2.12 for general validation. - flink-scala-2-12-tests: - runs-on: ubuntu-22.04 - strategy: - matrix: - jvm: [11, 17, 21] - flink: ['1.18', '1.19', '1.20'] - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: ${{ matrix.jvm }} - - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- - - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - run: ./gradlew -DsparkVersions= -DhiveVersions= -DkafkaVersions= -DflinkVersions=${{ matrix.flink }} :iceberg-flink:iceberg-flink-${{ matrix.flink }}:check :iceberg-flink:iceberg-flink-runtime-${{ matrix.flink }}:check -Pquick=true -x javadoc - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test logs - path: | - **/build/testlogs diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml deleted file mode 100644 index d95ca1bd5c6a..000000000000 --- a/.github/workflows/hive-ci.yml +++ /dev/null @@ -1,125 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Hive CI" -on: - push: - branches: - - 'main' - - '0.*' - - '1.*' - - '2.*' - tags: - - 'apache-iceberg-**' - pull_request: - paths-ignore: - - '.github/ISSUE_TEMPLATE/**' - - '.github/workflows/api-binary-compatibility.yml' - - '.github/workflows/delta-conversion-ci.yml' - - '.github/workflows/flink-ci.yml' - - '.github/workflows/java-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' - - '.github/workflows/kafka-connect-ci.yml' - - '.github/workflows/labeler.yml' - - '.github/workflows/licence-check.yml' - - '.github/workflows/open-api.yml' - - '.github/workflows/publish-snapshot.yml' - - '.github/workflows/recurring-jmh-benchmarks.yml' - - '.github/workflows/site-ci.yml' - - '.github/workflows/spark-ci.yml' - - '.github/workflows/stale.yml' - - '.gitignore' - - '.asf.yml' - - 'dev/**' - - 'arrow/**' - - 'spark/**' - - 'flink/**' - - 'kafka-connect/**' - - 'docs/**' - - 'site/**' - - 'open-api/**' - - 'format/**' - - '.gitattributes' - - 'README.md' - - 'CONTRIBUTING.md' - - 'LICENSE' - - 'NOTICE' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - hive2-tests: - runs-on: ubuntu-22.04 - strategy: - matrix: - jvm: [11, 17, 21] - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: ${{ matrix.jvm }} - - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- - - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - run: ./gradlew -DsparkVersions= -DhiveVersions=2 -DflinkVersions= -DkafkaVersions= -Pquick=true :iceberg-mr:check :iceberg-hive-runtime:check -x javadoc - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test logs - path: | - **/build/testlogs - - hive3-tests: - runs-on: ubuntu-22.04 - strategy: - matrix: - jvm: [11, 17, 21] - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: ${{ matrix.jvm }} - - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- - - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - run: ./gradlew -DsparkVersions= -DhiveVersions=3 -DflinkVersions= -DkafkaVersions= -Pquick=true :iceberg-hive3-orc-bundle:check :iceberg-hive3:check :iceberg-hive-runtime:check -x javadoc - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test logs - path: | - **/build/testlogs diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml deleted file mode 100644 index 1da7a673a865..000000000000 --- a/.github/workflows/java-ci.yml +++ /dev/null @@ -1,118 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Java CI" -on: - push: - branches: - - 'main' - - '0.*' - - '1.*' - - '2.*' - tags: - - 'apache-iceberg-**' - pull_request: - paths-ignore: - - '.github/ISSUE_TEMPLATE/**' - - '.github/workflows/api-binary-compatibility.yml' - - '.github/workflows/delta-conversion-ci.yml' - - '.github/workflows/flink-ci.yml' - - '.github/workflows/hive-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' - - '.github/workflows/kafka-connect-ci.yml' - - '.github/workflows/labeler.yml' - - '.github/workflows/licence-check.yml' - - '.github/workflows/open-api.yml' - - '.github/workflows/publish-snapshot.yml' - - '.github/workflows/recurring-jmh-benchmarks.yml' - - '.github/workflows/site-ci.yml' - - '.github/workflows/spark-ci.yml' - - '.github/workflows/stale.yml' - - '.gitignore' - - '.asf.yml' - - 'dev/**' - - 'docs/**' - - 'site/**' - - 'open-api/**' - - 'format/**' - - '.gitattributes' - - 'README.md' - - 'CONTRIBUTING.md' - - 'LICENSE' - - 'NOTICE' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - core-tests: - runs-on: ubuntu-22.04 - strategy: - matrix: - jvm: [11, 17, 21] - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: ${{ matrix.jvm }} - - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- - - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - run: ./gradlew check -DsparkVersions= -DhiveVersions= -DflinkVersions= -DkafkaVersions= -Pquick=true -x javadoc - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test logs - path: | - **/build/testlogs - - build-checks: - runs-on: ubuntu-22.04 - strategy: - matrix: - jvm: [11, 17, 21] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: ${{ matrix.jvm }} - - run: ./gradlew -DallModules build -x test -x javadoc -x integrationTest - - build-javadoc: - runs-on: ubuntu-22.04 - strategy: - matrix: - jvm: [11, 17, 21] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: ${{ matrix.jvm }} - - run: ./gradlew -Pquick=true javadoc diff --git a/.github/workflows/jmh-benchmarks.yml b/.github/workflows/jmh-benchmarks.yml deleted file mode 100644 index cfb53513e743..000000000000 --- a/.github/workflows/jmh-benchmarks.yml +++ /dev/null @@ -1,103 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "JMH Benchmarks" -on: - workflow_dispatch: - inputs: - repo: - description: 'Repo name with owner, such as apache/iceberg' - required: true - ref: - description: 'The branch name' - required: true - spark_version: - description: 'The spark project version to use, such as iceberg-spark-3.5' - default: 'iceberg-spark-3.5' - required: true - benchmarks: - description: 'A list of comma-separated double-quoted Benchmark names, such as "IcebergSourceFlatParquetDataReadBenchmark", "IcebergSourceFlatParquetDataFilterBenchmark"' - required: true - -jobs: - matrix: - runs-on: ubuntu-22.04 - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - foundlabel: ${{ steps.set-matrix.outputs.foundlabel }} - steps: - - uses: actions/checkout@v4 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.ref }} - - id: set-matrix - run: | - matrix=$(echo '[${{ github.event.inputs.benchmarks }}]' | jq '.[] | select(endswith("Benchmark")) | .') - matrix=$(echo $matrix | sed 's/ /,/g' | sed 's/"/\"/g') - echo "matrix=[$matrix]" >> $GITHUB_OUTPUT - echo "foundlabel=$(echo "[$matrix]" | jq 'if . | length > 0 then true else false end')" >> $GITHUB_OUTPUT - - show-matrix: - needs: matrix - runs-on: ubuntu-22.04 - steps: - - run: | - echo "Repo: ${{ github.event.inputs.repo }}" - echo "Ref: ${{ github.event.inputs.ref }}" - echo "Benchmarks: ${{ needs.matrix.outputs.matrix }}" - echo "Spark Project Version: ${{ needs.matrix.outputs.spark_version }}" - echo "Found Benchmarks? ${{ needs.matrix.outputs.foundlabel }}" - - run-benchmark: - if: ${{ needs.matrix.outputs.foundlabel == 'true' }} - needs: matrix - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - benchmark: ${{ fromJson(needs.matrix.outputs.matrix) }} - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v4 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.ref }} - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: 11 - - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- - - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - - name: Run Benchmark - run: ./gradlew :iceberg-spark:${{ github.event.inputs.spark_version }}:jmh -PjmhIncludeRegex=${{ matrix.benchmark }} -PjmhOutputPath=benchmark/${{ matrix.benchmark }}.txt - - - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: benchmark-results - path: | - **/benchmark/*.txt diff --git a/.github/workflows/kafka-connect-ci.yml b/.github/workflows/kafka-connect-ci.yml deleted file mode 100644 index 60cd9188b61d..000000000000 --- a/.github/workflows/kafka-connect-ci.yml +++ /dev/null @@ -1,104 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Kafka Connect CI" -on: - push: - branches: - - 'main' - - '0.*' - - '1.*' - - '2.*' - tags: - - 'apache-iceberg-**' - pull_request: - paths-ignore: - - '.github/ISSUE_TEMPLATE/**' - - '.github/workflows/api-binary-compatibility.yml' - - '.github/workflows/delta-conversion-ci.yml' - - '.github/workflows/flink-ci.yml' - - '.github/workflows/hive-ci.yml' - - '.github/workflows/java-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' - - '.github/workflows/labeler.yml' - - '.github/workflows/licence-check.yml' - - '.github/workflows/open-api.yml' - - '.github/workflows/publish-snapshot.yml' - - '.github/workflows/recurring-jmh-benchmarks.yml' - - '.github/workflows/site-ci.yml' - - '.github/workflows/spark-ci.yml' - - '.github/workflows/stale.yml' - - '.gitignore' - - '.asf.yml' - - 'dev/**' - - 'mr/**' - - 'flink/**' - - 'hive3/**' - - 'hive3-orc-bundle/**' - - 'hive-runtime/**' - - 'spark/**' - - 'docs/**' - - 'site/**' - - 'open-api/**' - - 'format/**' - - '.gitattributes' - - 'README.md' - - 'CONTRIBUTING.md' - - 'LICENSE' - - 'NOTICE' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - - kafka-connect-tests: - runs-on: ubuntu-22.04 - strategy: - matrix: - jvm: [11, 17, 21] - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: ${{ matrix.jvm }} - - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- - - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - run: | - ./gradlew -DsparkVersions= -DhiveVersions= -DflinkVersions= -DkafkaVersions=3 \ - :iceberg-kafka-connect:iceberg-kafka-connect-events:check \ - :iceberg-kafka-connect:iceberg-kafka-connect:check \ - :iceberg-kafka-connect:iceberg-kafka-connect-runtime:check \ - -Pquick=true -x javadoc - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test logs - path: | - **/build/testlogs diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml deleted file mode 100644 index 7d6cd5bd2177..000000000000 --- a/.github/workflows/labeler.yml +++ /dev/null @@ -1,34 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Pull Request Labeler" -on: pull_request_target - -permissions: - contents: read - pull-requests: write - -jobs: - triage: - runs-on: ubuntu-22.04 - steps: - - uses: actions/labeler@v5 - with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" - sync-labels: true diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml deleted file mode 100644 index ba730f6f2ed7..000000000000 --- a/.github/workflows/license-check.yml +++ /dev/null @@ -1,29 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Run License Check" -on: pull_request - -jobs: - rat: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - run: | - dev/check-license diff --git a/.github/workflows/open-api.yml b/.github/workflows/open-api.yml deleted file mode 100644 index 40aefc989244..000000000000 --- a/.github/workflows/open-api.yml +++ /dev/null @@ -1,62 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Open-API" -on: - push: - branches: - - 'main' - - '0.*' - - '1.*' - - '2.*' - tags: - - 'apache-iceberg-**' - pull_request: - paths: - - '.github/workflows/open-api.yml' - - 'open-api/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - openapi-spec-validator: - runs-on: ubuntu-22.04 - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - name: Install - working-directory: ./open-api - run: make install - - name: Validate REST catalog spec - working-directory: ./open-api - run: make lint - - name: Generate REST catalog spec Python code - working-directory: ./open-api - run: make generate - - name: Check if code is up to date - working-directory: ./open-api - run: git diff --exit-code - - name: Validate S3 REST Signer spec - working-directory: ./aws/src/main/resources - run: openapi-spec-validator s3-signer-open-api.yaml diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml new file mode 100644 index 000000000000..f078c3781b7e --- /dev/null +++ b/.github/workflows/publish-docker.yml @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Build and Push Docker Image + +on: + push: + tags: + - 'apache-iceberg-[0-9]+.[0-9]+.[0-9]+' + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v4 + with: + distribution: zulu + java-version: 21 + - name: Build Iceberg Open API project + run: ./gradlew :iceberg-open-api:shadowJar + - name: Login to Docker Hub + run: | + docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build Docker Image + run: docker build -t fokkodriesprong/iceberg-rest-adapter -f docker/iceberg-rest-adapter-image/Dockerfile . + - name: Push Docker Image with latest tag + if: github.event_name == 'workflow_dispatch' + run: docker push fokkodriesprong/iceberg-rest-adapter:latest + - name: Push Docker Image with version tag + if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') + run: | + echo "TAG=`echo ${{ github.ref }} | tr -d -c 0-9.`" >>${GITHUB_ENV} + docker push fokkodriesprong/iceberg-rest-adapter:$TAG \ No newline at end of file diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml deleted file mode 100644 index 7ff6b56da576..000000000000 --- a/.github/workflows/publish-snapshot.yml +++ /dev/null @@ -1,44 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Publish Snapshot to Maven" -on: - schedule: - # * is a special character in YAML so you have to quote this string - # this schedules a workflow to run at specific UTC times using POSIX cron syntax -> https://crontab.guru/ - # we're publishing a new snapshot every night at 00:00 UTC - - cron: '0 0 * * *' - -jobs: - publish-snapshot: - if: github.repository_owner == 'apache' - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - with: - # we need to fetch all tags so that getProjectVersion() in build.gradle correctly determines the next SNAPSHOT version from the newest tag - fetch-depth: 0 - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: 11 - - run: | - ./gradlew printVersion - ./gradlew -DallModules publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }} - ./gradlew -DflinkVersions= -DsparkVersions=3.3,3.4,3.5 -DscalaVersion=2.13 -DkafkaVersions=3 -DhiveVersions= publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }} diff --git a/.github/workflows/recurring-jmh-benchmarks.yml b/.github/workflows/recurring-jmh-benchmarks.yml deleted file mode 100644 index 71a52640b2f6..000000000000 --- a/.github/workflows/recurring-jmh-benchmarks.yml +++ /dev/null @@ -1,73 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Recurring JMH Benchmarks" -on: - schedule: - # * is a special character in YAML so you have to quote this string - # this schedules a workflow to run at specific UTC times using POSIX cron syntax -> https://crontab.guru/ - # we're running benchmarks every Sunday at 00:00 UTC - - cron: '0 0 * * 0' - -jobs: - run-benchmark: - if: github.repository_owner == 'apache' - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - # TODO: "IcebergSortCompactionBenchmark" seems to run indefinitely - benchmark: ["SparkParquetReadersFlatDataBenchmark", "SparkParquetReadersNestedDataBenchmark", - "SparkParquetWritersFlatDataBenchmark", "SparkParquetWritersNestedDataBenchmark", - "IcebergSourceFlatParquetDataFilterBenchmark", - "IcebergSourceFlatParquetDataReadBenchmark", "IcebergSourceFlatParquetDataWriteBenchmark", - "IcebergSourceNestedListParquetDataWriteBenchmark", "IcebergSourceNestedParquetDataFilterBenchmark", - "IcebergSourceNestedParquetDataReadBenchmark", "IcebergSourceNestedParquetDataWriteBenchmark", - "IcebergSourceParquetEqDeleteBenchmark", "IcebergSourceParquetMultiDeleteFileBenchmark", - "IcebergSourceParquetPosDeleteBenchmark", "IcebergSourceParquetWithUnrelatedDeleteBenchmark"] - spark_version: ['iceberg-spark-3.5'] - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v4 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.ref }} - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: 11 - - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- - - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - - name: Run Benchmark - run: ./gradlew :iceberg-spark:${{ matrix.spark_version }}:jmh -PjmhIncludeRegex=${{ matrix.benchmark }} -PjmhOutputPath=benchmark/${{ matrix.benchmark }}.txt -PjmhJsonOutputPath=benchmark/${{ matrix.benchmark }}.json - - - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: benchmark-results - path: | - **/benchmark/* diff --git a/.github/workflows/site-ci.yml b/.github/workflows/site-ci.yml deleted file mode 100644 index c085009a3997..000000000000 --- a/.github/workflows/site-ci.yml +++ /dev/null @@ -1,41 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -name: site-ci -on: - push: - branches: - - main - paths: - - docs/** - - site/** - workflow_dispatch: -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.x - - name: Deploy Iceberg documentation - run: | - git config --global user.name 'GitHub Actions' - git config --global user.email 'actions@github.com' - make deploy - working-directory: ./site diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml deleted file mode 100644 index 0d7bd2d3d3e7..000000000000 --- a/.github/workflows/spark-ci.yml +++ /dev/null @@ -1,115 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Spark CI" -on: - push: - branches: - - 'main' - - '0.*' - - '1.*' - - '2.*' - tags: - - 'apache-iceberg-**' - pull_request: - paths-ignore: - - '.github/ISSUE_TEMPLATE/**' - - '.github/workflows/api-binary-compatibility.yml' - - '.github/workflows/delta-conversion-ci.yml' - - '.github/workflows/flink-ci.yml' - - '.github/workflows/hive-ci.yml' - - '.github/workflows/java-ci.yml' - - '.github/workflows/jmh-benchmarks-ci.yml' - - '.github/workflows/kafka-connect-ci.yml' - - '.github/workflows/labeler.yml' - - '.github/workflows/licence-check.yml' - - '.github/workflows/open-api.yml' - - '.github/workflows/publish-snapshot.yml' - - '.github/workflows/recurring-jmh-benchmarks.yml' - - '.github/workflows/site-ci.yml' - - '.github/workflows/stale.yml' - - '.gitignore' - - '.asf.yml' - - 'dev/**' - - 'site/**' - - 'mr/**' - - 'hive3/**' - - 'hive3-orc-bundle/**' - - 'hive-runtime/**' - - 'flink/**' - - 'kafka-connect/**' - - 'docs/**' - - 'open-api/**' - - 'format/**' - - '.gitattributes' - - 'README.md' - - 'CONTRIBUTING.md' - - 'LICENSE' - - 'NOTICE' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - spark-tests: - runs-on: ubuntu-22.04 - strategy: - matrix: - jvm: [11, 17, 21] - spark: ['3.3', '3.4', '3.5'] - scala: ['2.12', '2.13'] - exclude: - # Spark 3.5 is the first version not failing on Java 21 (https://issues.apache.org/jira/browse/SPARK-42369) - # Full Java 21 support is coming in Spark 4 (https://issues.apache.org/jira/browse/SPARK-43831) - - jvm: 21 - spark: '3.3' - - jvm: 21 - spark: '3.4' - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: ${{ matrix.jvm }} - - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- - - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - with: - tool-cache: false - - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - run: | - ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=${{ matrix.scala }} -DhiveVersions= -DflinkVersions= -DkafkaVersions= \ - :iceberg-spark:iceberg-spark-${{ matrix.spark }}_${{ matrix.scala }}:check \ - :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_${{ matrix.scala }}:check \ - :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_${{ matrix.scala }}:check \ - -Pquick=true -x javadoc - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test logs - path: | - **/build/testlogs diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index 03276d60b524..000000000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,58 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Close Stale Issues and PRs" -on: - schedule: - - cron: '0 0 * * *' - -permissions: - # All other permissions are set to none - issues: write - pull-requests: write - -jobs: - stale: - if: github.repository_owner == 'apache' - runs-on: ubuntu-22.04 - steps: - - uses: actions/stale@v9.0.0 - with: - # stale issues - stale-issue-label: 'stale' - exempt-issue-labels: 'not-stale' - days-before-issue-stale: 180 - days-before-issue-close: 14 - stale-issue-message: > - This issue has been automatically marked as stale because it has been open for 180 days - with no activity. It will be closed in next 14 days if no further activity occurs. To - permanently prevent this issue from being considered stale, add the label 'not-stale', - but commenting on the issue is preferred when possible. - close-issue-message: > - This issue has been closed because it has not received any activity in the last 14 days - since being marked as 'stale' - # stale PRs - stale-pr-label: 'stale' - exempt-pr-labels: 'not-stale,security' - stale-pr-message: 'This pull request has been marked as stale due to 30 days of inactivity. It will be closed in 1 week if no further activity occurs. If you think that’s incorrect or this pull request requires a review, please simply write any comment. If closed, you can revive the PR at any time and @mention a reviewer or discuss it on the dev@iceberg.apache.org list. Thank you for your contributions.' - close-pr-message: 'This pull request has been closed due to lack of activity. This is not a judgement on the merit of the PR in any way. It is just a way of keeping the PR queue manageable. If you think that is incorrect, or the pull request requires review, you can revive the PR at any time.' - days-before-pr-stale: 30 - days-before-pr-close: 7 - ascending: true - operations-per-run: 200