Merge branch 'main' of github.com:apache/iceberg-python into fd-bump-…

…java
apache · Dec 8, 2024 · 31f44fd · 31f44fd
2 parents a5dc566 + d967050
commit 31f44fd
Show file tree

Hide file tree

Showing 58 changed files with 1,592 additions and 1,050 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -38,6 +38,7 @@ github:
         required_approving_review_count: 1
 
       required_linear_history: true
+  del_branch_on_merge: true
   features:
     wiki: true
     issues: true

diff --git a/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml b/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 ---
 name: Iceberg Bug report 🐞
 description: Problems, bugs and issues with Apache Iceberg
@@ -9,7 +28,8 @@ body:
       description: What Apache Iceberg version are you using?
       multiple: false
       options:
-        - "0.8.0 (latest release)"
+        - "0.8.1 (latest release)"
+        - "0.8.0"
         - "0.7.1"
         - "0.7.0"
         - "0.6.1"
@@ -31,3 +51,11 @@ body:
         You can include files by dragging and dropping them here.
     validations:
       required: true
+  - type: checkboxes
+    attributes:
+      label: Willingness to contribute
+      description: The Apache Iceberg community encourages bug-fix contributions. Would you or another member of your organization be willing to contribute a fix for this bug to the PyIceberg codebase?
+      options:
+        - label: I can contribute a fix for this bug independently
+        - label: I would be willing to contribute a fix for this bug with guidance from the Iceberg community
+        - label: I cannot contribute a fix for this bug at this time
diff --git a/.github/ISSUE_TEMPLATE/iceberg_improvement.yml b/.github/ISSUE_TEMPLATE/iceberg_improvement.yml
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 ---
 name: Iceberg Improvement / Feature Request
 description: New features with Apache Iceberg

diff --git a/.github/ISSUE_TEMPLATE/iceberg_question.yml b/.github/ISSUE_TEMPLATE/iceberg_question.yml
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 ---
 name: Iceberg Question
 description: Questions around Apache Iceberg

diff --git a/.github/workflows/check-md-link.yml b/.github/workflows/check-md-link.yml
@@ -1,12 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 name: Check Markdown links
 
 on:
   push:
     paths:
-      - mkdocs/**
+      - '.github/workflows/check-md-link.yml'
+      - 'mkdocs/**'
     branches:
     - 'main'
   pull_request:
+    paths:
+      - '.github/workflows/check-md-link.yml'
+      - 'mkdocs/**'
 
 jobs:
   markdown-link-check:

diff --git a/.github/workflows/python-ci-docs.yml b/.github/workflows/python-ci-docs.yml
@@ -17,9 +17,14 @@
 # under the License.
 #
 
-name: "Python Docs"
+name: "Python CI Docs"
+
 on:
-  workflow_dispatch:
+  push:
+    branches:
+      - 'main'
+  pull_request:
+
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -33,24 +38,10 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python }}
+          python-version: 3.12
       - name: Install
         working-directory: ./mkdocs
         run: pip install -r requirements.txt
       - name: Build
         working-directory: ./mkdocs
         run: mkdocs build --strict
-      - name: Copy
-        working-directory: ./mkdocs
-        run: mv ./site /tmp/site
-      - name: Push changes to gh-pages branch
-        run: |
-          git checkout --orphan gh-pages-tmp
-          git rm --quiet -rf .
-          cp -r /tmp/site/* .
-          git config --global user.name 'GitHub Actions'
-          git config --global user.email '[email protected]'
-          echo "py.iceberg.apache.org" > CNAME
-          git add --all
-          git commit -m 'Publish Python docs'
-          git push -f origin gh-pages-tmp:gh-pages || true
diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
@@ -24,6 +24,19 @@ on:
     branches:
     - 'main'
   pull_request:
+    paths:
+    - '**'                              # Include all files and directories in the repository by default.
+    - '!.github/workflows/**'           # Exclude all workflow files
+    - '.github/workflows/python-ci.yml' # except the current file.
+    - '!.github/ISSUE_TEMPLATE/**'      # Exclude files and directories that don't impact tests or code like templates, metadata, and documentation.
+    - '!.gitignore'
+    - '!.asf.yml'
+    - '!mkdocs/**'
+    - '!.gitattributes'
+    - '!README.md'
+    - '!CONTRIBUTING.md'
+    - '!LICENSE'
+    - '!NOTICE'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

diff --git a/.github/workflows/python-integration.yml b/.github/workflows/python-integration.yml
@@ -24,6 +24,19 @@ on:
     branches:
     - 'main'
   pull_request:
+    paths:
+    - '**'                                       # Include all files and directories in the repository by default.
+    - '!.github/workflows/**'                    # Exclude all workflow files
+    - '.github/workflows/python-integration.yml' # except the current file.
+    - '!.github/ISSUE_TEMPLATE/**'               # Exclude files and directories that don't impact tests or code like templates, metadata, and documentation.
+    - '!.gitignore'
+    - '!.asf.yml'
+    - '!mkdocs/**'
+    - '!.gitattributes'
+    - '!README.md'
+    - '!CONTRIBUTING.md'
+    - '!LICENSE'
+    - '!NOTICE'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

diff --git a/.github/workflows/python-release-docs.yml b/.github/workflows/python-release-docs.yml
@@ -0,0 +1,56 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Release Docs"
+on:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  docs:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install
+        working-directory: ./mkdocs
+        run: pip install -r requirements.txt
+      - name: Build
+        working-directory: ./mkdocs
+        run: mkdocs build --strict
+      - name: Copy
+        working-directory: ./mkdocs
+        run: mv ./site /tmp/site
+      - name: Push changes to gh-pages branch
+        run: |
+          git checkout --orphan gh-pages-tmp
+          git rm --quiet -rf .
+          cp -r /tmp/site/* .
+          git config --global user.name 'GitHub Actions'
+          git config --global user.email '[email protected]'
+          echo "py.iceberg.apache.org" > CNAME
+          git add --all
+          git commit -m 'Publish Python docs'
+          git push -f origin gh-pages-tmp:gh-pages || true
diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml
@@ -34,7 +34,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-22.04, windows-2022, macos-12, macos-13, macos-14 ]
+        os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ]
 
     steps:
       - uses: actions/checkout@v4
@@ -63,7 +63,7 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.21.3
+        uses: pypa/cibuildwheel@v2.22.0
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"
@@ -84,7 +84,17 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
         run: ls -lah dist/* && cp dist/* wheelhouse/
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
-          name: "release-${{ github.event.inputs.version }}"
+          name: "release-${{ matrix.os }}"
           path: ./wheelhouse/*
+  merge:
+    runs-on: ubuntu-latest
+    needs: build_wheels
+    steps:
+      - name: Merge Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: "release-${{ github.event.inputs.version }}"
+          pattern: release-*
+          delete-merged: true
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -19,7 +19,7 @@ exclude: ^vendor/
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -29,7 +29,7 @@ repos:
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version (Used for linting)
-    rev: v0.1.8
+    rev: v0.7.4
     hooks:
       - id: ruff
         args: [ --fix, --exit-non-zero-on-fix, --preview ]
@@ -47,7 +47,7 @@ repos:
       - id: pycln
         args: [--config=pyproject.toml]
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.41.0
+    rev: v0.42.0
     hooks:
       - id: markdownlint
         args: ["--fix"]

diff --git a/Makefile b/Makefile
@@ -22,7 +22,7 @@ help:  ## Display this help
 install-poetry:  ## Install poetry if the user has not done that yet.
 	 @if ! command -v poetry &> /dev/null; then \
          echo "Poetry could not be found. Installing..."; \
-         pip install --user poetry==1.8.3; \
+         pip install --user poetry==1.8.4; \
      else \
          echo "Poetry is already installed."; \
      fi

diff --git a/dev/.rat-excludes b/dev/.rat-excludes
@@ -1,4 +1,5 @@
 .rat-excludes
 build
+.git
 .gitignore
 poetry.lock
diff --git a/dev/Dockerfile b/dev/Dockerfile
@@ -36,21 +36,21 @@ ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$
 RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/spark-events
 WORKDIR ${SPARK_HOME}
 
-ENV SPARK_VERSION=3.5.0
+ENV SPARK_VERSION=3.5.3
 ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12
 ENV ICEBERG_VERSION=1.7.1
-ENV PYICEBERG_VERSION=0.8.0
+ENV PYICEBERG_VERSION=0.8.1
 
-RUN curl --retry 3 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
+RUN curl --retry 5 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
  && tar xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \
  && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz
 
 # Download iceberg spark runtime
-RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar -Lo iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
+RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar -Lo iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
  && mv iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar /opt/spark/jars
 
 # Download AWS bundle
-RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
+RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
 
 COPY spark-defaults.conf /opt/spark/conf
 ENV PATH="/opt/spark/sbin:/opt/spark/bin:${PATH}"