Upgrade to spark 3.5.1 (#28)

datamindedbe · Feb 29, 2024 · 8b67cf8 · 8b67cf8
1 parent 0363f61
commit 8b67cf8
Show file tree

Hide file tree

Showing 7 changed files with 13 additions and 30 deletions.
diff --git a/basic/pi_spark/.dockerignore b/basic/pi_spark/.dockerignore
@@ -29,3 +29,4 @@ dags
 Dockerfile
 tests
 pytest.ini
+.idea
diff --git a/basic/pi_spark/.python-version b/basic/pi_spark/.python-version
@@ -1 +1 @@
-3.9.17
+3.10.12
diff --git a/basic/pi_spark/Dockerfile b/basic/pi_spark/Dockerfile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/dataminded/spark-k8s-glue:v3.3.1-hadoop-3.3.4-v2
+FROM public.ecr.aws/dataminded/spark-k8s-glue:v3.5.1-hadoop-3.3.6-v1
 
 ENV PYSPARK_PYTHON python3
 WORKDIR /opt/spark/work-dir

diff --git a/basic/pi_spark/dev-requirements.in b/basic/pi_spark/dev-requirements.in
@@ -1,6 +1,6 @@
 -c requirements.txt
-pyspark==3.3.1
-pandas==1.1.5
+pyspark==3.5.1
+pandas==1.5.2
 pytest>=6.0
 pytest-cov
 black

diff --git a/basic/pi_spark/dev-requirements.txt b/basic/pi_spark/dev-requirements.txt
@@ -1,19 +1,13 @@
-#
-# This file is autogenerated by pip-compile with Python 3.9
-# by the following command:
-#
-#    pip-compile dev-requirements.in
-#
 attrs==22.2.0
     # via pytest
 black==22.12.0
     # via -r dev-requirements.in
 click==8.1.3
     # via black
 coverage[toml]==7.0.5
-    # via pytest-cov
-exceptiongroup==1.1.0
-    # via pytest
+    # via
+    #   coverage
+    #   pytest-cov
 flake8==6.0.0
     # via -r dev-requirements.in
 iniconfig==2.0.0
@@ -26,21 +20,21 @@ numpy==1.24.1
     # via pandas
 packaging==23.0
     # via pytest
-pandas==1.1.5
+pandas==1.5.2
     # via -r dev-requirements.in
 pathspec==0.10.3
     # via black
 platformdirs==2.6.2
     # via black
 pluggy==1.0.0
     # via pytest
-py4j==0.10.9.5
+py4j==0.10.9.7
     # via pyspark
 pycodestyle==2.10.0
     # via flake8
 pyflakes==3.0.1
     # via flake8
-pyspark==3.3.1
+pyspark==3.5.1
     # via -r dev-requirements.in
 pytest==7.2.1
     # via
@@ -54,10 +48,3 @@ pytz==2022.7.1
     # via pandas
 six==1.16.0
     # via python-dateutil
-tomli==2.0.1
-    # via
-    #   black
-    #   coverage
-    #   pytest
-typing-extensions==4.4.0
-    # via black
diff --git a/basic/pi_spark/requirements.txt b/basic/pi_spark/requirements.txt
@@ -1,6 +1 @@
-#
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
-#
-#    pip-compile requirements.in
-#
+
diff --git a/basic/pi_spark/setup.py b/basic/pi_spark/setup.py
@@ -10,7 +10,7 @@
     name="pi_spark",
     version="0.0.1",
     description="pyspark sample template",
-    python_requires=">=3.6",
+    python_requires=">=3.10",
     packages=find_packages("src"),
     package_dir={"": "src"},
     py_modules=[splitext(basename(path))[0] for path in glob("src/*.py")],
-Original file line number
+Diff line change
@@ Expand Up / @@ -29,3 +29,4 @@ dags @@
     Dockerfile
     tests
     pytest.ini
+    .idea