Upgrade to spark 3.5.0 (#108)

datamindedbe · Nov 20, 2023 · 78c1d2c · 78c1d2c
1 parent 3cf4960
commit 78c1d2c
Show file tree

Hide file tree

Showing 6 changed files with 15 additions and 9 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,11 @@
 <a name="unreleased"></a>
 ## unreleased
 
+## [1.5.0 - 2023-11-20]
+
+### features
+- Upgrade to spark 3.5.0
+
 ## [1.4.1 - 2023-11-06]
 
 ### features
@@ -284,7 +289,8 @@
 
 First release of the templates
 
-[Unreleased]: https://github.com/datamindedbe/datafy-templates/compare/1.4.1...HEAD
+[Unreleased]: https://github.com/datamindedbe/datafy-templates/compare/1.5.0...HEAD
+[1.4.1 - 2023-08-21]: https://github.com/datamindedbe/datafy-templates/compare/1.4.1...1.5.0
 [1.4.1 - 2023-08-21]: https://github.com/datamindedbe/datafy-templates/compare/1.4.0...1.4.1
 [1.4.0 - 2023-08-21]: https://github.com/datamindedbe/datafy-templates/compare/1.3.2...1.4.0
 [1.3.2 - 2023-08-07]: https://github.com/datamindedbe/datafy-templates/compare/1.3.1...1.3.2

diff --git a/project/pyspark/{{ cookiecutter.project_name }}/Dockerfile b/project/pyspark/{{ cookiecutter.project_name }}/Dockerfile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/dataminded/spark-k8s-glue:v3.4.1-2.13-hadoop-3.3.6-v1
+FROM public.ecr.aws/dataminded/spark-k8s-glue:v3.5.0-hadoop-3.3.6-v2
 
 ENV PYSPARK_PYTHON python3
 WORKDIR /opt/spark/work-dir

diff --git a/project/pyspark/{{ cookiecutter.project_name }}/dev-requirements.in b/project/pyspark/{{ cookiecutter.project_name }}/dev-requirements.in
@@ -1,5 +1,5 @@
 -c requirements.txt
-pyspark==3.4.1
+pyspark==3.5.0
 pandas==1.5.2
 pytest>=6.0
 pytest-cov

diff --git a/project/pyspark/{{ cookiecutter.project_name }}/dev-requirements.txt b/project/pyspark/{{ cookiecutter.project_name }}/dev-requirements.txt
@@ -40,7 +40,7 @@ pycodestyle==2.10.0
     # via flake8
 pyflakes==3.0.1
     # via flake8
-pyspark==3.4.1
+pyspark==3.5.0
     # via -r dev-requirements.in
 pytest==7.2.1
     # via

diff --git a/project/spark/{{ cookiecutter.project_name }}/Dockerfile b/project/spark/{{ cookiecutter.project_name }}/Dockerfile
@@ -1,3 +1,3 @@
-FROM public.ecr.aws/dataminded/spark-k8s-glue:v3.4.1-hadoop-3.3.6-v1
+FROM public.ecr.aws/dataminded/spark-k8s-glue:v3.5.0-hadoop-3.3.6-v2
 
 COPY build/libs/*-all.jar /opt/spark/user-files/app.jar
diff --git a/project/spark/{{ cookiecutter.project_name }}/build.gradle b/project/spark/{{ cookiecutter.project_name }}/build.gradle
@@ -34,16 +34,16 @@ dependencies {
     zinc group: 'com.typesafe.zinc', name: 'zinc', version: '0.3.15'
 
     implementation 'org.scala-lang:scala-library:2.13'
-    shadow group: 'org.apache.spark', name: 'spark-sql_2.13', version: '3.4.1'
-    shadow group: 'org.apache.spark', name: 'spark-hive_2.13', version: '3.4.1'
+    shadow group: 'org.apache.spark', name: 'spark-sql_2.13', version: '3.5.0'
+    shadow group: 'org.apache.spark', name: 'spark-hive_2.13', version: '3.5.0'
     shadow group: 'com.fasterxml.jackson', name: 'jackson-bom', version: '2.14.0'
     implementation group: 'com.typesafe.scala-logging', name: 'scala-logging_2.13', version: '3.9.5'
     implementation group: 'com.github.pathikrit', name: 'better-files_2.13', version: '3.9.1'
     implementation group: 'com.github.scopt', name: 'scopt_2.13', version: '3.7.1'
 
     testImplementation group: 'org.scalatest', name: 'scalatest_2.13', version: '3.2.10'
-    testImplementation group: 'org.apache.spark', name: 'spark-sql_2.13', version: '3.4.1'
-    testImplementation group: 'org.apache.spark', name: 'spark-hive_2.13', version: '3.4.1'
+    testImplementation group: 'org.apache.spark', name: 'spark-sql_2.13', version: '3.5.0'
+    testImplementation group: 'org.apache.spark', name: 'spark-hive_2.13', version: '3.5.0'
     testImplementation group: 'com.fasterxml.jackson', name: 'jackson-bom', version: '2.14.0'
     testRuntimeOnly group: 'org.pegdown', name: 'pegdown', version: '1.6.0'
     testRuntimeOnly group: 'org.ow2.asm', name: 'asm', version: '5.0.3'