From 78c1d2ce45b3b2da25ea2ae9295203f425972564 Mon Sep 17 00:00:00 2001
From: Stijn De Haes <stijndehaes@gmail.com>
Date: Mon, 20 Nov 2023 14:03:58 +0000
Subject: [PATCH] Upgrade to spark 3.5.0 (#108)

---
 CHANGELOG.md                                              | 8 +++++++-
 .../pyspark/{{ cookiecutter.project_name }}/Dockerfile    | 2 +-
 .../{{ cookiecutter.project_name }}/dev-requirements.in   | 2 +-
 .../{{ cookiecutter.project_name }}/dev-requirements.txt  | 2 +-
 project/spark/{{ cookiecutter.project_name }}/Dockerfile  | 2 +-
 .../spark/{{ cookiecutter.project_name }}/build.gradle    | 8 ++++----
 6 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b813d3a..0b166fa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,11 @@
 <a name="unreleased"></a>
 ## unreleased
 
+## [1.5.0 - 2023-11-20]
+
+### features
+- Upgrade to spark 3.5.0
+
 ## [1.4.1 - 2023-11-06]
 
 ### features
@@ -284,7 +289,8 @@
 
 First release of the templates
 
-[Unreleased]: https://github.com/datamindedbe/datafy-templates/compare/1.4.1...HEAD
+[Unreleased]: https://github.com/datamindedbe/datafy-templates/compare/1.5.0...HEAD
+[1.4.1 - 2023-08-21]: https://github.com/datamindedbe/datafy-templates/compare/1.4.1...1.5.0
 [1.4.1 - 2023-08-21]: https://github.com/datamindedbe/datafy-templates/compare/1.4.0...1.4.1
 [1.4.0 - 2023-08-21]: https://github.com/datamindedbe/datafy-templates/compare/1.3.2...1.4.0
 [1.3.2 - 2023-08-07]: https://github.com/datamindedbe/datafy-templates/compare/1.3.1...1.3.2
diff --git a/project/pyspark/{{ cookiecutter.project_name }}/Dockerfile b/project/pyspark/{{ cookiecutter.project_name }}/Dockerfile
index 846d98a..fe89541 100644
--- a/project/pyspark/{{ cookiecutter.project_name }}/Dockerfile	
+++ b/project/pyspark/{{ cookiecutter.project_name }}/Dockerfile	
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/dataminded/spark-k8s-glue:v3.4.1-2.13-hadoop-3.3.6-v1
+FROM public.ecr.aws/dataminded/spark-k8s-glue:v3.5.0-hadoop-3.3.6-v2
 
 ENV PYSPARK_PYTHON python3
 WORKDIR /opt/spark/work-dir
diff --git a/project/pyspark/{{ cookiecutter.project_name }}/dev-requirements.in b/project/pyspark/{{ cookiecutter.project_name }}/dev-requirements.in
index b2bfc05..5203012 100644
--- a/project/pyspark/{{ cookiecutter.project_name }}/dev-requirements.in	
+++ b/project/pyspark/{{ cookiecutter.project_name }}/dev-requirements.in	
@@ -1,5 +1,5 @@
 -c requirements.txt
-pyspark==3.4.1
+pyspark==3.5.0
 pandas==1.5.2
 pytest>=6.0
 pytest-cov
diff --git a/project/pyspark/{{ cookiecutter.project_name }}/dev-requirements.txt b/project/pyspark/{{ cookiecutter.project_name }}/dev-requirements.txt
index 973818e..aa0a112 100644
--- a/project/pyspark/{{ cookiecutter.project_name }}/dev-requirements.txt	
+++ b/project/pyspark/{{ cookiecutter.project_name }}/dev-requirements.txt	
@@ -40,7 +40,7 @@ pycodestyle==2.10.0
     # via flake8
 pyflakes==3.0.1
     # via flake8
-pyspark==3.4.1
+pyspark==3.5.0
     # via -r dev-requirements.in
 pytest==7.2.1
     # via
diff --git a/project/spark/{{ cookiecutter.project_name }}/Dockerfile b/project/spark/{{ cookiecutter.project_name }}/Dockerfile
index 63ff9b5..e75f601 100644
--- a/project/spark/{{ cookiecutter.project_name }}/Dockerfile	
+++ b/project/spark/{{ cookiecutter.project_name }}/Dockerfile	
@@ -1,3 +1,3 @@
-FROM public.ecr.aws/dataminded/spark-k8s-glue:v3.4.1-hadoop-3.3.6-v1
+FROM public.ecr.aws/dataminded/spark-k8s-glue:v3.5.0-hadoop-3.3.6-v2
 
 COPY build/libs/*-all.jar /opt/spark/user-files/app.jar
diff --git a/project/spark/{{ cookiecutter.project_name }}/build.gradle b/project/spark/{{ cookiecutter.project_name }}/build.gradle
index cb8577b..a384a7a 100644
--- a/project/spark/{{ cookiecutter.project_name }}/build.gradle	
+++ b/project/spark/{{ cookiecutter.project_name }}/build.gradle	
@@ -34,16 +34,16 @@ dependencies {
     zinc group: 'com.typesafe.zinc', name: 'zinc', version: '0.3.15'
 
     implementation 'org.scala-lang:scala-library:2.13'
-    shadow group: 'org.apache.spark', name: 'spark-sql_2.13', version: '3.4.1'
-    shadow group: 'org.apache.spark', name: 'spark-hive_2.13', version: '3.4.1'
+    shadow group: 'org.apache.spark', name: 'spark-sql_2.13', version: '3.5.0'
+    shadow group: 'org.apache.spark', name: 'spark-hive_2.13', version: '3.5.0'
     shadow group: 'com.fasterxml.jackson', name: 'jackson-bom', version: '2.14.0'
     implementation group: 'com.typesafe.scala-logging', name: 'scala-logging_2.13', version: '3.9.5'
     implementation group: 'com.github.pathikrit', name: 'better-files_2.13', version: '3.9.1'
     implementation group: 'com.github.scopt', name: 'scopt_2.13', version: '3.7.1'
 
     testImplementation group: 'org.scalatest', name: 'scalatest_2.13', version: '3.2.10'
-    testImplementation group: 'org.apache.spark', name: 'spark-sql_2.13', version: '3.4.1'
-    testImplementation group: 'org.apache.spark', name: 'spark-hive_2.13', version: '3.4.1'
+    testImplementation group: 'org.apache.spark', name: 'spark-sql_2.13', version: '3.5.0'
+    testImplementation group: 'org.apache.spark', name: 'spark-hive_2.13', version: '3.5.0'
     testImplementation group: 'com.fasterxml.jackson', name: 'jackson-bom', version: '2.14.0'
     testRuntimeOnly group: 'org.pegdown', name: 'pegdown', version: '1.6.0'
     testRuntimeOnly group: 'org.ow2.asm', name: 'asm', version: '5.0.3'