#99 core examples in a subfolder - spark 3 update: spark.close

AbsaOSS · Jul 30, 2021 · 335d801 · 335d801
1 parent cdce13f
commit 335d801
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 3 deletions.
diff --git a/examples/atum-examples/pom.xml b/examples/atum-examples/pom.xml
@@ -175,6 +175,7 @@
     </build>
 
     <profiles>
+        <!-- Cross-building profiles -->
         <profile>
             <id>scala-2.11</id>
             <properties>
@@ -189,5 +190,19 @@
                 <scala.version>${scala_2.12.version}</scala.version>
             </properties>
         </profile>
+
+        <!-- Spark profiles -->
+        <profile>
+            <id>spark-2.4</id>
+            <properties>
+                <spark.version>${spark-24.version}</spark.version>
+            </properties>
+        </profile>
+        <profile>
+            <id>spark-3.1</id>
+            <properties>
+                <spark.version>${spark-31.version}</spark.version>
+            </properties>
+        </profile>
     </profiles>
 </project>
diff --git a/examples/atum-examples/src/main/scala/za/co/absa/atum/examples/SampleMeasurements1.scala b/examples/atum-examples/src/main/scala/za/co/absa/atum/examples/SampleMeasurements1.scala
@@ -53,8 +53,8 @@ object SampleMeasurements1 {
       .parquet("data/output/stage1_job_results")
 
     spark.disableControlMeasuresTracking()
+    spark.close()
 
-    // todo todo test/adjust for spark3
     if (!Files.exists(Paths.get("data/output/stage1_job_results/_INFO"))) {
       throw new Exception("_INFO file not found at data/output/stage1_job_results")
     } else {

diff --git a/examples/atum-examples/src/main/scala/za/co/absa/atum/examples/SampleMeasurements2.scala b/examples/atum-examples/src/main/scala/za/co/absa/atum/examples/SampleMeasurements2.scala
@@ -61,8 +61,8 @@ object SampleMeasurements2 {
       .parquet("data/output/stage2_job_results")
 
     spark.disableControlMeasuresTracking()
+    spark.close()
 
-    // todo test/adjust for spark3
     if (!Files.exists(Paths.get("data/output/stage2_job_results/_INFO"))) {
       throw new Exception("_INFO file not found at data/output/stage2_job_results")
     } else {

diff --git a/examples/atum-examples/src/main/scala/za/co/absa/atum/examples/SampleMeasurements3.scala b/examples/atum-examples/src/main/scala/za/co/absa/atum/examples/SampleMeasurements3.scala
@@ -57,8 +57,8 @@ object SampleMeasurements3 {
 
 
     spark.disableControlMeasuresTracking()
+    spark.close()
 
-    // todo test/adjust for spark3
     if (!Files.exists(Paths.get("data/output/stage3_job_results/_INFO"))) {
       throw new Exception("_INFO file not found at data/output/stage3_job_results")
     } else {