#99 eventually back to 10 secs, spark.disableControlMeasuresTracking(…

…) called at the end instead.
AbsaOSS · Aug 2, 2021 · 6a01998 · 6a01998
1 parent f119266
commit 6a01998
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 12 deletions.
diff --git a/examples/atum-examples/src/main/scala/za/co/absa/atum/examples/SampleMeasurements1.scala b/examples/atum-examples/src/main/scala/za/co/absa/atum/examples/SampleMeasurements1.scala
@@ -55,13 +55,12 @@ object SampleMeasurements1 extends Eventually {
       .write.mode(SaveMode.Overwrite)
       .parquet("data/output/stage1_job_results")
 
-    spark.disableControlMeasuresTracking()
-    spark.close()
-
-    eventually(timeout(scaled(60.seconds)), interval(scaled(5.seconds))) {
+    eventually(timeout(scaled(10.seconds)), interval(scaled(500.millis))) {
       if (!Files.exists(Paths.get("data/output/stage1_job_results/_INFO"))) {
         throw new Exception("_INFO file not found at data/output/stage1_job_results")
       }
     }
+
+    spark.disableControlMeasuresTracking()
   }
 }
diff --git a/examples/atum-examples/src/main/scala/za/co/absa/atum/examples/SampleMeasurements2.scala b/examples/atum-examples/src/main/scala/za/co/absa/atum/examples/SampleMeasurements2.scala
@@ -64,13 +64,12 @@ object SampleMeasurements2 extends Eventually {
       .write.mode(SaveMode.Overwrite)
       .parquet("data/output/stage2_job_results")
 
-    spark.disableControlMeasuresTracking()
-    spark.close()
-
-    eventually(timeout(scaled(60.seconds)), interval(scaled(5.seconds))) {
+    eventually(timeout(scaled(10.seconds)), interval(scaled(500.millis))) {
       if (!Files.exists(Paths.get("data/output/stage2_job_results/_INFO"))) {
         throw new Exception("_INFO file not found at data/output/stage2_job_results")
       }
     }
+
+    spark.disableControlMeasuresTracking()
   }
 }
diff --git a/examples/atum-examples/src/main/scala/za/co/absa/atum/examples/SampleMeasurements3.scala b/examples/atum-examples/src/main/scala/za/co/absa/atum/examples/SampleMeasurements3.scala
@@ -58,10 +58,7 @@ object SampleMeasurements3 extends Eventually {
       .write.mode(SaveMode.Overwrite)
       .parquet("data/output/stage3_job_results")
 
-    spark.disableControlMeasuresTracking()
-    spark.close()
-
-    eventually(timeout(scaled(60.seconds)), interval(scaled(5.seconds))) { // scaling will help on slow environments
+    eventually(timeout(scaled(10.seconds)), interval(scaled(500.millis))) { // scaling will help on slow environments
       if (!Files.exists(Paths.get("data/output/stage3_job_results/_INFO"))) {
         throw new Exception("_INFO file not found at data/output/stage3_job_results")
       }
@@ -77,5 +74,7 @@ object SampleMeasurements3 extends Eventually {
       }
 
     }
+
+    spark.disableControlMeasuresTracking()
   }
 }