Fix cores management

astrolabsoftware · Apr 26, 2024 · 77fc90c · 77fc90c
1 parent 570df04
commit 77fc90c
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 6 deletions.
diff --git a/_e2e/distribution.out.expected b/_e2e/distribution.out.expected
@@ -17,8 +17,8 @@ spark-submit --master "k8s://https://127.0.0.1:34729" \
     --conf "spark.executor.extraJavaOptions=-Djava.security.auth.login.config=/etc/fink-broker/kafka-jaas.conf" \
     --conf spark.driver.memory=2Gi \
     --conf spark.executor.memory=2Gi \
-    --conf spark.kubernetes.driver.request.cores=1 \
-    --conf spark.kubernetes.executor.request.cores=1 \
+    --conf spark.driver.cores=1 \
+    --conf spark.executor.cores=1 \
     local:///home/fink/fink-broker/bin/distribute.py \
     -log_level "INFO" \
     -online_data_prefix "s3a://fink-broker-online-20000101" \

diff --git a/_e2e/raw2science.out.expected b/_e2e/raw2science.out.expected
@@ -16,8 +16,8 @@ spark-submit --master "k8s://https://127.0.0.1:34729" \
     --conf spark.executor.instances=4 \
     --conf spark.driver.memory=3Gi \
     --conf spark.executor.memory=3Gi \
-    --conf spark.kubernetes.driver.request.cores=1 \
-    --conf spark.kubernetes.executor.request.cores=1 \
+    --conf spark.driver.cores=1 \
+    --conf spark.executor.cores=1 \
     local:///home/fink/fink-broker/bin/raw2science.py \
     -log_level "INFO" \
     -online_data_prefix "s3a://fink-broker-online-20000101" \

diff --git a/cmd/run.go b/cmd/run.go
@@ -203,8 +203,8 @@ org.apache.hadoop:hadoop-aws:3.2.3`
 	}
 	// TODO make it configurable at the task level
 	if rc.Cpu != "" {
-		cmdTpl += fmt.Sprintf(`--conf spark.kubernetes.driver.request.cores=%[1]s \
-    --conf spark.kubernetes.executor.request.cores=%[1]s \
+		cmdTpl += fmt.Sprintf(`--conf spark.driver.cores=%[1]s \
+    --conf spark.executor.cores=%[1]s \
     `, rc.Cpu)
 	}
 	cmdTpl += `local:///home/fink/fink-broker/bin/{{ .Binary }} \