diff --git a/_e2e/distribution.out.expected b/_e2e/distribution.out.expected index f87195d..aae5b1f 100644 --- a/_e2e/distribution.out.expected +++ b/_e2e/distribution.out.expected @@ -17,8 +17,8 @@ spark-submit --master "k8s://https://127.0.0.1:34729" \ --conf "spark.executor.extraJavaOptions=-Djava.security.auth.login.config=/etc/fink-broker/kafka-jaas.conf" \ --conf spark.driver.memory=2Gi \ --conf spark.executor.memory=2Gi \ - --conf spark.kubernetes.driver.request.cores=1 \ - --conf spark.kubernetes.executor.request.cores=1 \ + --conf spark.driver.cores=1 \ + --conf spark.executor.cores=1 \ local:///home/fink/fink-broker/bin/distribute.py \ -log_level "INFO" \ -online_data_prefix "s3a://fink-broker-online-20000101" \ diff --git a/_e2e/raw2science.out.expected b/_e2e/raw2science.out.expected index 2cddd15..74ec960 100644 --- a/_e2e/raw2science.out.expected +++ b/_e2e/raw2science.out.expected @@ -16,8 +16,8 @@ spark-submit --master "k8s://https://127.0.0.1:34729" \ --conf spark.executor.instances=4 \ --conf spark.driver.memory=3Gi \ --conf spark.executor.memory=3Gi \ - --conf spark.kubernetes.driver.request.cores=1 \ - --conf spark.kubernetes.executor.request.cores=1 \ + --conf spark.driver.cores=1 \ + --conf spark.executor.cores=1 \ local:///home/fink/fink-broker/bin/raw2science.py \ -log_level "INFO" \ -online_data_prefix "s3a://fink-broker-online-20000101" \ diff --git a/cmd/run.go b/cmd/run.go index f2a0e39..196429d 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -203,8 +203,8 @@ org.apache.hadoop:hadoop-aws:3.2.3` } // TODO make it configurable at the task level if rc.Cpu != "" { - cmdTpl += fmt.Sprintf(`--conf spark.kubernetes.driver.request.cores=%[1]s \ - --conf spark.kubernetes.executor.request.cores=%[1]s \ + cmdTpl += fmt.Sprintf(`--conf spark.driver.cores=%[1]s \ + --conf spark.executor.cores=%[1]s \ `, rc.Cpu) } cmdTpl += `local:///home/fink/fink-broker/bin/{{ .Binary }} \