From 66fb6b82c0ef71366ce7e86e799cb1d0f1df3ea3 Mon Sep 17 00:00:00 2001 From: HuangWei Date: Fri, 24 Nov 2023 12:07:26 +0800 Subject: [PATCH 1/2] fix: connect abort and improve log --- .../main/java/com/_4paradigm/openmldb/spark/OpenmldbTable.java | 1 + .../openmldb/taskmanager/config/TaskManagerConfig.java | 1 + .../openmldb/taskmanager/server/TaskManagerServer.java | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/java/openmldb-spark-connector/src/main/java/com/_4paradigm/openmldb/spark/OpenmldbTable.java b/java/openmldb-spark-connector/src/main/java/com/_4paradigm/openmldb/spark/OpenmldbTable.java index 0cf98b7d19e..9a971e92ce5 100644 --- a/java/openmldb-spark-connector/src/main/java/com/_4paradigm/openmldb/spark/OpenmldbTable.java +++ b/java/openmldb-spark-connector/src/main/java/com/_4paradigm/openmldb/spark/OpenmldbTable.java @@ -63,6 +63,7 @@ public OpenmldbTable(String dbName, String tableName, SdkOption option, String w // no need to check table exists, schema() will check it later } catch (SqlException e) { e.printStackTrace(); + throw new RuntimeException("connect to openmldb failed", e); // abort immediately } // TODO: cache schema & delete executor? } diff --git a/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/config/TaskManagerConfig.java b/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/config/TaskManagerConfig.java index bba740a2ffa..f9b2b3e8d82 100644 --- a/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/config/TaskManagerConfig.java +++ b/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/config/TaskManagerConfig.java @@ -243,6 +243,7 @@ private void init() throws ConfigException { } // Get properties and check + // server.host will be used by RunBatchSql, it's not good to be null if (props.getProperty("server.host") == null) { props.setProperty("server.host", "0.0.0.0"); } diff --git a/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/server/TaskManagerServer.java b/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/server/TaskManagerServer.java index 0a75c2e37b2..a3a75c5ac4e 100644 --- a/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/server/TaskManagerServer.java +++ b/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/server/TaskManagerServer.java @@ -138,7 +138,7 @@ public static void main(String[] args) { server.start(); } catch (Exception e) { e.printStackTrace(); - logger.error(String.format("Fail to start TaskManager, message: ", e.getMessage())); + logger.error("Fail to start TaskManager", e); } } From 06a11c2d0ad266252edef134dca7c400a610cf01 Mon Sep 17 00:00:00 2001 From: HuangWei Date: Wed, 29 Nov 2023 13:12:56 +0800 Subject: [PATCH 2/2] spark home config in TaskManagerConfig.java --- .../openmldb/taskmanager/config/TaskManagerConfig.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/config/TaskManagerConfig.java b/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/config/TaskManagerConfig.java index f9b2b3e8d82..4102f78d5e8 100644 --- a/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/config/TaskManagerConfig.java +++ b/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/config/TaskManagerConfig.java @@ -358,14 +358,13 @@ private void init() throws ConfigException { } } - String SPARK_HOME = firstNonEmpty(props.getProperty("spark.home"), System.getenv("SPARK_HOME")); + String SPARK_HOME = firstNonEmpty(props.getProperty("spark.home")); // isEmpty checks null and empty if (isEmpty(SPARK_HOME)) { throw new ConfigException("spark.home", "should set config 'spark.home' or environment variable 'SPARK_HOME'"); } - if (SPARK_HOME != null) { - props.setProperty("spark.home", SPARK_HOME); - } + // rewrite env for spark submit + props.setProperty("spark.home", SPARK_HOME); // TODO: Check if we can get spark-submit