make logging a tiny bit more customizable (#299)

eogrow/core/logging.py

@@ -117,7 +117,7 @@ def get_pipeline_logs_folder(self, pipeline_execution_name: str, full_path: bool
             return join_path(main_logs_folder, pipeline_execution_name)
         return fs.path.combine(main_logs_folder, pipeline_execution_name)
 
-    def start_logging(self, pipeline_execution_name: str) -> list[Handler]:
+    def start_logging(self, logger: logging.Logger, pipeline_execution_name: str, filename: str) -> list[Handler]:
         """Creates a folder for logs and sets up (and returns) logging handlers
 
         Supported handlers:
@@ -129,22 +129,21 @@ def start_logging(self, pipeline_execution_name: str) -> list[Handler]:
             self.storage.filesystem.makedirs(logs_folder, recreate=True)
             self._add_cluster_config_to_logs(logs_folder)
 
-        global_logger = logging.getLogger()
-        global_logger.setLevel(logging.DEBUG)
+        logger.setLevel(logging.DEBUG)
 
-        for default_handler in global_logger.handlers:
+        for default_handler in logger.handlers:
             default_handler.setLevel(logging.WARNING)
 
         new_handlers: list[Handler] = []
 
         if self.config.save_logs:
-            file_handler = self._create_file_handler(pipeline_execution_name)
-            global_logger.addHandler(file_handler)
+            file_handler = self._create_file_handler(pipeline_execution_name, filename)
+            logger.addHandler(file_handler)
             new_handlers.append(file_handler)
 
         if self.config.show_logs:
             stdout_handler = self._create_stdout_handler()
-            global_logger.addHandler(stdout_handler)
+            logger.addHandler(stdout_handler)
             new_handlers.append(stdout_handler)
 
         if self.config.capture_warnings:
@@ -160,9 +159,9 @@ def _add_cluster_config_to_logs(self, logs_folder: str) -> None:
             os_fs = OSFS(os_folder)  # the file is on the head node, might not be visible in storage.filesystem
             fs.copy.copy_file(os_fs, os_file, self.storage.filesystem, fs.path.join(logs_folder, "cluster.yaml"))
 
-    def _create_file_handler(self, pipeline_execution_name: str) -> Handler:
+    def _create_file_handler(self, pipeline_execution_name: str, filename: str) -> Handler:
         """Creates a logging handler to write a pipeline log to a file."""
-        logs_filename = fs.path.combine(self.get_pipeline_logs_folder(pipeline_execution_name), "pipeline.log")
+        logs_filename = fs.path.combine(self.get_pipeline_logs_folder(pipeline_execution_name), filename)
         file_handler = RegularBackupHandler(
             logs_filename,
             filesystem=self.storage.filesystem,
@@ -194,19 +193,18 @@ def _create_stdout_handler(self) -> Handler:
 
         return stdout_handler
 
-    def stop_logging(self, handlers: list[Handler]) -> None:
+    def stop_logging(self, logger: logging.Logger, handlers: list[Handler]) -> None:
         """Updates logs, removes pipeline handlers from the global logger and puts global logging level back to
         default
         """
         if self.config.capture_warnings:
             logging.captureWarnings(False)
 
-        global_logger = logging.getLogger()
         for handler in handlers:
             handler.close()
-            global_logger.removeHandler(handler)
+            logger.removeHandler(handler)
 
-        global_logger.setLevel(logging.WARNING)
+        logger.setLevel(logging.WARNING)
 
     def update_pipeline_report(
         self,

eogrow/core/pipeline.py

@@ -222,7 +222,8 @@ def run(self) -> None:
         timestamp = current_timestamp()
         self.current_execution_name = self.get_pipeline_execution_name(timestamp)
 
-        handlers = self.logging_manager.start_logging(self.current_execution_name)
+        root_logger = logging.getLogger()
+        handlers = self.logging_manager.start_logging(root_logger, self.current_execution_name, "pipeline.log")
         try:
             self.logging_manager.update_pipeline_report(
                 pipeline_execution_name=self.current_execution_name,
@@ -259,7 +260,7 @@ def run(self) -> None:
                 pipeline_execution_name=self.current_execution_name, finished=finished, failed=failed
             )
         finally:
-            self.logging_manager.stop_logging(handlers)
+            self.logging_manager.stop_logging(root_logger, handlers)
 
     def run_procedure(self) -> tuple[list[str], list[str]]:
         """Execution procedure of pipeline. Can be overridden if needed.