diff --git a/eogrow/core/logging.py b/eogrow/core/logging.py index f63c6029..7c2bc2ab 100644 --- a/eogrow/core/logging.py +++ b/eogrow/core/logging.py @@ -117,7 +117,7 @@ def get_pipeline_logs_folder(self, pipeline_execution_name: str, full_path: bool return join_path(main_logs_folder, pipeline_execution_name) return fs.path.combine(main_logs_folder, pipeline_execution_name) - def start_logging(self, pipeline_execution_name: str) -> list[Handler]: + def start_logging(self, logger: logging.Logger, pipeline_execution_name: str, filename: str) -> list[Handler]: """Creates a folder for logs and sets up (and returns) logging handlers Supported handlers: @@ -129,22 +129,21 @@ def start_logging(self, pipeline_execution_name: str) -> list[Handler]: self.storage.filesystem.makedirs(logs_folder, recreate=True) self._add_cluster_config_to_logs(logs_folder) - global_logger = logging.getLogger() - global_logger.setLevel(logging.DEBUG) + logger.setLevel(logging.DEBUG) - for default_handler in global_logger.handlers: + for default_handler in logger.handlers: default_handler.setLevel(logging.WARNING) new_handlers: list[Handler] = [] if self.config.save_logs: - file_handler = self._create_file_handler(pipeline_execution_name) - global_logger.addHandler(file_handler) + file_handler = self._create_file_handler(pipeline_execution_name, filename) + logger.addHandler(file_handler) new_handlers.append(file_handler) if self.config.show_logs: stdout_handler = self._create_stdout_handler() - global_logger.addHandler(stdout_handler) + logger.addHandler(stdout_handler) new_handlers.append(stdout_handler) if self.config.capture_warnings: @@ -160,9 +159,9 @@ def _add_cluster_config_to_logs(self, logs_folder: str) -> None: os_fs = OSFS(os_folder) # the file is on the head node, might not be visible in storage.filesystem fs.copy.copy_file(os_fs, os_file, self.storage.filesystem, fs.path.join(logs_folder, "cluster.yaml")) - def _create_file_handler(self, pipeline_execution_name: str) -> Handler: + def _create_file_handler(self, pipeline_execution_name: str, filename: str) -> Handler: """Creates a logging handler to write a pipeline log to a file.""" - logs_filename = fs.path.combine(self.get_pipeline_logs_folder(pipeline_execution_name), "pipeline.log") + logs_filename = fs.path.combine(self.get_pipeline_logs_folder(pipeline_execution_name), filename) file_handler = RegularBackupHandler( logs_filename, filesystem=self.storage.filesystem, @@ -194,19 +193,18 @@ def _create_stdout_handler(self) -> Handler: return stdout_handler - def stop_logging(self, handlers: list[Handler]) -> None: + def stop_logging(self, logger: logging.Logger, handlers: list[Handler]) -> None: """Updates logs, removes pipeline handlers from the global logger and puts global logging level back to default """ if self.config.capture_warnings: logging.captureWarnings(False) - global_logger = logging.getLogger() for handler in handlers: handler.close() - global_logger.removeHandler(handler) + logger.removeHandler(handler) - global_logger.setLevel(logging.WARNING) + logger.setLevel(logging.WARNING) def update_pipeline_report( self, diff --git a/eogrow/core/pipeline.py b/eogrow/core/pipeline.py index 4552b89c..e20fafa9 100644 --- a/eogrow/core/pipeline.py +++ b/eogrow/core/pipeline.py @@ -222,7 +222,8 @@ def run(self) -> None: timestamp = current_timestamp() self.current_execution_name = self.get_pipeline_execution_name(timestamp) - handlers = self.logging_manager.start_logging(self.current_execution_name) + root_logger = logging.getLogger() + handlers = self.logging_manager.start_logging(root_logger, self.current_execution_name, "pipeline.log") try: self.logging_manager.update_pipeline_report( pipeline_execution_name=self.current_execution_name, @@ -259,7 +260,7 @@ def run(self) -> None: pipeline_execution_name=self.current_execution_name, finished=finished, failed=failed ) finally: - self.logging_manager.stop_logging(handlers) + self.logging_manager.stop_logging(root_logger, handlers) def run_procedure(self) -> tuple[list[str], list[str]]: """Execution procedure of pipeline. Can be overridden if needed.