Store exception traceback in logs f pipeline fails

eogrow/core/pipeline.py

@@ -4,9 +4,11 @@
 
 import logging
 import time
+import traceback
 import uuid
 from typing import Any, TypeVar
 
+import fs
 import ray
 
 from eolearn.core import CreateEOPatchTask, EOExecutor, EONode, EOWorkflow, LoadTask, SaveTask, WorkflowResults
@@ -255,6 +257,14 @@ def run(self) -> None:
 
             if failed and self.config.raise_on_failure:
                 raise PipelineExecutionError(f"Pipeline failed some executions. Check {log_folder}.")
+        except Exception as e:
+            # store any exception info you can
+            failure_log_path = fs.path.join(
+                self.logging_manager.get_pipeline_logs_folder(self.current_execution_name), "failure.log"
+            )
+            with self.storage.filesystem.open(failure_log_path, mode="w") as log_file:
+                log_file.writelines(traceback.format_exc())
+            raise e
         finally:
             self.logging_manager.stop_logging(root_logger, handlers)
 

tests/core/test_pipeline.py

@@ -1,5 +1,7 @@
 import logging
 import os
+from contextlib import suppress
+from pathlib import Path
 from typing import List, Tuple, Union
 
 import pytest
@@ -98,6 +100,23 @@ def test_get_patch_list_filtration_error(test_subset: List[Union[int, str]], sim
         pipeline.get_patch_list()
 
 
+def test_pipeline_logs_exception(simple_config_filename: str) -> None:
+    def cookie_alarm():
+        raise RuntimeError("Oh no, someone stole my cookie! :(")
+
+    config = interpret_config_from_path(simple_config_filename)
+    pipeline = SimplePipeline.from_raw_config(config)
+    pipeline.run_procedure = cookie_alarm
+
+    with suppress(RuntimeError):
+        pipeline.run()
+
+    log_folder = pipeline.logging_manager.get_pipeline_logs_folder(pipeline.current_execution_name, full_path=True)
+    log_path = Path(log_folder) / "failure.log"
+    assert log_path.exists()
+    assert "Oh no, someone stole my cookie! :(" in log_path.read_text()
+
+
 @pytest.mark.parametrize("fail", [True, False])
 @pytest.mark.parametrize("raise_on_failure", [True, False])
 def test_pipeline_raises_on_failure(fail: bool, raise_on_failure: bool, simple_config_filename: str):