From f9052550c3b775047e3090c49545793329106dd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=BDiga=20Luk=C5=A1i=C4=8D?=
 <31988337+zigaLuksic@users.noreply.github.com>
Date: Fri, 8 Mar 2024 15:18:23 +0100
Subject: [PATCH] Store exception traceback in logs f pipeline fails (#331)

* add test

* store failure info
---
 eogrow/core/pipeline.py     | 10 ++++++++++
 tests/core/test_pipeline.py | 19 +++++++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/eogrow/core/pipeline.py b/eogrow/core/pipeline.py
index 78db70f6..39d33ea1 100644
--- a/eogrow/core/pipeline.py
+++ b/eogrow/core/pipeline.py
@@ -4,9 +4,11 @@
 
 import logging
 import time
+import traceback
 import uuid
 from typing import Any, TypeVar
 
+import fs
 import ray
 
 from eolearn.core import CreateEOPatchTask, EOExecutor, EONode, EOWorkflow, LoadTask, SaveTask, WorkflowResults
@@ -255,6 +257,14 @@ def run(self) -> None:
 
             if failed and self.config.raise_on_failure:
                 raise PipelineExecutionError(f"Pipeline failed some executions. Check {log_folder}.")
+        except Exception as e:
+            # store any exception info you can
+            failure_log_path = fs.path.join(
+                self.logging_manager.get_pipeline_logs_folder(self.current_execution_name), "failure.log"
+            )
+            with self.storage.filesystem.open(failure_log_path, mode="w") as log_file:
+                log_file.writelines(traceback.format_exc())
+            raise e
         finally:
             self.logging_manager.stop_logging(root_logger, handlers)
 
diff --git a/tests/core/test_pipeline.py b/tests/core/test_pipeline.py
index 13b7df0d..1fe1c13c 100644
--- a/tests/core/test_pipeline.py
+++ b/tests/core/test_pipeline.py
@@ -1,5 +1,7 @@
 import logging
 import os
+from contextlib import suppress
+from pathlib import Path
 from typing import List, Tuple, Union
 
 import pytest
@@ -98,6 +100,23 @@ def test_get_patch_list_filtration_error(test_subset: List[Union[int, str]], sim
         pipeline.get_patch_list()
 
 
+def test_pipeline_logs_exception(simple_config_filename: str) -> None:
+    def cookie_alarm():
+        raise RuntimeError("Oh no, someone stole my cookie! :(")
+
+    config = interpret_config_from_path(simple_config_filename)
+    pipeline = SimplePipeline.from_raw_config(config)
+    pipeline.run_procedure = cookie_alarm
+
+    with suppress(RuntimeError):
+        pipeline.run()
+
+    log_folder = pipeline.logging_manager.get_pipeline_logs_folder(pipeline.current_execution_name, full_path=True)
+    log_path = Path(log_folder) / "failure.log"
+    assert log_path.exists()
+    assert "Oh no, someone stole my cookie! :(" in log_path.read_text()
+
+
 @pytest.mark.parametrize("fail", [True, False])
 @pytest.mark.parametrize("raise_on_failure", [True, False])
 def test_pipeline_raises_on_failure(fail: bool, raise_on_failure: bool, simple_config_filename: str):