From 86448020d82a048193037984d4420b89beb5ef25 Mon Sep 17 00:00:00 2001
From: Eugene Batalov <eugene@tensorlake.ai>
Date: Mon, 16 Dec 2024 15:38:58 +0000
Subject: [PATCH] Terminate Indexify processes in the end of GH Action test
 runs

Currently we don't cleanely terminate Indexify Server and Executor
in the end of test runs. This results in a misleading collection
of many orphaned processes in the end of test runs. Do the clean
termination to remove the misleading orphaned processes collection
and to run the code in Executor that terminates Function Executors
cleanely. This ensures better line coverage in our CI GH Actions
testing.
---
 .github/workflows/tests.yaml | 48 +++++++++++++++++++++++++++++++++---
 1 file changed, 45 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index a5fbf4599..ad26e9f3a 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -115,6 +115,7 @@ jobs:
           run: |
             chmod u+x ./indexify-server
             RUST_LOG=debug ./indexify-server &
+            echo $! > /tmp/indexify-server.pid &
 
           wait-on: |
             tcp:localhost:8900
@@ -130,6 +131,7 @@ jobs:
           run: |
             cd python-sdk
             poetry run indexify-cli executor &
+            echo $! > /tmp/indexify-executor.pid &
 
           wait-on: |
             tcp:localhost:8900
@@ -158,12 +160,31 @@ jobs:
                 sleep 5
             fi
           done
-
       - name: Run All Tests
         run: |
           cd python-sdk
           export INDEXIFY_URL=http://localhost:8900
           make test
+      - name: Terminate processes
+        # We want to test clean termination of processes.
+        run: |
+          pkill -SIGTERM -F /tmp/indexify-server.pid
+          pkill -SIGTERM -F /tmp/indexify-executor.pid
+      - name: Wait for processes termination
+        run: |
+            pid_files="/tmp/indexify-server.pid /tmp/indexify-executor.pid"
+            for pid_file in $pid_files; do
+              while pkill -0 -F "$pid_file" 2>/dev/null; do
+                echo "waiting for process $pid_file to exit..."
+                sleep 1
+              done
+            done
+      - name: Validate that all processes terminated
+        run: |
+          if pgrep -f indexify; then
+            echo "Error: Some Indexify processes are still running."
+            exit 1
+          fi
 
   last_release_acceptance_tests:
     name: 'Last Release Acceptance Tests (trigger with label: ci_compat_test)'
@@ -198,6 +219,7 @@ jobs:
           run: |
             chmod u+x ./indexify-server
             RUST_LOG=debug ./indexify-server &
+            echo $! > /tmp/indexify-server.pid &
 
           wait-on: |
             tcp:localhost:8900
@@ -213,6 +235,7 @@ jobs:
           run: |
             cd python-sdk
             poetry run indexify-cli executor &
+            echo $! > /tmp/indexify-executor.pid &
 
           wait-on: |
             tcp:localhost:8900
@@ -241,9 +264,28 @@ jobs:
                 sleep 5
             fi
           done
-
       - name: Run All Tests
         run: |
           cd python-sdk
           export INDEXIFY_URL=http://localhost:8900
-          make test
\ No newline at end of file
+          make test
+      - name: Terminate processes
+        # We want to test clean termination of processes.
+        run: |
+          pkill -SIGTERM -F /tmp/indexify-server.pid
+          pkill -SIGTERM -F /tmp/indexify-executor.pid
+      - name: Wait for processes termination
+        run: |
+            pid_files="/tmp/indexify-server.pid /tmp/indexify-executor.pid"
+            for pid_file in $pid_files; do
+              while pkill -0 -F "$pid_file" 2>/dev/null; do
+                echo "waiting for process $pid_file to exit..."
+                sleep 1
+              done
+            done
+      - name: Validate that all processes terminated
+        run: |
+          if pgrep -f indexify; then
+            echo "Error: Some Indexify processes are still running."
+            exit 1
+          fi
\ No newline at end of file