Respond to comments

Changed criteria to be either DOWN or DRAINED
Added the exception into hv_patch_and_reboot

lib/workflows/hv_patch_and_reboot.py

@@ -1,6 +1,8 @@
 import datetime
+
+from paramiko import SSHException
 from enums.icinga.icinga_objects import IcingaObject
-from icinga_api.downtime import schedule_downtime
+from icinga_api.downtime import schedule_downtime, remove_downtime
 from structs.icinga.downtime_details import DowntimeDetails
 from structs.icinga.icinga_account import IcingaAccount
 from structs.ssh.ssh_connection_details import SSHDetails
@@ -37,6 +39,17 @@ def patch_and_reboot(
         duration=end_timestamp - start_timestamp,
     )
     schedule_downtime(icinga_account=icinga_account, details=downtime_details)
-    patch_out = ssh_client.run_command_on_host("patch")
-    reboot_out = ssh_client.run_command_on_host("reboot")
-    return {"patch_output": patch_out.decode(), "reboot_output": reboot_out.decode()}
+    try:
+        patch_out = ssh_client.run_command_on_host("patch")
+        reboot_out = ssh_client.run_command_on_host("reboot")
+        return {
+            "patch_output": patch_out.decode(),
+            "reboot_output": reboot_out.decode(),
+        }
+    except SSHException as exc:
+        remove_downtime(
+            icinga_account=icinga_account,
+            object_type=IcingaObject.HOST,
+            object_name=hypervisor_name,
+        )
+        return {"error": f"SSHException occurred: {str(exc)}"}

rules/hv.remove.downtime.yaml

@@ -6,8 +6,8 @@ enabled: true
 
 criteria:
   trigger.previous_state:
-    type: equals
-    pattern: DOWN
+    type: regex
+    pattern: (DOWN|DRAINED)
   trigger.current_state:
     type: equals
     pattern: REBOOTED

sensors/hypervisor.state_change.yaml

@@ -2,7 +2,7 @@
 class_name: HypervisorStateSensor
 entry_point: src/hypervisor_state_sensor.py
 description: Monitor state of Hypervisors
-poll_interval: 30
+poll_interval: 600
 trigger_types:
   - name: "hypervisor.state_change"
     description: "Triggers when the state of the hypervisor changes"

tests/lib/workflows/test_hv_patch_and_reboot.py

@@ -1,5 +1,7 @@
 import datetime
 from unittest.mock import MagicMock, patch
+
+from paramiko import SSHException
 from enums.icinga.icinga_objects import IcingaObject
 from structs.icinga.downtime_details import DowntimeDetails
 from structs.ssh.ssh_connection_details import SSHDetails
@@ -60,7 +62,10 @@ def test_successful_patch_and_reboot(
 
 @patch("workflows.hv_patch_and_reboot.schedule_downtime")
 @patch("workflows.hv_patch_and_reboot.SSHConnection")
-def test_failed_schedule_downtime(mock_ssh_conn, mock_schedule_downtime):
+@patch("workflows.hv_patch_and_reboot.remove_downtime")
+def test_failed_schedule_downtime(
+    mock_remove_downtime, mock_ssh_conn, mock_schedule_downtime
+):
     """
     Test unsuccessful running of patch and reboot workflow - where the schedule
     downtime raises an exception.
@@ -87,26 +92,27 @@ def test_failed_schedule_downtime(mock_ssh_conn, mock_schedule_downtime):
     )
 
     mock_ssh_conn.return_value.run_command_on_host.assert_not_called()
+    mock_remove_downtime.assert_not_called()
 
 
+@patch("workflows.hv_patch_and_reboot.remove_downtime")
 @patch("workflows.hv_patch_and_reboot.schedule_downtime")
 @patch("workflows.hv_patch_and_reboot.SSHConnection")
-def test_failed_ssh(mock_ssh_conn, mock_schedule_downtime):
+def test_failed_ssh(mock_ssh_conn, mock_schedule_downtime, mock_remove_downtime):
     """
     Test unsuccessful running of patch and reboot workflow - where either ssh command
     fails
     """
     icinga_account = MagicMock()
     mock_hypervisor_name = "test_host"
     mock_private_key_path = "/home/stackstorm/.ssh/id_rsa"
-    mock_ssh_conn.return_value.run_command_on_host.side_effect = Exception
+    mock_ssh_conn.return_value.run_command_on_host.side_effect = SSHException
 
-    with pytest.raises(Exception):
-        patch_and_reboot(
-            icinga_account,
-            hypervisor_name=mock_hypervisor_name,
-            private_key_path=mock_private_key_path,
-        )
+    patch_and_reboot(
+        icinga_account,
+        hypervisor_name=mock_hypervisor_name,
+        private_key_path=mock_private_key_path,
+    )
 
     mock_ssh_conn.assert_called_once_with(
         SSHDetails(
@@ -131,3 +137,8 @@ def test_failed_ssh(mock_ssh_conn, mock_schedule_downtime):
             duration=mock_end_timestamp - mock_start_timestamp,
         ),
     )
+    mock_remove_downtime.assert_called_once_with(
+        icinga_account=icinga_account,
+        object_type=IcingaObject.HOST,
+        object_name=mock_hypervisor_name,
+    )