Merge branch 'main' into Matvey-Kuk/prom-metrics

keep-ui/app/ai/ai.tsx

@@ -200,7 +200,7 @@ export default function Ai() {
                   className={"p-4 flex flex-col w-full border-white border-2"}
                 >
                   <h3 className="text-lg sm:text-xl font-semibold line-clamp-2">
-                    Summorization v0.1
+                    Summarization v0.1
                   </h3>
                   <p className="text-sm top-0">
                     Using LLMs to provide a human-readable incident summary.

keep-ui/app/incidents/create-or-update-incident.tsx

@@ -117,8 +117,7 @@ export default function CreateOrUpdateIncident({
 
   const submitEnabled = (): boolean => {
     return (
-      !!incidentName &&
-      !!incidentUserSummary
+      !!incidentName
     );
   };
 
@@ -140,10 +139,10 @@ export default function CreateOrUpdateIncident({
         />
       </div>
       <div className="mt-2.5">
-        <Text>Summary<span className="text-red-500 text-xs">*</span></Text>
+        <Text>Summary</Text>
         <Textarea
           placeholder="What happened?"
-          required={true}
+          required={false}
           value={incidentUserSummary}
           onValueChange={setIncidentUserSummary}
         />

keep-ui/components/navbar/AlertsLinks.tsx

@@ -55,7 +55,6 @@ export const AlertsLinks = ({ session }: AlertsLinksProps) => {
   const dismissedPreset = staticPresets.find(
     (preset) => preset.name === "dismissed"
   );
-  const groupsPreset = staticPresets.find((preset) => preset.name === "groups");
 
   const handleTagSelect = (
     newValue: MultiValue<{ value: string; label: string }>,

keep/api/api.py

@@ -19,8 +19,9 @@
 import keep.api.logging
 import keep.api.observability
 from keep.api.arq_worker import get_arq_worker
+from keep.api.consts import KEEP_ARQ_TASK_POOL, KEEP_ARQ_TASK_POOL_NONE
 from keep.api.core.config import AuthenticationType
-from keep.api.core.db import get_user
+from keep.api.core.db import get_api_key, get_user
 from keep.api.core.dependencies import SINGLE_TENANT_UUID
 from keep.api.logging import CONFIG as logging_config
 from keep.api.routes import (
@@ -48,10 +49,6 @@
 from keep.event_subscriber.event_subscriber import EventSubscriber
 from keep.posthog.posthog import get_posthog_client
 from keep.workflowmanager.workflowmanager import WorkflowManager
-from keep.api.consts import (
-    KEEP_ARQ_TASK_POOL, 
-    KEEP_ARQ_TASK_POOL_NONE,
-)
 
 load_dotenv(find_dotenv())
 keep.api.logging.setup_logging()
@@ -87,6 +84,17 @@ def _extract_identity(request: Request, attribute="email") -> str:
         token = request.headers.get("Authorization").split(" ")[1]
         decoded_token = jwt.decode(token, options={"verify_signature": False})
         return decoded_token.get(attribute)
+    # case api key
+    except AttributeError:
+        # try api key
+        api_key = request.headers.get("x-api-key")
+        if not api_key:
+            return "anonymous"
+
+        api_key = get_api_key(api_key)
+        if api_key:
+            return api_key.tenant_id
+        return "anonymous"
     except Exception:
         return "anonymous"
 

keep/api/consts.py

@@ -24,20 +24,6 @@
         static=True,
         tags=[],
     ),
-    "groups": PresetDto(
-        id=StaticPresetsId.GROUPS_PRESET_ID.value,
-        name="groups",
-        options=[
-            {"label": "CEL", "value": "group"},
-            {"label": "SQL", "value": {"sql": '"group"=true', "params": {}}},
-        ],
-        created_by=None,
-        is_private=False,
-        is_noisy=False,
-        should_do_noise_now=False,
-        static=True,
-        tags=[],
-    ),
     "dismissed": PresetDto(
         id=StaticPresetsId.DISMISSED_PRESET_ID.value,
         name="dismissed",
@@ -55,14 +41,14 @@
 }
 
 ###
-# Set ARQ_TASK_POOL_TO_EXECUTE to "none", "all", "basic_processing" or "ai" 
+# Set ARQ_TASK_POOL_TO_EXECUTE to "none", "all", "basic_processing" or "ai"
 # to split the tasks between the workers.
 ###
 
 KEEP_ARQ_TASK_POOL_NONE = "none"  # Arq workers explicitly disabled for this service
 KEEP_ARQ_TASK_POOL_ALL = "all"  # All arq workers enabled for this service
-KEEP_ARQ_TASK_POOL_BASIC_PROCESSING = "basic_processing" # Everything except AI
-KEEP_ARQ_TASK_POOL_AI = "ai" # Only AI
+KEEP_ARQ_TASK_POOL_BASIC_PROCESSING = "basic_processing"  # Everything except AI
+KEEP_ARQ_TASK_POOL_AI = "ai"  # Only AI
 
 REDIS = os.environ.get("REDIS", "false") == "true"
 KEEP_ARQ_TASK_POOL = os.environ.get("KEEP_ARQ_TASK_POOL", None)

keep/api/core/db.py

@@ -2303,6 +2303,18 @@ def add_alerts_to_incident_by_incident_id(
         return incident
 
 
+def get_incident_unique_fingerprint_count(tenant_id: str, incident_id: str) -> int:
+    with Session(engine) as session:
+        return session.execute(
+            select(func.count(1))
+            .select_from(AlertToIncident)
+            .join(Alert, AlertToIncident.alert_id == Alert.id)
+            .where(
+                Alert.tenant_id == tenant_id,
+                AlertToIncident.incident_id == incident_id,
+            )
+        ).scalar()
+
 def remove_alerts_to_incident_by_incident_id(
     tenant_id: str, incident_id: str | UUID, alert_ids: List[UUID]
 ) -> Optional[int]:

keep/api/routes/incidents.py

@@ -11,6 +11,7 @@
 from pusher import Pusher
 from pydantic.types import UUID
 
+from keep.api.arq_pool import get_pool
 from keep.api.core.db import (
     add_alerts_to_incident_by_incident_id,
     confirm_predicted_incident_by_id,
@@ -21,6 +22,7 @@
     get_last_incidents,
     remove_alerts_to_incident_by_incident_id,
     update_incident_from_dto_by_id,
+    get_incident_unique_fingerprint_count,
 )
 from keep.api.core.dependencies import (
     AuthenticatedEntity,
@@ -35,13 +37,15 @@
 router = APIRouter()
 logger = logging.getLogger(__name__)
 
+MIN_INCIDENT_ALERTS_FOR_SUMMARY_GENERATION = int(os.environ.get("MIN_INCIDENT_ALERTS_FOR_SUMMARY_GENERATION", 5))
+
 ee_enabled = os.environ.get("EE_ENABLED", "false") == "true"
 if ee_enabled:
     path_with_ee = (
         str(pathlib.Path(__file__).parent.resolve()) + "/../../../ee/experimental"
     )
     sys.path.insert(0, path_with_ee)
-    from ee.experimental.incident_utils import mine_incidents  # noqa
+    from ee.experimental.incident_utils import mine_incidents, ALGORITHM_VERBOSE_NAME  # noqa
 
 
 def __update_client_on_incident_change(
@@ -277,7 +281,7 @@ def get_incident_alerts(
     status_code=202,
     response_model=List[AlertDto],
 )
-def add_alerts_to_incident(
+async def add_alerts_to_incident(
     incident_id: str,
     alert_ids: List[UUID],
     authenticated_entity: AuthenticatedEntity = Depends(AuthVerifier(["read:alert"])),
@@ -298,6 +302,21 @@ def add_alerts_to_incident(
     add_alerts_to_incident_by_incident_id(tenant_id, incident_id, alert_ids)
     __update_client_on_incident_change(pusher_client, tenant_id, incident_id)
 
+    fingerprints_count = get_incident_unique_fingerprint_count(tenant_id, incident_id)
+
+    if ee_enabled and fingerprints_count > MIN_INCIDENT_ALERTS_FOR_SUMMARY_GENERATION and not incident.user_summary:
+        pool = await get_pool()
+        job = await pool.enqueue_job(
+            "process_summary_generation",
+            tenant_id=tenant_id,
+            incident_id=incident_id,
+        )
+        logger.info(
+            f"Summary generation for incident {incident_id} scheduled, job: {job}",
+            extra={"algorithm": ALGORITHM_VERBOSE_NAME,
+                   "tenant_id": tenant_id, "incident_id": incident_id},
+        )
+
     return Response(status_code=202)
 
 

keep/api/routes/preset.py

@@ -148,7 +148,6 @@ def get_presets(
     presets_dto = [PresetDto(**preset.to_dict()) for preset in presets]
     # add static presets
     presets_dto.append(STATIC_PRESETS["feed"])
-    presets_dto.append(STATIC_PRESETS["groups"])
     presets_dto.append(STATIC_PRESETS["dismissed"])
     logger.info("Got all presets")
 

keep/api/tasks/process_event_task.py

@@ -315,7 +315,7 @@ def __handle_formatted_events(
     # Now we need to run the rules engine
     try:
         rules_engine = RulesEngine(tenant_id=tenant_id)
-        incidents: List[IncidentDto] = rules_engine.run_rules(formatted_events)
+        incidents: List[IncidentDto] = rules_engine.run_rules(enriched_formatted_events)
 
         # TODO: Replace with incidents workflow triggers. Ticket: https://github.com/keephq/keep/issues/1527
         # if new grouped incidents were created, we need to push them to the client

keep/providers/openobserve_provider/openobserve_provider.py

@@ -387,6 +387,7 @@ def _format_alert(
         description = event.pop("alert_type", "")
 
         alert_url = event.pop("alert_url", "")
+
         org_name = event.pop("org_name", "")
         # Our only way to distinguish between non aggregated alert and aggregated alerts is the alert_agg_value
         if "alert_agg_value" in event and len(
@@ -435,7 +436,15 @@ def _format_alert(
                             "group_by_value": group_by_value,
                         },
                     )
+
                     alert_id = str(uuid.uuid4())
+
+                    # we already take the value from the agg_value
+                    event.pop("value", "")
+                    # if the group_by_key is already in the event, remove it
+                    #   since we are adding it to the alert_dto
+                    event.pop(group_by_key, "")
+
                     alert_dto = AlertDto(
                         id=f"{alert_id}",
                         name=f"{name}",
@@ -448,7 +457,7 @@ def _format_alert(
                         source=["openobserve"],
                         org_name=org_name,
                         value=value,
-                        url=alert_url,
+                        alert_url=alert_url,  # I'm not putting on URL since sometimes it doesn't return full URL so pydantic will throw an error
                         **event,
                         **{group_by_key: group_by_value},
                     )
@@ -487,7 +496,7 @@ def _format_alert(
                 labels=labels,
                 source=["openobserve"],
                 org_name=org_name,
-                url=alert_url,
+                alert_url=alert_url,  # I'm not putting on URL since sometimes it doesn't return full URL so pydantic will throw an error
                 **event,  # any other fields
             )
             # calculate fingerprint based on name + environment + event keys (e.g. host)

keep/providers/zabbix_provider/zabbix_provider.py

@@ -5,6 +5,7 @@
 import dataclasses
 import datetime
 import json
+import logging
 import os
 import random
 from typing import Literal, Optional
@@ -20,6 +21,8 @@
 from keep.providers.models.provider_method import ProviderMethod
 from keep.providers.providers_factory import ProvidersFactory
 
+logger = logging.getLogger(__name__)
+
 
 @pydantic.dataclasses.dataclass
 class ZabbixProviderAuthConfig:
@@ -559,10 +562,13 @@ def _format_alert(
         event: dict, provider_instance: Optional["ZabbixProvider"] = None
     ) -> AlertDto:
         environment = "unknown"
-        tags = {
-            tag.get("tag"): tag.get("value")
-            for tag in json.loads(event.pop("tags", "[]"))
-        }
+        tags_raw = event.pop("tags", "[]")
+        try:
+            tags = {tag.get("tag"): tag.get("value") for tag in json.loads(tags_raw)}
+        except json.JSONDecodeError:
+            logger.error("Failed to extract Zabbix tags", extra={"tags_raw": tags_raw})
+            # We failed to extract tags for some reason.
+            tags = {}
         if isinstance(tags, dict):
             environment = tags.pop("environment", "unknown")
             # environment exists in tags but is None

keep/providers/zabbix_provider/zabbix_provider_script.js

@@ -35,19 +35,11 @@ try {
   resp = req.post(keepApiUrl, JSON.stringify(params));
   Zabbix.log(4, '[Keep Webhook] Received response: HTTP ' + req.getStatus() + ': ' + resp);
 
-  if (req.getStatus() != 200) {
-      throw 'Response code not 200';
-      Zabbix.log(3, '[Keep Webhook] Error:' + JSON.stringify(resp));
+  if (req.getStatus() != 202) {
+    throw 'Response code not 202';
   }
   else {
-      try {
-          resp = JSON.parse(resp);
-      }
-      catch (error) {
-          throw 'Incorrect response. Keep returned a non-JSON object.';
-          Zabbix.log(3, '[Keep Webhook] Error: Incorrect response. Keep returned a non-JSON object ' + JSON.stringify(resp));
-      }
-      return JSON.stringify(result);
+    return resp;
   }
 }
 catch (error) {

tests/e2e_tests/docker-compose-e2e-mysql.yml

@@ -26,8 +26,6 @@ services:
       - AUTH_TYPE=NO_AUTH
       - API_URL=http://keep-backend:8080
       - POSTHOG_DISABLED=true
-    depends_on:
-      - keep-backend
 
   keep-backend:
     extends:

tests/e2e_tests/docker-compose-e2e-postgres.yml

@@ -24,8 +24,6 @@ services:
       - AUTH_TYPE=NO_AUTH
       - API_URL=http://keep-backend:8080
       - POSTHOG_DISABLED=true
-    depends_on:
-      - keep-backend
 
   keep-backend:
     extends: