Merge pull request #570 from opencybersecurityalliance/k2-kestrel-too…

…l-suffix-fix

fix kestrel-tool suffix field handling

mapping-examples/security-datasets/GoldenSAML/GoldenSAML_Microsoft365DefenderEvents.yaml

@@ -85,14 +85,14 @@ http_request:
 
 # https://schema.ocsf.io/1.1.0/objects/query_info
 query_info:
-    uid: ReportId_long
-    attr_list: AdditionalFields_string.AttributeList
-    search_filter: AdditionalFields_string.SearchFilter
+    uid: ReportId
+    attr_list: AdditionalFields.AttributeList
+    search_filter: AdditionalFields.SearchFilter
 
 
 # https://schema.ocsf.io/1.1.0/objects/managed_entity
 entity:
-    uid: ReportId_long
+    uid: ReportId
     data: ActivityObjects
 
 

packages/kestrel_core/src/kestrel/mapping/fields/ecs.yaml

@@ -1,6 +1,10 @@
 # "*" is for entity/event projection mapping besides a single field
 # if the submap is referred, there will be multiple reversed mappings, the first will be used
 
+
+time: "@timestamp"
+
+
 # endpoint: see https://schema.ocsf.io/1.1.0/objects/endpoint
 device: &endpoint
     "*": host.*

packages/kestrel_core/src/kestrel/mapping/fields/stix.yaml

@@ -1,5 +1,9 @@
 # "*" is for entity/event projection mapping besides a single field
 
+
+time: timestamp
+
+
 # https://schema.ocsf.io/1.1.0/objects/file
 file:
     "*": file.*

packages/kestrel_tool/src/kestrel_tool/mkdb.py

@@ -49,6 +49,18 @@ def _normalize_event(event: dict) -> dict:
             except json.JSONDecodeError:
                 pass  # maybe it's NOT JSON
 
+    for k in list(event):
+        if k.endswith("_string"):
+            base_key = k[:-7]
+            if base_key not in event or not event[base_key]:
+                event[base_key] = event[k]
+                del event[k]
+        if k.endswith("_long"):
+            base_key = k[:-5]
+            if base_key not in event or not event[base_key]:
+                event[base_key] = int(event[k])
+                del event[k]
+
     return event