Merge branch 'develop' into dependabot/pip/stix_shifter/aioboto3-12.1.0

stix_shifter_modules/elastic_ecs/stix_translation/json/beats_from_stix_map.json

@@ -43,16 +43,16 @@
   },
   "file": {
     "fields": {
-      "name": ["file.name", "dll.name", "file.path", "process.name.keyword", "process.executable.keyword", "process.parent.name.keyword", "process.parent.executable.keyword"],
+      "name": ["file.name", "dll.name", "file.path", "process.name.keyword", "process.executable.keyword", "process.parent.name.keyword", "process.parent.executable.keyword", "email.attachments.file.name"],
       "created": ["file.created", "file.ctime"],
       "modified": ["file.mtime"],
       "accessed": ["file.accessed"],
       "size": ["file.size"],
       "mime_type": ["file.mime_type"],
-      "hashes.MD5": ["file.hash.md5"],
-      "hashes.'SHA-1'": ["file.hash.sha1"],
-      "hashes.'SHA-256'": ["file.hash.sha256"],
-      "hashes.'SHA-512'": ["file.hash.sha512"],
+      "hashes.MD5": ["file.hash.md5", "email.attachments.file.hash.md5"],
+      "hashes.'SHA-1'": ["file.hash.sha1", "email.attachments.file.hash.sha1"],
+      "hashes.'SHA-256'": ["file.hash.sha256", "email.attachments.file.hash.256"],
+      "hashes.'SHA-512'": ["file.hash.sha512", "email.attachments.file.hash.512"],
       "parent_directory_ref.path": ["file.directory"],
       "x_attributes": ["file.attributes"],
       "x_extension": ["file.extension"],

stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json

@@ -49,10 +49,10 @@
       "accessed": ["file.accessed"],
       "size": ["file.size"],
       "mime_type": ["file.mime_type"],
-      "hashes.MD5": ["file.hash.md5"],
-      "hashes.'SHA-1'": ["file.hash.sha1"],
-      "hashes.'SHA-256'": ["file.hash.sha256"],
-      "hashes.'SHA-512'": ["file.hash.sha512"],
+      "hashes.MD5": ["file.hash.md5", "email.attachments.file.hash.md5"],
+      "hashes.'SHA-1'": ["file.hash.sha1", "email.attachments.file.hash.sha1"],
+      "hashes.'SHA-256'": ["file.hash.sha256", "email.attachments.file.hash.256"],
+      "hashes.'SHA-512'": ["file.hash.sha512", "email.attachments.file.hash.512"],
       "parent_directory_ref.path": ["file.directory"],
       "x_attributes": ["file.attributes"],
       "x_extension": ["file.extension"],

stix_shifter_modules/elastic_ecs/stix_translation/json/to_stix_map.json

@@ -2619,12 +2619,14 @@
   "email": {
     "subject": {
       "key": "email-message.subject",
+      "unwrap": true,
       "object": "mail"
     },
     "from": {
       "address": [
         {
           "key": "email-addr.value",
+          "unwrap": true,
           "object": "emailfrom"
         },
         {
@@ -2666,7 +2668,30 @@
       "file": {
         "name": {
           "key": "file.name",
+          "transformer": "UnlistIfList",
           "object": "attachment"
+        },
+        "hash": {
+          "md5": {
+            "key": "file.hashes.MD5",
+            "transformer": "UnlistIfList",
+            "object": "attachment"
+          },
+          "sha1": {
+            "key": "file.hashes.SHA-1",
+            "transformer": "UnlistIfList",
+            "object": "attachment"
+          },
+          "sha256": {
+            "key": "file.hashes.SHA-256",
+            "transformer": "UnlistIfList",
+            "object": "attachment"
+          },
+          "sha512": {
+            "key": "file.hashes.SHA-512",
+            "transformer": "UnlistIfList",
+            "object": "attachment"
+          }
         }
       }
     }

stix_shifter_modules/elastic_ecs/stix_transmission/api_client.py

@@ -66,7 +66,7 @@ async def search_pagination(self, query_expression, lastsortvalue=None, length=D
             "_source": {
                 "includes": ["@timestamp", "source.*", "destination.*", "event.*", "client.*", "server.*", "observer.*",
                              "host.*", "network.*", "process.*", "user.*", "file.*", "url.*", "registry.*", "dns.*",
-                             "tags"]
+                             "email.*", "tags"]
             },
             "size": length,
             "query": {

stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_json_to_stix.py

@@ -403,6 +403,32 @@
     }
 }
 
+email_data2 = {
+    "email": {
+        "attachments": [
+            { "file": { "name": [ "tabby.html" ]
+                      , "mime_type": [ "text/html" ]
+                      , "hash": { "md5": [ "5fbb1f4a0b7f798bb3f74e64bfeaeea2" ]
+                                , "sha256": [ "29952af0823274dd686f8969968f07d618579dbb0159a8f8f9ca2a6e35dd9a98" ]
+                                }
+                      }
+            },
+            { "file": { "name": [ "tabby.zip" ]
+                      , "mime_type": [ "application/zip" ]
+                      , "hash": { "md5": [ "187e31980d3368428c88d7a4e8da4572" ]
+                                , "sha256": [ "08756f43e5536eaa1500ea10adb88e67816f5586a5dde1d2a4f77c814d8abc57" ]
+                                }
+                      }
+            }
+        ],
+        "subject": [ "Check out this picture of a cat!" ],
+        "from": { "address": [ "[email protected]" ] },
+        "to": { "address": [ "[email protected]" ] },
+        "cc": { "address": [ "[email protected]" ] }
+    }
+}
+
+
 class TestElasticEcsTransform(unittest.TestCase, object):
     @staticmethod
     def get_first(itr, constraint):
@@ -881,8 +907,6 @@ def test_email(self):
         observed_data = translation_objects[1]
         stix_objects = observed_data.get("objects")
         assert len(stix_objects) == 8
-        for k,v in stix_objects.items():
-            print(f"{k}: {v}")
         emailmsg = stix_objects.get("2")
         assert (
             emailmsg and emailmsg.get("type") == "email-message" and
@@ -894,3 +918,37 @@ def test_email(self):
         assert to_emails == {"[email protected]", "[email protected]"}
         filenames = {stix_objects[str(ref)]["name"] for ref in range(len(stix_objects)) if stix_objects[str(ref)] and stix_objects[str(ref)]["type"] == "file"}
         assert filenames == {"tabby.zip", "tabby.html"}
+
+    def test_email2(self):
+        result_bundle = run_in_thread(entry_point.translate_results, data_source, [email_data2])
+        assert (result_bundle['type'] == 'bundle')
+        translation_objects = result_bundle.get('objects')
+        assert (translation_objects and len(translation_objects) == 2)
+        observed_data = translation_objects[1]
+        stix_objects = observed_data.get("objects")
+        assert len(stix_objects) == 6
+        emailmsg = stix_objects.get("2")
+        assert (
+            emailmsg and emailmsg.get("type") == "email-message" and
+            emailmsg.get("subject") == "Check out this picture of a cat!"
+        )
+        from_email = stix_objects.get(emailmsg.get("from_ref"))
+        assert from_email and from_email.get("value") == "[email protected]"
+        to_emails = [stix_objects.get(ref).get("value") for ref in emailmsg.get("to_refs")]
+        assert to_emails == ["[email protected]"]
+        to_emails = [stix_objects.get(ref).get("value") for ref in emailmsg.get("cc_refs")]
+        assert to_emails == ["[email protected]"]
+        file0 = stix_objects.get("0")
+        assert (
+            file0 and file0.get("type") == "file" and
+            file0.get("name") == "tabby.html" and
+            file0.get("hashes").get("MD5") == "5fbb1f4a0b7f798bb3f74e64bfeaeea2" and
+            file0.get("hashes").get("SHA-256") == "29952af0823274dd686f8969968f07d618579dbb0159a8f8f9ca2a6e35dd9a98"
+        )
+        file1 = stix_objects.get("1")
+        assert (
+            file1 and file1.get("type") == "file" and
+            file1.get("name") == "tabby.zip" and
+            file1.get("hashes").get("MD5") == "187e31980d3368428c88d7a4e8da4572" and
+            file1.get("hashes").get("SHA-256") == "08756f43e5536eaa1500ea10adb88e67816f5586a5dde1d2a4f77c814d8abc57"
+        )