Unfurl integration

data/context_links.yaml

@@ -3,8 +3,32 @@
 # ------------------------------------------------------------------------
 #
 # This is a config file to define context links for event attributes.
+# Documentation: https://timesketch.org/guides/admin/context-links/
 #
-# Each context link consists of the following fields:
+# There are two types of context links:
+#
+# 1. Hardcoded modules: These are modules that are hardcoded into Timesketch.
+# The config is used to define the match fields for the module.
+#
+#   module_name:
+#
+#       match_fields:       Type: list[str] | List of field keys where
+#                           this context link should be available. Will
+#                           be checked as case insensitive!
+#
+#       validation_regex:   Type: str | OPTIONAL
+#                           A regex pattern that needs to be
+#                           matched by the field value to to make the
+#                           context link available. This can be used to
+#                           validate the format of a value (e.g. a hash).
+#
+# Currnetly supported modules are:
+#   - XML formatter: Displays a formatted XML in a pop-up dialog.
+#   - Unfurl graph: Displays a graph of an URL using unfurl results.
+#
+# 2. External services: These are context links that are defined by each admin.
+# Those links use external services to provide additional information about the
+# attribute value.
 #
 #   context_link_name:
 #
@@ -37,10 +61,28 @@
 #                                    pages.)
 #
 # ------------------------------------------------------------------------
-## Virustotal Example:
-# virustotal_hash_lookup:
-#   short_name: 'VirusTotal'
-#   match_fields: ['hash', 'sha256_hash', 'sha256', 'sha1_hash', 'sha1', 'md5_hash', 'md5']
-#   validation_regex: '/^[0-9a-f]{64}$|^[0-9a-f]{40}$|^[0-9a-f]{32}$/i'
-#   context_link: 'https://www.virustotal.com/gui/search/<ATTR_VALUE>'
-#   redirect_warning: TRUE
+## Hardcoded Modules
+hardcoded_modules:
+### format xml dialog
+  xml_formatter:
+    short_name: 'Prettify XML'
+    match_fields:
+      - xml
+      - xml_string
+### unfurl dialog
+  unfurl_graph:
+    short_name: 'Unfurl URL'
+    match_fields:
+      - url
+      - uri
+      - original_url
+
+## External Services
+linked_services:
+### Virustotal Example:
+  # virustotal_hash_lookup:
+  #   short_name: 'VirusTotal'
+  #   match_fields: ['hash', 'sha256_hash', 'sha256', 'sha1_hash', 'sha1', 'md5_hash', 'md5', 'url']
+  #   validation_regex: '/^[0-9a-f]{64}$|^[0-9a-f]{40}$|^[0-9a-f]{32}$/i'
+  #   context_link: 'https://www.virustotal.com/gui/search/<ATTR_VALUE>'
+  #   redirect_warning: TRUE

requirements.txt

@@ -3,6 +3,7 @@ altair==4.1.0
 celery==5.2.7
 cryptography==41.0.4
 datasketch==1.5.0
+dfir-unfurl==20230901
 opensearch-py==2.3.1
 Flask==2.3.2
 flask_bcrypt==1.0.1

test_tools/test_events/mock_context_links.yaml

@@ -1,20 +1,37 @@
-## Mock configuration file for testing the contrext links API endpoint!
-lookupone:
-  short_name: 'LookupOne'
-  match_fields: ['hash']
-  validation_regex: '/^[0-9a-f]{40}$|^[0-9a-f]{32}$/i'
-  context_link: 'https://lookupone.local/q=<ATTR_VALUE>'
-  redirect_warning: TRUE
+# Mock configuration file for testing the context links API endpoint!
+## Hardcoded Modules
+hardcoded_modules:
+  module_one:
+    short_name: "ModuleOne"
+    validation_regex: "/^[0-9a-f]{64}$/i"
+    match_fields:
+      - xml
+      - xml_string
+  module_two:
+    short_name: "ModuleTwo"
+    match_fields:
+      - url
+      - uri
+      - original_url
 
-lookuptwo:
-  short_name: 'LookupTwo'
-  match_fields: ['sha256_hash', 'hash']
-  validation_regex: '/^[0-9a-f]{64}$/i'
-  context_link: 'https://lookuptwo.local/q=<ATTR_VALUE>'
-  redirect_warning: FALSE
+## External Services
+linked_services:
+  lookupone:
+    short_name: "LookupOne"
+    match_fields: ["hash"]
+    validation_regex: "/^[0-9a-f]{40}$|^[0-9a-f]{32}$/i"
+    context_link: "https://lookupone.local/q=<ATTR_VALUE>"
+    redirect_warning: TRUE
 
-lookupthree:
-  short_name: 'LookupThree'
-  match_fields: ['url']
-  context_link: 'https://lookupthree.local/q=<ATTR_VALUE>'
-  redirect_warning: TRUE
+  lookuptwo:
+    short_name: "LookupTwo"
+    match_fields: ["sha256_hash", "hash"]
+    validation_regex: "/^[0-9a-f]{64}$/i"
+    context_link: "https://lookuptwo.local/q=<ATTR_VALUE>"
+    redirect_warning: FALSE
+
+  lookupthree:
+    short_name: "LookupThree"
+    match_fields: ["url"]
+    context_link: "https://lookupthree.local/q=<ATTR_VALUE>"
+    redirect_warning: TRUE

timesketch/api/v1/resources/contextlinks.py

@@ -47,11 +47,36 @@ def get(self):
         if not context_link_yaml:
             return jsonify(response)
 
-        for entry in context_link_yaml:
-            entry_dict = context_link_yaml[entry]
-            context_link_config = deepcopy(entry_dict)
-            del context_link_config["match_fields"]
-            for field in entry_dict.get("match_fields"):
-                response.setdefault(field.lower(), []).append(context_link_config)
+        if context_link_yaml.get("hardcoded_modules"):
+            for entry in context_link_yaml.get("hardcoded_modules", []):
+                context_link_config = {
+                    "type": "hardcoded_modules",
+                    "short_name": context_link_yaml["hardcoded_modules"][entry][
+                        "short_name"
+                    ],
+                    "module": entry,
+                }
+                if context_link_yaml["hardcoded_modules"][entry].get(
+                    "validation_regex"
+                ):
+                    context_link_config["validation_regex"] = context_link_yaml[
+                        "hardcoded_modules"
+                    ][entry]["validation_regex"]
+                for field in context_link_yaml["hardcoded_modules"][entry][
+                    "match_fields"
+                ]:
+                    response.setdefault(field.lower(), []).append(context_link_config)
+
+        if context_link_yaml.get("linked_services"):
+            for entry in context_link_yaml.get("linked_services", []):
+                context_link_config = deepcopy(
+                    context_link_yaml["linked_services"][entry]
+                )
+                context_link_config["type"] = "linked_services"
+                del context_link_config["match_fields"]
+                for field in context_link_yaml["linked_services"][entry][
+                    "match_fields"
+                ]:
+                    response.setdefault(field.lower(), []).append(context_link_config)
 
         return jsonify(response)

timesketch/api/v1/resources/unfurl.py

@@ -0,0 +1,65 @@
+# Copyright 2023 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unfurl API for version 1 of the Timesketch API."""
+
+import logging
+import unfurl
+
+from flask import jsonify
+from flask import request
+from flask import abort
+from flask_restful import Resource
+from flask_login import login_required
+
+from timesketch.lib.definitions import HTTP_STATUS_CODE_BAD_REQUEST
+from timesketch.lib.definitions import HTTP_STATUS_CODE_INTERNAL_SERVER_ERROR
+
+logger = logging.getLogger("timesketch.api_unfurl")
+
+
+class UnfurlResource(Resource):
+    """Resource to get unfurl information."""
+
+    @login_required
+    def post(self):
+        """Handles POST request to the resource.
+
+        Returns:
+            JSON object including version info
+        """
+        form = request.json
+        if not form:
+            abort(
+                HTTP_STATUS_CODE_BAD_REQUEST,
+                "No JSON data provided",
+            )
+
+        if "url" not in form:
+            abort(
+                HTTP_STATUS_CODE_BAD_REQUEST,
+                "url parameter is required",
+            )
+
+        url = form.get("url")
+
+        try:
+            unfurl_result = unfurl.run(url)
+        except Exception as e:  # pylint: disable=broad-except
+            logger.error("Error unfurling URL: {}".format(e))
+            abort(
+                HTTP_STATUS_CODE_INTERNAL_SERVER_ERROR,
+                e,
+            )
+
+        return jsonify(unfurl_result)

timesketch/api/v1/resources_test.py

@@ -1003,46 +1003,88 @@ def test_get_intelligence_tag_metadata(self):
         data = json.loads(response.get_data(as_text=True))
         self.assertIsNotNone(response)
         self.assertEqual(response.status_code, HTTP_STATUS_CODE_OK)
-        self.assertEqual(data, expected_tag_metadata)
+        self.assertDictEqual(data, expected_tag_metadata)
 
 
 class ContextLinksResourceTest(BaseTest):
     """Test Context Links resources."""
 
+    maxDiff = None
+
     def test_get_context_links_config(self):
         """Authenticated request to get the context links configuration."""
 
         expected_configuration = {
             "hash": [
                 {
-                    "short_name": "LookupOne",
-                    "validation_regex": "/^[0-9a-f]{40}$|^[0-9a-f]{32}$/i",
                     "context_link": "https://lookupone.local/q=<ATTR_VALUE>",
                     "redirect_warning": True,
+                    "short_name": "LookupOne",
+                    "type": "linked_services",
+                    "validation_regex": "/^[0-9a-f]{40}$|^[0-9a-f]{32}$/i",
                 },
                 {
-                    "short_name": "LookupTwo",
-                    "validation_regex": "/^[0-9a-f]{64}$/i",
                     "context_link": "https://lookuptwo.local/q=<ATTR_VALUE>",
                     "redirect_warning": False,
+                    "short_name": "LookupTwo",
+                    "type": "linked_services",
+                    "validation_regex": "/^[0-9a-f]{64}$/i",
                 },
             ],
+            "original_url": [
+                {
+                    "module": "module_two",
+                    "short_name": "ModuleTwo",
+                    "type": "hardcoded_modules",
+                }
+            ],
             "sha256_hash": [
                 {
-                    "short_name": "LookupTwo",
-                    "validation_regex": "/^[0-9a-f]{64}$/i",
                     "context_link": "https://lookuptwo.local/q=<ATTR_VALUE>",
                     "redirect_warning": False,
-                },
+                    "short_name": "LookupTwo",
+                    "type": "linked_services",
+                    "validation_regex": "/^[0-9a-f]{64}$/i",
+                }
+            ],
+            "uri": [
+                {
+                    "module": "module_two",
+                    "short_name": "ModuleTwo",
+                    "type": "hardcoded_modules",
+                }
             ],
             "url": [
                 {
-                    "short_name": "LookupThree",
+                    "module": "module_two",
+                    "short_name": "ModuleTwo",
+                    "type": "hardcoded_modules",
+                },
+                {
                     "context_link": "https://lookupthree.local/q=<ATTR_VALUE>",
                     "redirect_warning": True,
+                    "short_name": "LookupThree",
+                    "type": "linked_services",
                 },
             ],
+            "xml": [
+                {
+                    "module": "module_one",
+                    "short_name": "ModuleOne",
+                    "type": "hardcoded_modules",
+                    "validation_regex": "/^[0-9a-f]{64}$/i",
+                }
+            ],
+            "xml_string": [
+                {
+                    "module": "module_one",
+                    "short_name": "ModuleOne",
+                    "type": "hardcoded_modules",
+                    "validation_regex": "/^[0-9a-f]{64}$/i",
+                }
+            ],
         }
+
         self.login()
         response = self.client.get("/api/v1/contextlinks/")
         data = json.loads(response.get_data(as_text=True))

timesketch/api/v1/routes.py

@@ -72,6 +72,7 @@
 from .resources.graph import GraphCacheResource
 from .resources.intelligence import TagMetadataResource
 from .resources.contextlinks import ContextLinkConfigResource
+from .resources.unfurl import UnfurlResource
 
 from .resources.scenarios import ScenarioTemplateListResource
 from .resources.scenarios import ScenarioListResource
@@ -181,6 +182,7 @@
     (DataFinderResource, "/sketches/<int:sketch_id>/data/find/"),
     (TagMetadataResource, "/intelligence/tagmetadata/"),
     (ContextLinkConfigResource, "/contextlinks/"),
+    (UnfurlResource, "/unfurl/"),
     # Scenarios
     (ScenarioTemplateListResource, "/scenarios/"),
     (ScenarioListResource, "/sketches/<int:sketch_id>/scenarios/"),