Merge branch 'master' into 2796

data/context_links.yaml

@@ -3,8 +3,32 @@
 # ------------------------------------------------------------------------
 #
 # This is a config file to define context links for event attributes.
+# Documentation: https://timesketch.org/guides/admin/context-links/
 #
-# Each context link consists of the following fields:
+# There are two types of context links:
+#
+# 1. Hardcoded modules: These are modules that are hardcoded into Timesketch.
+# The config is used to define the match fields for the module.
+#
+#   module_name:
+#
+#       match_fields:       Type: list[str] | List of field keys where
+#                           this context link should be available. Will
+#                           be checked as case insensitive!
+#
+#       validation_regex:   Type: str | OPTIONAL
+#                           A regex pattern that needs to be
+#                           matched by the field value to to make the
+#                           context link available. This can be used to
+#                           validate the format of a value (e.g. a hash).
+#
+# Currnetly supported modules are:
+#   - XML formatter: Displays a formatted XML in a pop-up dialog.
+#   - Unfurl graph: Displays a graph of an URL using unfurl results.
+#
+# 2. External services: These are context links that are defined by each admin.
+# Those links use external services to provide additional information about the
+# attribute value.
 #
 #   context_link_name:
 #
@@ -37,10 +61,28 @@
 #                                    pages.)
 #
 # ------------------------------------------------------------------------
-## Virustotal Example:
-# virustotal_hash_lookup:
-#   short_name: 'VirusTotal'
-#   match_fields: ['hash', 'sha256_hash', 'sha256', 'sha1_hash', 'sha1', 'md5_hash', 'md5']
-#   validation_regex: '/^[0-9a-f]{64}$|^[0-9a-f]{40}$|^[0-9a-f]{32}$/i'
-#   context_link: 'https://www.virustotal.com/gui/search/<ATTR_VALUE>'
-#   redirect_warning: TRUE
+## Hardcoded Modules
+hardcoded_modules:
+### format xml dialog
+  xml_formatter:
+    short_name: 'Prettify XML'
+    match_fields:
+      - xml
+      - xml_string
+### unfurl dialog
+  unfurl_graph:
+    short_name: 'Unfurl URL'
+    match_fields:
+      - url
+      - uri
+      - original_url
+
+## External Services
+linked_services:
+### Virustotal Example:
+  # virustotal_hash_lookup:
+  #   short_name: 'VirusTotal'
+  #   match_fields: ['hash', 'sha256_hash', 'sha256', 'sha1_hash', 'sha1', 'md5_hash', 'md5', 'url']
+  #   validation_regex: '/^[0-9a-f]{64}$|^[0-9a-f]{40}$|^[0-9a-f]{32}$/i'
+  #   context_link: 'https://www.virustotal.com/gui/search/<ATTR_VALUE>'
+  #   redirect_warning: TRUE

data/timesketch.conf

@@ -311,6 +311,14 @@ EMAIL_RECIPIENTS = []
 # Configuration to construct URLs for resources.
 EXTERNAL_HOST_URL = 'https://localhost'
 
+# SSL/TLS support for emails
+EMAIL_TLS = False
+EMAIL_SSL = False
+
+# Email support for authentication
+EMAIL_AUTH_USERNAME = ""
+EMAIL_AUTH_PASSWORD = ""
+
 #-------------------------------------------------------------------------------
 # Sigma Settings
 

requirements.txt

@@ -3,6 +3,7 @@ altair==4.1.0
 celery==5.2.7
 cryptography==41.0.4
 datasketch==1.5.0
+dfir-unfurl==20230901
 opensearch-py==2.3.1
 Flask==2.3.2
 flask_bcrypt==1.0.1

test_tools/test_events/mock_context_links.yaml

@@ -1,20 +1,37 @@
-## Mock configuration file for testing the contrext links API endpoint!
-lookupone:
-  short_name: 'LookupOne'
-  match_fields: ['hash']
-  validation_regex: '/^[0-9a-f]{40}$|^[0-9a-f]{32}$/i'
-  context_link: 'https://lookupone.local/q=<ATTR_VALUE>'
-  redirect_warning: TRUE
+# Mock configuration file for testing the context links API endpoint!
+## Hardcoded Modules
+hardcoded_modules:
+  module_one:
+    short_name: "ModuleOne"
+    validation_regex: "/^[0-9a-f]{64}$/i"
+    match_fields:
+      - xml
+      - xml_string
+  module_two:
+    short_name: "ModuleTwo"
+    match_fields:
+      - url
+      - uri
+      - original_url
 
-lookuptwo:
-  short_name: 'LookupTwo'
-  match_fields: ['sha256_hash', 'hash']
-  validation_regex: '/^[0-9a-f]{64}$/i'
-  context_link: 'https://lookuptwo.local/q=<ATTR_VALUE>'
-  redirect_warning: FALSE
+## External Services
+linked_services:
+  lookupone:
+    short_name: "LookupOne"
+    match_fields: ["hash"]
+    validation_regex: "/^[0-9a-f]{40}$|^[0-9a-f]{32}$/i"
+    context_link: "https://lookupone.local/q=<ATTR_VALUE>"
+    redirect_warning: TRUE
 
-lookupthree:
-  short_name: 'LookupThree'
-  match_fields: ['url']
-  context_link: 'https://lookupthree.local/q=<ATTR_VALUE>'
-  redirect_warning: TRUE
+  lookuptwo:
+    short_name: "LookupTwo"
+    match_fields: ["sha256_hash", "hash"]
+    validation_regex: "/^[0-9a-f]{64}$/i"
+    context_link: "https://lookuptwo.local/q=<ATTR_VALUE>"
+    redirect_warning: FALSE
+
+  lookupthree:
+    short_name: "LookupThree"
+    match_fields: ["url"]
+    context_link: "https://lookupthree.local/q=<ATTR_VALUE>"
+    redirect_warning: TRUE

test_tools/test_events/validate_timestamp_conversion.csv

@@ -0,0 +1,4 @@
+"message","timestamp","datetime","timestamp_desc","data_type"
+"Checking timestamp conversion","1331698658000000","2012-03-14T04:17:38+00:00","Time Logged","This event has timestamp"
+"Checking timestamp conversion","1658689261000000","2022-07-24T19:01:01+0000","Time Logged","This event has timestamp"
+"Make sure message is same","1437789661000000","2015-07-25 02:01:01+00:00","Logging","This data_type should stay the same"

timesketch/api/v1/resources/__init__.py

@@ -275,6 +275,7 @@ class ResourceMixin(object):
         "name": fields.String,
         "display_name": fields.String,
         "description": fields.String,
+        "dfiq_identifier": fields.String,
         "spec_json": fields.String,
         "user": fields.Nested(user_fields),
         "approaches": fields.List(fields.Nested(approach_fields)),
@@ -288,6 +289,7 @@ class ResourceMixin(object):
         "name": fields.String,
         "display_name": fields.String,
         "description": fields.String,
+        "dfiq_identifier": fields.String,
         "spec_json": fields.String,
         "user": fields.Nested(user_fields),
         "questions": fields.List(fields.Nested(question_fields)),

timesketch/api/v1/resources/contextlinks.py

@@ -47,11 +47,42 @@ def get(self):
         if not context_link_yaml:
             return jsonify(response)
 
-        for entry in context_link_yaml:
-            entry_dict = context_link_yaml[entry]
-            context_link_config = deepcopy(entry_dict)
-            del context_link_config["match_fields"]
-            for field in entry_dict.get("match_fields"):
-                response.setdefault(field.lower(), []).append(context_link_config)
+        # Support for earlier version format of context links without hardcoded modules:
+        if not context_link_yaml.get("linked_services"):
+            context_link_yaml = {
+                "linked_services": context_link_yaml,
+            }
+
+        if context_link_yaml.get("hardcoded_modules"):
+            for entry in context_link_yaml.get("hardcoded_modules", []):
+                context_link_config = {
+                    "type": "hardcoded_modules",
+                    "short_name": context_link_yaml["hardcoded_modules"][entry][
+                        "short_name"
+                    ],
+                    "module": entry,
+                }
+                if context_link_yaml["hardcoded_modules"][entry].get(
+                    "validation_regex"
+                ):
+                    context_link_config["validation_regex"] = context_link_yaml[
+                        "hardcoded_modules"
+                    ][entry]["validation_regex"]
+                for field in context_link_yaml["hardcoded_modules"][entry][
+                    "match_fields"
+                ]:
+                    response.setdefault(field.lower(), []).append(context_link_config)
+
+        if context_link_yaml.get("linked_services"):
+            for entry in context_link_yaml.get("linked_services", []):
+                context_link_config = deepcopy(
+                    context_link_yaml["linked_services"][entry]
+                )
+                context_link_config["type"] = "linked_services"
+                del context_link_config["match_fields"]
+                for field in context_link_yaml["linked_services"][entry][
+                    "match_fields"
+                ]:
+                    response.setdefault(field.lower(), []).append(context_link_config)
 
         return jsonify(response)

timesketch/api/v1/resources/explore.py

@@ -44,6 +44,9 @@
 from timesketch.models.sketch import Sketch
 from timesketch.models.sketch import View
 from timesketch.models.sketch import SearchHistory
+from timesketch.models.sketch import Scenario
+from timesketch.models.sketch import Facet
+from timesketch.models.sketch import InvestigativeQuestion
 
 # Metrics definitions
 METRICS = {
@@ -93,6 +96,42 @@ def post(self, sketch_id):
                 "Unable to explore data, unable to validate form data",
             )
 
+        # DFIQ context
+        scenario = None
+        facet = None
+        question = None
+
+        scenario_id = request.json.get("scenario", None)
+        facet_id = request.json.get("facet", None)
+        question_id = request.json.get("question", None)
+
+        if scenario_id:
+            scenario = Scenario.query.get(scenario_id)
+            if scenario:
+                if scenario.sketch_id != sketch.id:
+                    abort(
+                        HTTP_STATUS_CODE_BAD_REQUEST,
+                        "Scenario is not part of this sketch.",
+                    )
+
+        if facet_id:
+            facet = Facet.query.get(facet_id)
+            if facet:
+                if facet.scenario.sketch_id != sketch.id:
+                    abort(
+                        HTTP_STATUS_CODE_BAD_REQUEST,
+                        "Facet is not part of this sketch.",
+                    )
+
+        if question_id:
+            question = InvestigativeQuestion.query.get(question_id)
+            if question:
+                if question.facet.scenario.sketch_id != sketch.id:
+                    abort(
+                        HTTP_STATUS_CODE_BAD_REQUEST,
+                        "Question is not part of this sketch.",
+                    )
+
         # TODO: Remove form and use json instead.
         query_dsl = form.dsl.data
         enable_scroll = form.enable_scroll.data
@@ -340,6 +379,11 @@ def post(self, sketch_id):
             new_search.query_result_count = count_total_complete
             new_search.query_time = result["took"]
 
+            # Add DFIQ context
+            new_search.scenario = scenario
+            new_search.facet = facet
+            new_search.question = question
+
             if previous_search:
                 new_search.parent = previous_search
 

timesketch/api/v1/resources/unfurl.py

@@ -0,0 +1,65 @@
+# Copyright 2023 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unfurl API for version 1 of the Timesketch API."""
+
+import logging
+import unfurl
+
+from flask import jsonify
+from flask import request
+from flask import abort
+from flask_restful import Resource
+from flask_login import login_required
+
+from timesketch.lib.definitions import HTTP_STATUS_CODE_BAD_REQUEST
+from timesketch.lib.definitions import HTTP_STATUS_CODE_INTERNAL_SERVER_ERROR
+
+logger = logging.getLogger("timesketch.api_unfurl")
+
+
+class UnfurlResource(Resource):
+    """Resource to get unfurl information."""
+
+    @login_required
+    def post(self):
+        """Handles POST request to the resource.
+
+        Returns:
+            JSON object including version info
+        """
+        form = request.json
+        if not form:
+            abort(
+                HTTP_STATUS_CODE_BAD_REQUEST,
+                "No JSON data provided",
+            )
+
+        if "url" not in form:
+            abort(
+                HTTP_STATUS_CODE_BAD_REQUEST,
+                "url parameter is required",
+            )
+
+        url = form.get("url")
+
+        try:
+            unfurl_result = unfurl.run(url)
+        except Exception as e:  # pylint: disable=broad-except
+            logger.error("Error unfurling URL: {}".format(e))
+            abort(
+                HTTP_STATUS_CODE_INTERNAL_SERVER_ERROR,
+                e,
+            )
+
+        return jsonify(unfurl_result)