Merge branch 'google:master' into llm-search

.github/workflows/linters.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04, ubuntu-22.04]
-        python-version: ['3.8', '3.10']
+        python-version: ['3.9', '3.10']
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/unit-tests.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04, ubuntu-22.04]
-        python-version: ['3.8', '3.10']
+        python-version: ['3.9', '3.10']
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
@@ -22,10 +22,12 @@ jobs:
         run: |
           pip install pipenv
           pipenv install -d
-          pipenv install -r test_requirements.txt
+          pipenv run pip install -r test_requirements.txt
+      - name: Check pytest installation
+        run: pipenv run pip show pytest
       - name: Run unit tests
         run: |
-          pipenv run python run_tests.py
+          pipenv run python3 run_tests.py
 
   # Frontend tests (VueJS)
   VueJS:

.gitignore

@@ -1,5 +1,7 @@
 # Files to ignore by git
 
+.DS_Store
+
 # Back-up files
 *~
 *.swp
@@ -26,6 +28,16 @@ importer_client/python/build/
 node_modules
 timesketch/static/dist
 
+# Frontend Log files
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+
+# local frontend env files
+.env.local
+.env.*.local
+
 # Test files
 .coverage
 tests-coverage.txt

api_client/python/timesketch_api_client/aggregation.py

@@ -16,6 +16,7 @@
 import getpass
 import json
 import logging
+from typing import Any
 
 import altair
 import pandas
@@ -41,6 +42,8 @@ class Aggregation(resource.SketchResource):
             saved search.
     """
 
+    resource_data: dict[str, Any]
+
     def __init__(self, sketch):
         self._created_at = ""
         self._name = ""
@@ -307,7 +310,7 @@ def description(self, description):
         """Set the description of an aggregation."""
         if "meta" not in self.resource_data:
             return
-        meta = self.resource_data.get("meta")
+        meta = self.resource_data.get("meta", {})
         meta["description"] = description
 
     @property

api_client/python/timesketch_api_client/client.py

@@ -106,11 +106,11 @@ def __init__(
         self._flow = None
 
         if not create_session:
-            self.session = None
+            self._session = None
             return
 
         try:
-            self.session = self._create_session(
+            self._session = self._create_session(
                 username,
                 password,
                 verify=verify,
@@ -145,13 +145,20 @@ def version(self):
 
         return "API Client: {0:s}".format(version.get_version())
 
+    @property
+    def session(self):
+        """Property that returns the session object."""
+        if self._session is None:
+            raise ValueError("Session is not set.")
+        return self._session
+
     def set_credentials(self, credential_object):
         """Sets the credential object."""
         self.credentials = credential_object
 
     def set_session(self, session_object):
         """Sets the session object."""
-        self.session = session_object
+        self._session = session_object
 
     def _authenticate_session(self, session, username, password):
         """Post username/password to authenticate the HTTP session.

api_client/python/timesketch_api_client/graph.py

@@ -231,12 +231,14 @@ def from_graph(self, graph_obj):
         self._created_at = time
         self._updated_at = time
 
-    def from_manual(self, data, **kwargs):  # pylint: disable=arguments-differ
+    def from_manual(
+        self, data, **kwargs
+    ):  # pylint: disable=arguments-differ; pytype: disable=signature-mismatch
         """Generate a new graph using a dictionary.
 
         Args:
             data (dict): A dictionary of dictionaries adjacency representation.
-            kwargs (dict[str, object]): Depending on the resource they may
+            **kwargs (dict[str, object]): Depending on the resource they may
                 require different sets of arguments to be able to run a raw
                 API request.
 

api_client/python/timesketch_api_client/search.py

@@ -493,12 +493,19 @@ def _execute_query(self, file_name="", count=False):
         """Execute a search request and store the results.
 
         Args:
-            file_name (str): optional file path to a filename that
+            file_name (str): Optional file path to a filename that
                 all the results will be saved to. If not provided
                 the results will be stored in the search object.
-            count (bool): optional boolean that determines whether
+            count (bool): Optional boolean that determines whether
                 we want to execute the query or only count the
-                number of events that the query would produce.
+                number of events that the query would produce. If
+                set to True, the results will be stored in the
+                search object, and the number of events will be
+                returned.
+
+        Returns:
+            A dict with the search results or the total number of events
+            (if count=True) or None if saved to file.
         """
         query_filter = self.query_filter
         if not isinstance(query_filter, dict):
@@ -531,14 +538,14 @@ def _execute_query(self, file_name="", count=False):
         if file_name:
             with open(file_name, "wb") as fw:
                 fw.write(response.content)
-            return
+            return None
 
         response_json = error.get_response_json(response, logger)
 
         if count:
             meta = response_json.get("meta", {})
             self._total_elastic_size = meta.get("total_count", 0)
-            return
+            return meta.get("total_count", 0)
 
         scroll_id = response_json.get("meta", {}).get("scroll_id", "")
         form_data["scroll_id"] = scroll_id
@@ -579,6 +586,7 @@ def _execute_query(self, file_name="", count=False):
             )
 
         self._raw_response = response_json
+        return response_json
 
     def add_chip(self, chip):
         """Add a chip to the ..."""
@@ -647,7 +655,7 @@ def expected_size(self):
         if self._total_elastic_size:
             return self._total_elastic_size
 
-        self._execute_query(count=True)
+        _ = self._execute_query(count=True)
         return self._total_elastic_size
 
     def from_manual(  # pylint: disable=arguments-differ
@@ -1074,8 +1082,10 @@ def scrolling_enable(self):
 
     def to_dict(self):
         """Returns a dict with the respone of the query."""
-        if not self._raw_response:
+        if self._raw_response is None:
             self._execute_query()
+            if self._raw_response is None:
+                raise ValueError("No results to return.")
 
         return self._raw_response
 
@@ -1098,8 +1108,10 @@ def to_file(self, file_name):
 
     def to_pandas(self):
         """Returns a pandas DataFrame with the response of the query."""
-        if not self._raw_response:
-            self._execute_query()
+        if self._raw_response is None:
+            self._raw_response = self._execute_query()
+            if self._raw_response is None:
+                raise ValueError("No results to return.")
 
         return_list = []
         timelines = {t.id: t.name for t in self._sketch.list_timelines()}

api_client/python/timesketch_api_client/story.py

@@ -729,6 +729,9 @@ def to_string(self):
                 string_list.append(block.text)
             elif block.TYPE == "view":
                 search_obj = block.view
+                if search_obj is None:
+                    logging.warning("Block has no view. Skipping")
+                    continue
                 data_frame = search_obj.to_pandas()
                 string_list.append(data_frame.to_string(index=False))
             elif block.TYPE == "aggregation":

api_client/python/timesketch_api_client/version.py

@@ -14,7 +14,7 @@
 """Version information for Timesketch API Client."""
 
 
-__version__ = "20240828"
+__version__ = "20241129"
 
 
 def get_version():

data/regex_features.yaml

@@ -125,7 +125,7 @@ ssh_client_password_ipv4_addresses:
         query_string: 'reporter:"sshd"'
         attribute: 'message'
         store_as: 'client_ip'
-        re: '(?:Accepted|Failed) (?:password|publickey) for \w+ from ((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+'
+        re: '(?:Accepted|Failed) (?:password|publickey) for [A-Za-z0-9._-]+? from ((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+'
 
 ssh_disconnected_username:
         query_string: 'reporter:"sshd"'

data/sigma_config.yaml

@@ -78,6 +78,11 @@ logsources:
     product: windows
     conditions:
       data_type: "windows:evtx:record"
+  service_windows_certificate_services:
+    service: certificateservicesclient-lifecycle-system
+    conditions:
+      source_name:
+        - "Microsoft-Windows-CertificateServicesClient-Lifecycle-System"
   service_windows_security:
     service: security
     conditions:

data/tags.yaml

@@ -37,3 +37,83 @@ yara_match_tagger:
   modifiers: ['split']
   save_search: true
   search_name: 'Yara rule matches'
+
+aws_cloudtrail_readonly_true:
+  query_string: 'data_type:"aws:cloudtrail:entry" AND cloud_trail_event:"*readOnly\":true*"'
+  tags: ['readOnly_true']
+  emojis: ['MAGNIFYING_GLASS']
+  save_search: true
+  search_name: 'readOnly_true'
+
+aws_cloudtrail_readonly_false:
+  query_string: 'data_type:"aws:cloudtrail:entry" AND cloud_trail_event:"*readOnly\":false*"'
+  tags: ['readOnly_false']
+  emojis: ['SPARKLES']
+  save_search: true
+  search_name: 'readOnly_false'
+
+aws_cloudtrail_unauthorized_api_call:
+  query_string: 'data_type:"aws:cloudtrail:entry" AND cloud_trail_event: ("*errorCode\":\"AccessDenied*" OR "*errorCode\":\"UnauthorizedOperation*")'
+  tags: ['UnauthorizedAPICall']
+  save_search: true
+  search_name: 'UnauthorizedAPICall'
+
+aws_cloudtrail_failed_login_non_existent_iam_user:
+  query_string: 'data_type:"aws:cloudtrail:entry" AND cloud_trail_event:"*userIdentity\":\"HIDDEN_DUE_TO_SECURITY_REASONS*" AND cloud_trail_event:"*errorMessage\":\"No username found in supplied account*"'
+  tags: ['FailedLoginNonExistentIAMUser']
+  save_search: true
+  search_name: 'FailedLoginNonExistentIAMUser'
+
+aws_cloudtrail_security_group:
+  query_string: 'data_type:"aws:cloudtrail:entry" AND event_name: ("AuthorizeSecurityGroupEgress" OR "AuthorizeSecurityGroupIngress" OR "CreateSecurityGroup" OR "DeleteSecurityGroup" OR "ModifySecurityGroupRules" OR "RevokeSecurityGroupEgress" OR "RevokeSecurityGroupIngress")'
+  tags: ['NetworkChanged', 'SG']
+  save_search: true
+  search_name: 'NetworkChanged SecurityGroup'
+
+aws_cloudtrail_network_acl:
+  query_string: 'data_type:"aws:cloudtrail:entry" AND event_name: ("CreateNetworkAcl" OR "CreateNetworkAclEntry" OR "DeleteNetworkAcl" OR "DeleteNetworkAclEntry" OR "ReplaceNetworkAclAssociation" OR "ReplaceNetworkAclEntry")'
+  tags: ['NetworkChanged', 'NACL']
+  save_search: true
+  search_name: 'NetworkChanged NetworkACl'
+
+aws_cloudtrail_gateway:
+  query_string: 'data_type:"aws:cloudtrail:entry" AND event_name: (Accept* OR Associate* OR Attach* OR Create* OR Delete* OR Replace*) AND event_name:*Gateway'
+  tags: ['NetworkChanged', 'GW']
+  save_search: true
+  search_name: 'NetworkChanged GateWay'
+
+aws_cloudtrail_routetable:
+  query_string: 'data_type:"aws:cloudtrail:entry" AND event_name: ("CreateRoute" OR "CreateRouteTable" OR "DeleteRoute" OR "DeleteRouteTable" OR "DisassociateRouteTable" OR "ReplaceRoute" OR "ReplaceRouteTableAssociation")'
+  tags: ['NetworkChanged', 'RouteTable']
+  save_search: true
+  search_name: 'NetworkChanged RouteTable'
+
+aws_cloudtrail_vpc:
+  query_string: 'data_type:"aws:cloudtrail:entry" AND event_name: ("AcceptVpcPeeringConnection" OR "AttachClassicLinkVpc" OR "CreateVpc" OR "CreateVpcPeeringConnection" OR "DeleteVpc" OR "DeleteVpcPeeringConnection" OR "DetachClassicLinkVpc" OR "DisableVpcClassicLink" OR "EnableVpcClassicLink" OR "ModifyVpcAttribute" OR "RejectVpcPeeringConnection")'
+  tags: ['NetworkChanged', 'VPC']
+  save_search: true
+  search_name: 'NetworkChanged VPC'
+
+aws_cloudtrail_suspicous_iam_activity:
+  query_string: 'data_type:"aws:cloudtrail:entry" AND event_name: ("AddRoleToInstanceProfile" OR "AddUserToGroup" OR "AssumeRole" OR "AttachGroupPolicy" OR "AttachRolePolicy" OR "AttachUserPolicy" OR "CreateAccessKey" OR "CreateLoginProfile" OR "CreatePolicyVersion" OR "CreateRole" OR "PassRole" OR "PutGroupPolicy" OR "PutRolePolicy" OR "PutUserPolicy" OR "SetDefaultPolicyVersion" OR "UpdateAccessKey" OR "UpdateLoginProfile" OR "GetFederationToken" )'
+  tags: ['SuspicousIAMActivity']
+  save_search: true
+  search_name: 'SuspicousIAMActivity'
+
+aws_cloudtrail_suspicous_iam_identity_center_activity:
+  query_string: 'data_type:"aws:cloudtrail:entry" AND event_name: ("StartSSO" OR "CreateUser" OR "CreateGroup" OR "AddMemberToGroup" OR "CreatePermissionSet" OR "CreateAccountAssignment" OR "Authenticate" OR "Federate" OR "AssumeRoleWithSAML")'
+  tags: ['SuspicousIICActivity']
+  save_search: true
+  search_name: 'SuspicousIICActivity'
+
+aws_cloudtrail_console_login:
+  query_string: 'data_type:"aws:cloudtrail:entry" AND event_name:"ConsoleLogin"'
+  tags: ['ConsoleLogin']
+  save_search: true
+  search_name: 'ConsoleLogin'
+
+aws_cloudtrail_get_caller_identity:
+  query_string: 'data_type:"aws:cloudtrail:entry" AND event_name:"GetCallerIdentity"'
+  tags: ['GetCallerIdentity']
+  save_search: true
+  search_name: 'GetCallerIdentity'

docker/dev/build/docker-entrypoint.sh

@@ -23,6 +23,7 @@ if [ "$1" = 'timesketch' ]; then
   ln -s /usr/local/src/timesketch/data/dfiq /etc/timesketch/
   ln -s /usr/local/src/timesketch/data/context_links.yaml /etc/timesketch/context_links.yaml
   ln -s /usr/local/src/timesketch/data/plaso_formatters.yaml /etc/timesketch/plaso_formatters.yaml
+  ln -s /usr/local/src/timesketch/data/nl2q /etc/timesketch/
 
   # Set SECRET_KEY in /etc/timesketch/timesketch.conf if it isn't already set
   if grep -q "SECRET_KEY = '<KEY_GOES_HERE>'" /etc/timesketch/timesketch.conf; then

package.json

@@ -0,0 +1,5 @@
+{
+  "dependencies": {
+    "axios": "^1.7.7"
+  }
+}

requirements.txt

@@ -1,9 +1,9 @@
 alembic==1.11.1
 altair==4.1.0
 celery==5.2.7
-cryptography==43.0.0
+cryptography==43.0.1
 datasketch==1.5.0
-dfir-unfurl==20230901
+dfir-unfurl==20240627
 opensearch-py==2.6.0
 Flask==3.0.3
 flask_bcrypt==1.0.1
@@ -14,6 +14,7 @@ flask_sqlalchemy==3.0.3
 flask_wtf==1.2.1
 google-auth==2.32.0
 google_auth_oauthlib==0.4.1
+google-cloud-aiplatform==1.70.0
 gunicorn==22.0.0
 numpy==1.23.4
 oauthlib==3.1.0

run_tests.py

@@ -4,10 +4,7 @@
 
 
 def run_python_tests():
-    subprocess.check_call(
-        "python3 -m pytest timesketch/ api_client/",
-        shell=True,
-    )
+    subprocess.check_call(["python3", "-m", "pytest", "timesketch/", "api_client/"])
 
 
 def main():

timesketch/frontend-ng/dist/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang=en><head><meta name=csrf-token content="{{ csrf_token() }}"><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/dist/favicon.ico><link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900" rel=stylesheet><title>Timesketch</title><link href=/dist/css/chunk-vendors.f265108c.css rel=preload as=style><link href=/dist/css/index.008f2248.css rel=preload as=style><link href=/dist/js/chunk-vendors.eba39bd6.js rel=preload as=script><link href=/dist/js/index.fbd54591.js rel=preload as=script><link href=/dist/css/chunk-vendors.f265108c.css rel=stylesheet><link href=/dist/css/index.008f2248.css rel=stylesheet></head><body><div id=app></div><script src=/dist/js/chunk-vendors.eba39bd6.js></script><script src=/dist/js/index.fbd54591.js></script></body></html>
+<!DOCTYPE html><html lang=en><head><meta name=csrf-token content="{{ csrf_token() }}"><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/dist/favicon.ico><link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900" rel=stylesheet><title>Timesketch</title><link href=/dist/css/chunk-vendors.f265108c.css rel=preload as=style><link href=/dist/css/index.008f2248.css rel=preload as=style><link href=/dist/js/chunk-vendors.eba39bd6.js rel=preload as=script><link href=/dist/js/index.1ff10a58.js rel=preload as=script><link href=/dist/css/chunk-vendors.f265108c.css rel=stylesheet><link href=/dist/css/index.008f2248.css rel=stylesheet></head><body><div id=app></div><script src=/dist/js/chunk-vendors.eba39bd6.js></script><script src=/dist/js/index.1ff10a58.js></script></body></html>