From 84865e3ab6e7a4d237c9720094d3391cc501de54 Mon Sep 17 00:00:00 2001 From: Alexander J <741037+jaegeral@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:35:45 +0200 Subject: [PATCH] Add intelligence command to the CLI client (#2864) * adding a intelligence sub command to the Timesketch CLI + documentation * W0622: Redefining built-in 'type' (redefined-builtin * Apply suggestions from code review Co-authored-by: Johan Berggren * code review feedback * Update cli_client/python/timesketch_cli_client/commands/intelligence.py * Update cli_client/python/timesketch_cli_client/commands/intelligence.py * Update cli_client/python/timesketch_cli_client/commands/intelligence.py * s/type/ioc-type --------- Co-authored-by: Johan Berggren --- .../python/timesketch_cli_client/cli.py | 2 + .../commands/intelligence.py | 145 ++++++++++++++++++ .../commands/intelligence_test.py | 46 ++++++ docs/guides/user/cli-client.md | 53 ++++++- 4 files changed, 241 insertions(+), 5 deletions(-) create mode 100644 cli_client/python/timesketch_cli_client/commands/intelligence.py create mode 100644 cli_client/python/timesketch_cli_client/commands/intelligence_test.py diff --git a/cli_client/python/timesketch_cli_client/cli.py b/cli_client/python/timesketch_cli_client/cli.py index 1af79d4bf8..1c31de4c72 100644 --- a/cli_client/python/timesketch_cli_client/cli.py +++ b/cli_client/python/timesketch_cli_client/cli.py @@ -26,6 +26,7 @@ from timesketch_cli_client.commands import analyze from timesketch_cli_client.commands import config from timesketch_cli_client.commands import importer +from timesketch_cli_client.commands import intelligence from timesketch_cli_client.commands import search from timesketch_cli_client.commands import sketch as sketch_command from timesketch_cli_client.commands import timelines @@ -169,6 +170,7 @@ def cli(ctx, sketch, output): cli.add_command(importer.importer) cli.add_command(events.events_group) cli.add_command(sigma.sigma_group) +cli.add_command(intelligence.intelligence_group) # pylint: disable=no-value-for-parameter diff --git a/cli_client/python/timesketch_cli_client/commands/intelligence.py b/cli_client/python/timesketch_cli_client/commands/intelligence.py new file mode 100644 index 0000000000..7ed807f07e --- /dev/null +++ b/cli_client/python/timesketch_cli_client/commands/intelligence.py @@ -0,0 +1,145 @@ +# Copyright 2023 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Commands for interacting with intelligence within a Sketch.""" + +import sys +import json +import click + + +@click.group("intelligence") +def intelligence_group(): + """Manage intelligence within a sketch.""" + + +@intelligence_group.command("list") +@click.option( + "--header/--no-header", + default=True, + help="Include header in output. (default is to show header))", +) +@click.option( + "--columns", + default="ioc,type", + help="Comma separated list of columns to show. (default: ioc,type)", +) +@click.pass_context +def list_intelligence(ctx, header, columns): + """List all intelligence. + + Args: + ctx: Click context object. + header: Include header in output. (default is to show header) + columns: Comma separated list of columns to show. (default: ioc,type) + Other options: externalURI, tags + """ + + if not columns: + columns = "ioc,type" + + columns = columns.split(",") + + output = ctx.obj.output_format + sketch = ctx.obj.sketch + try: + intelligence = sketch.get_intelligence_attribute() + except ValueError as e: + click.echo(e) + sys.exit(1) + + if not intelligence: + click.echo("No intelligence found.") + ctx.exit(1) + if output == "json": + click.echo(json.dumps(intelligence, indent=4, sort_keys=True)) + elif output == "text": + if header: + click.echo("\t".join(columns)) + for entry in intelligence: + row = [] + for column in columns: + if column == "tags": + row.append(",".join(entry.get(column, []))) + else: + row.append(entry.get(column, "")) + click.echo("\t".join(row)) + elif output == "csv": + if header: + click.echo(",".join(columns)) + for entry in intelligence: + row = [] + for column in columns: + if column == "tags": + # Tags can be multiple values but they should only be + # one value on the csv so we join them with a comma + # surrounded the array by quotes + row.append(f'"{",".join(entry.get(column, []))}"') + else: + row.append(entry.get(column, "")) + click.echo(",".join(row)) + else: + click.echo(f"Output format {output} not implemented.") + + +@intelligence_group.command("add") +@click.option( + "--ioc", + required=True, + help="Indicator Of Compromise (IOC) value.", +) +@click.option( + "--ioc-type", + required=False, + help="Type of the intelligence (ipv4, hash_sha256, hash_sha1, hash_md5, other).", +) +@click.option( + "--tags", + required=False, + help="Comma separated list of tags.", +) +@click.pass_context +def add_intelligence(ctx, ioc, tags, ioc_type="other"): + """Add intelligence to a sketch. + + A sketch can have multiple intelligence entries. Each entry consists of + an indicator, a type and a list of tags. + + Reference: https://timesketch.org/guides/user/intelligence/ + + Args: + ctx: Click context object. + ioc: IOC value. + ioc_type: Type of the intelligence. This is defined in the ontology file. + If a string doesn't match any of the aforementioned IOC types, + the type will fall back to other. + tags: Comma separated list of tags. + """ + sketch = ctx.obj.sketch + + # Create a tags dict from the comma separated list + if tags: + tags = tags.split(",") + tags = {tag: [] for tag in tags} + else: + tags = [] + + ioc_dict = {"ioc": ioc, "type": ioc_type, "tags": tags} + # Put the ioc in a nested object to match the format of the API + data = {"data": [ioc_dict]} + try: + sketch.add_attribute(name="intelligence", ontology="intelligence", value=data) + except ValueError as e: + click.echo(e) + sys.exit(1) + click.echo(f"Intelligence added: {ioc}") diff --git a/cli_client/python/timesketch_cli_client/commands/intelligence_test.py b/cli_client/python/timesketch_cli_client/commands/intelligence_test.py new file mode 100644 index 0000000000..315e43e67f --- /dev/null +++ b/cli_client/python/timesketch_cli_client/commands/intelligence_test.py @@ -0,0 +1,46 @@ +# Copyright 2023 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for intelligence command.""" + +import unittest +import mock + +from click.testing import CliRunner + +# pylint: disable=import-error +from timesketch_api_client import test_lib as api_test_lib + +# pylint: enable=import-error + +from .. import test_lib +from .intelligence import intelligence_group + + +class IntelligenceTest(unittest.TestCase): + """Test Sigma CLI command.""" + + @mock.patch("requests.Session", api_test_lib.mock_session) + def setUp(self): + """Setup test case.""" + self.ctx = test_lib.get_cli_context() + + def test_list_intelligence(self): + """Test to list Sigma rules.""" + runner = CliRunner() + result = runner.invoke( + intelligence_group, + ["list"], + obj=self.ctx, + ) + assert 1 is result.exit_code diff --git a/docs/guides/user/cli-client.md b/docs/guides/user/cli-client.md index ca2c264960..088b503d6e 100644 --- a/docs/guides/user/cli-client.md +++ b/docs/guides/user/cli-client.md @@ -214,7 +214,7 @@ Name: ticket_id3: Ontology: 12345 Value: text Or as JSON -``` +```bash timesketch --output-format json sketch attributes { "intelligence": { @@ -283,6 +283,50 @@ To remove an attribute from a sketch timesketch sketch remove_attribute ``` +## Intelligence + +Intelligence is always sketch specific. The same can be achieved using +`timesketch attributes` command, but then the ontology type and data needs +to be provided in the correct format. + +Running `timesketch intelligence list` will show the intelligence added to a +sketch (if sketch id is set in the config file) + +The putput format can also be changed as follows + +```bash +timesketch --sketch 2 --output-format text intelligence list --columns ioc,externalURI,tags,type +ioc externalURI tags type +1.2.3.4 google.com foo ipv4 +3.3.3.3 fobar.com aaaa ipv4 +``` + +Or as CSV + +```bash +timesketch --sketch 2 --output-format csv intelligence list --columns ioc,externalURI,tags,type +ioc,externalURI,tags,type +1.2.3.4,google.com,"foo",ipv4 +3.3.3.3,fobar.com,"aaaa",ipv4 +aaaa.com,foobar.de,"foo,aaaa",hostname +``` + +### Adding Intelligence + +Adding an indicator works as following + +```bash +timesketch --sketch 2 intelligence add --ioc 8.8.4.4 --type ipv4 --tags foo,bar,ext +``` + +### Removing all of Intelligence + +To remove all intelligence indicators, run: + +```bash +timesketch --sketch 2 --output-format text sketch attributes remove --name intelligence --ontology intelligence +Attribute removed: Name: intelligence Ontology: intelligence +``` ## Run analyzers @@ -343,7 +387,6 @@ Running analyzer [domain] on [timeline 1]: .. Results [domain] = 217 domains discovered (150 TLDs) and 1 known CDN networks found. - ``` ### List analyzer results @@ -353,7 +396,7 @@ That can be done with `timesketch analyzer results`. It can show only the analyzer results directly: -``` +```bash timesketch --output-format text analyze results --analyzer account_finder --timeline 3 Results for analyzer [account_finder] on [sigma_events]: SUCCESS - NOTE - Account finder was unable to extract any accounts. @@ -396,7 +439,6 @@ Dependent: DONE - None - Feature extraction [ssh_failed_username] extracted 0 fe Dependent: DONE - None - Feature extraction [win_login_subject_domain] extracted 0 features. Dependent: DONE - None - Feature extraction [win_login_subject_logon_id] extracted 0 features. Dependent: DONE - None - Feature extraction [win_login_username] extracted 0 features. - ``` To get a result in `json` that can be piped into other CLI tools run something @@ -440,7 +482,8 @@ This new feature makes it easy to add events to Timesketch from the command line It can also be called with a output format `json` like following. -```timesketch --output-format json events add --message "foobar-message" --date 2023-03-04T11:31:12 --timestamp-desc "test" +```bash +timesketch --output-format json events add --message "foobar-message" --date 2023-03-04T11:31:12 --timestamp-desc "test" {'meta': {}, 'objects': [{'color': 'F37991', 'created_at': '2023-03-08T12:46:24.472587', 'datasources': [], 'deleted': None, 'description': 'internal timeline for user-created events', 'id': 19, 'label_string': '', 'name': 'Manual events', 'searchindex': {'created_at': '2023-03-08T12:46:24.047640', 'deleted': None, 'description': 'internal timeline for user-created events', 'id': 9, 'index_name': '49a318b0ba17867fd71b50903774a0c8', 'label_string': '', 'name': 'Manual events', 'status': [{'created_at': '2023-03-17T09:35:03.202520', 'id': 87, 'status': 'ready', 'updated_at': '2023-03-17T09:35:03.202520'}], 'updated_at': '2023-03-08T12:46:24.047640', 'user': {'active': True, 'admin': True, 'groups': [], 'username': 'dev'}}, 'status': [{'created_at': '2023-03-17T09:35:03.233973', 'id': 79, 'status': 'ready', 'updated_at': '2023-03-17T09:35:03.233973'}], 'updated_at': '2023-03-08T12:46:24.472587', 'user': {'active': True, 'admin': True, 'groups': [], 'username': 'dev'}}]} Event added to sketch: timefocus test ```