From 6ed297a9c66f63a6c0bb5bfff523a1ac153e40d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 13:37:06 -0300 Subject: [PATCH 01/13] Bump aiomysql from 0.1.1 to 0.2.0 in /stix_shifter (#1586) --- stix_shifter/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter/requirements.txt b/stix_shifter/requirements.txt index 4baa59df2..9df48d0e7 100644 --- a/stix_shifter/requirements.txt +++ b/stix_shifter/requirements.txt @@ -1,6 +1,6 @@ aioboto3==11.3.0 aiohttp-retry==2.8.3 -aiomysql==0.1.1 +aiomysql==0.2.0 antlr4-python3-runtime==4.8 asyncio==3.4.3 asynctest==0.13.0 From 7c03e98931c0bec3f0dbe71d9908ac0f26e7b2e3 Mon Sep 17 00:00:00 2001 From: Danny Elliott Date: Tue, 3 Oct 2023 13:40:18 -0300 Subject: [PATCH 02/13] set MS Graph default API to legacy alert endpoint (#1593) --- stix_shifter_modules/azure_sentinel/configuration/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter_modules/azure_sentinel/configuration/config.json b/stix_shifter_modules/azure_sentinel/configuration/config.json index 667bf753b..f88c6b275 100644 --- a/stix_shifter_modules/azure_sentinel/configuration/config.json +++ b/stix_shifter_modules/azure_sentinel/configuration/config.json @@ -26,7 +26,7 @@ "options": { "alert": { "type": "boolean", - "default": false + "default": true }, "alertV2": { "type": "boolean", From 8c176d4345b104e183a4da8de5298a9c4f96dede Mon Sep 17 00:00:00 2001 From: Danny Elliott Date: Tue, 3 Oct 2023 13:45:31 -0300 Subject: [PATCH 03/13] 6.2.2 CHANGELOG --- CHANGELOG.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebd274de3..7dceba925 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,27 @@ We have started this changelogs from version 4.0.0. So, changes on previously re -------------------------------------- +## 6.2.2 (2023-10-03) + +### Changes: + +* include connector type in logger error [#1585](https://github.com/opencybersecurityalliance/stix-shifter/pull/1585) +* Add new screen shots to CLI Lab [#1576](https://github.com/opencybersecurityalliance/stix-shifter/pull/1576) + +### Fixes: + +* Update Azure Log Analytics stix transmission to use BaseJsonSyncConnector [#1584](https://github.com/opencybersecurityalliance/stix-shifter/pull/1584) +* Fixing authentication token handling [#1583](https://github.com/opencybersecurityalliance/stix-shifter/pull/1583) +* allow host address input in MS Graph configuration [#1582](https://github.com/opencybersecurityalliance/stix-shifter/pull/1582) +* fix coding lab [#1578](https://github.com/opencybersecurityalliance/stix-shifter/pull/1578) +* Fix and update coding lab [#1577](https://github.com/opencybersecurityalliance/stix-shifter/pull/1577) + +### Dependency update: + +* Bump aioboto3 from 11.2.0 to 11.3.0 in /stix_shifter [#1575](https://github.com/opencybersecurityalliance/stix-shifter/pull/1575) + +-------------------------------------- + ## 6.2.1 (2023-09-07) ### Changes: From 7fb3fdef861c0dc645f6d85c8527b86544fc8800 Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Thu, 5 Oct 2023 14:51:42 -0400 Subject: [PATCH 04/13] add from stix mapping of OS in ECS (#1597) --- .../elastic_ecs/stix_translation/json/from_stix_map.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json index 01e2ce1f0..9b988627e 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/elastic_ecs/stix_translation/json/from_stix_map.json @@ -272,7 +272,7 @@ "egress.interface.id": ["observer.egress.interface.id"], "egress.interface.name": ["observer.egress.interface.name"], "uptime": ["host.uptime"], - "os_ref.name": ["host.os.name", "observer.os.name", "observer.product"], + "os_ref.name": ["host.os.name", "os.name", "os.type", "observer.os.name", "observer.product"], "os_ref.vendor": ["host.os.platform", "observer.os.platform", "observer.vendor"], "os_ref.version": ["host.os.version", "observer.os.version", "observer.version"], "container.id": ["container.id"], From 76264e96e82f8399da7ab7386d6119322914d2cf Mon Sep 17 00:00:00 2001 From: Md Azam Date: Thu, 5 Oct 2023 16:11:01 -0300 Subject: [PATCH 05/13] Upgrade urllib3 version in dependency (#1594) --- stix_shifter/requirements.txt | 2 +- stix_shifter_modules/azure_log_analytics/requirements.txt | 1 - stix_shifter_modules/datadog/requirements.txt | 3 +-- stix_shifter_modules/gcp_chronicle/requirements.txt | 3 +-- stix_shifter_modules/onelogin/requirements.txt | 3 +-- stix_shifter_modules/sumologic/requirements.txt | 1 - 6 files changed, 4 insertions(+), 9 deletions(-) diff --git a/stix_shifter/requirements.txt b/stix_shifter/requirements.txt index 9df48d0e7..67f51de5d 100644 --- a/stix_shifter/requirements.txt +++ b/stix_shifter/requirements.txt @@ -17,5 +17,5 @@ requests_toolbelt==0.10.1 stix2-matcher==3.0.0 stix2-patterns==1.3.2 xmltodict==0.13.0 -urllib3==1.26.15 +urllib3==1.26.17 regex==2023.8.8 \ No newline at end of file diff --git a/stix_shifter_modules/azure_log_analytics/requirements.txt b/stix_shifter_modules/azure_log_analytics/requirements.txt index c33b49f87..773c9be12 100644 --- a/stix_shifter_modules/azure_log_analytics/requirements.txt +++ b/stix_shifter_modules/azure_log_analytics/requirements.txt @@ -1,4 +1,3 @@ azure-monitor-query==1.0.2 pandas==1.5.2 -urllib3==1.26.15 jsonref==1.1.0 \ No newline at end of file diff --git a/stix_shifter_modules/datadog/requirements.txt b/stix_shifter_modules/datadog/requirements.txt index ce5787711..794cb9d46 100644 --- a/stix_shifter_modules/datadog/requirements.txt +++ b/stix_shifter_modules/datadog/requirements.txt @@ -1,2 +1 @@ -datadog_api_client[async]==2.12.0 -urllib3==1.26.15 +datadog_api_client[async]==2.12.0 \ No newline at end of file diff --git a/stix_shifter_modules/gcp_chronicle/requirements.txt b/stix_shifter_modules/gcp_chronicle/requirements.txt index 632c9c928..ad89dd3ae 100644 --- a/stix_shifter_modules/gcp_chronicle/requirements.txt +++ b/stix_shifter_modules/gcp_chronicle/requirements.txt @@ -1,2 +1 @@ -aiogoogle==5.1.0 -urllib3==1.26.15 +aiogoogle==5.1.0 \ No newline at end of file diff --git a/stix_shifter_modules/onelogin/requirements.txt b/stix_shifter_modules/onelogin/requirements.txt index 73b1d51ea..09d427e2b 100644 --- a/stix_shifter_modules/onelogin/requirements.txt +++ b/stix_shifter_modules/onelogin/requirements.txt @@ -1,2 +1 @@ -onelogin==2.0.1 -urllib3==1.26.15 +onelogin==2.0.1 \ No newline at end of file diff --git a/stix_shifter_modules/sumologic/requirements.txt b/stix_shifter_modules/sumologic/requirements.txt index cde879fda..dc3a80d65 100644 --- a/stix_shifter_modules/sumologic/requirements.txt +++ b/stix_shifter_modules/sumologic/requirements.txt @@ -1,2 +1 @@ sumologic-sdk==0.1.13 -urllib3==1.26.15 From 068e9eeada98faed4f22575669ac3e0a14b913d0 Mon Sep 17 00:00:00 2001 From: thangaraj-ramesh <92723742+thangaraj-ramesh@users.noreply.github.com> Date: Wed, 11 Oct 2023 00:08:29 +0530 Subject: [PATCH 06/13] Vectra config changes (#1581) --- stix_shifter_modules/vectra/README.md | 5 +++-- .../vectra/configuration/config.json | 12 ++++++------ .../vectra/configuration/lang_en.json | 7 ++++++- .../stix_translation/json/stix_2_1/to_stix_map.json | 3 ++- .../vectra/stix_translation/json/to_stix_map.json | 3 ++- .../vectra/stix_translation/transformers.py | 2 +- .../vectra/stix_transmission/api_client.py | 2 +- .../vectra/stix_transmission/connector.py | 6 ++++++ .../vectra/test/stix_transmission/test_vectra.py | 5 ++++- 9 files changed, 31 insertions(+), 14 deletions(-) diff --git a/stix_shifter_modules/vectra/README.md b/stix_shifter_modules/vectra/README.md index ddc37e43b..89bfe6d70 100644 --- a/stix_shifter_modules/vectra/README.md +++ b/stix_shifter_modules/vectra/README.md @@ -44,7 +44,7 @@ translate vectra query {} "[ipv4-addr:value='1.1.1.1' AND x-ibm-finding:name='Hi ```shell transmit vectra -"{\"host\":\"instance.vectra.com\"}" +"{\"host\":\"instance.vectra.com\", \"port\":xxxx}" "{\"auth\":{\"apitoken\": \"xxxx\"}}" results "[query_string=(detection.detection_type:\"Hidden HTTP Tunnel\" AND (detection.src_ip:\"1.1.1.1\" OR detection.grouped_details.dst_ips:\"1.1.1.1\" OR detection.grouped_details.dst_hosts.dst_ip:\"1.1.1.1\" OR detection.grouped_details.origin_ip:\"1.1.1.1\" OR detection.grouped_details.sessions.dst_ip:\"1.1.1.1\" OR detection.grouped_details.subnet:\"1.1.1.1\" OR detection.grouped_details.events.dst_ip:\"1.1.1.1\" OR detection.grouped_details.events.dst_ips:\"1.1.1.1\" OR detection.grouped_details.events.sessions.dst_ip:\"1.1.1.1\" OR detection.grouped_details.connection_events.target_host.ip:\"1.1.1.1\") AND (detection.last_timestamp:[2023-04-01T0000 to 2023-06-12T0000]))]" @@ -301,7 +301,7 @@ execute vectra vectra "{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"Vectra NDR\",\"identity_class\":\"system\",\"created\":\"2023-02-23T13:22:50.336Z\",\"modified\":\"2022-02-23T13:22:50.336Z\"}" -"{\"host\":\"xyz\"}" +"{\"host\":\"xyz\", \"port\":xxxx}" "{\"auth\":{\"api_token\": \"xxx\"}}" "([x-ibm-finding:confidence>20 AND x-sql-request-info:response_code=404] AND [x-ibm-finding:severity>20 AND x-sql-request-info:user_agent LIKE 'Mozilla']) START t'2023-04-01T00:00:00.000Z' STOP t'2023-06-12T00:00:00.000Z'" ``` @@ -437,3 +437,4 @@ vectra - [Advanced Search Reference Guide](https://support.vectra.ai/s/article/KB-VS-1116) - [Understanding Vectra AI](https://support.vectra.ai/s/article/KB-VS-1285) - [Detection and Campaign lifespan and retention periods](https://support.vectra.ai/s/article/KB-VS-1099) + diff --git a/stix_shifter_modules/vectra/configuration/config.json b/stix_shifter_modules/vectra/configuration/config.json index d15c190bf..3bebf80d8 100644 --- a/stix_shifter_modules/vectra/configuration/config.json +++ b/stix_shifter_modules/vectra/configuration/config.json @@ -8,15 +8,15 @@ "type": "text", "regex": "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\\-]*[a-zA-Z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\\-]*[A-Za-z0-9])$" }, + "port": { + "type": "number", + "default": 443, + "min": 1, + "max": 65535 + }, "help": { "type": "link", "default": "data-sources.html" - }, - "options": { - "type": "fields", - "result_limit": { - "max": 10000 - } } }, "configuration": { diff --git a/stix_shifter_modules/vectra/configuration/lang_en.json b/stix_shifter_modules/vectra/configuration/lang_en.json index 2f5708bd1..5b587b26f 100644 --- a/stix_shifter_modules/vectra/configuration/lang_en.json +++ b/stix_shifter_modules/vectra/configuration/lang_en.json @@ -4,6 +4,10 @@ "label": "Management IP address or Hostname", "description": "Specify the IP address or hostname of the data source" }, + "port": { + "label": "Host port", + "description": "Set the port number that is associated with the hostname or IP address" + }, "help": { "label": "Need additional help?", "description": "More details on the data source setting can be found in the specified link" @@ -12,7 +16,8 @@ "configuration": { "auth": { "api_token": { - "type": "password" + "label": "API token", + "description": "Vectra API token to authenticate requests for Vectra APIs" } } } diff --git a/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json index 00e982618..eb1784cd1 100644 --- a/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json +++ b/stix_shifter_modules/vectra/stix_translation/json/stix_2_1/to_stix_map.json @@ -62,7 +62,8 @@ }, "num_successes": { "key": "x-ibm-finding.x_num_successes", - "object": "detection" + "object": "detection", + "transformer": "ToString" }, "dst_ports": { "key": "x-ibm-finding.x_dst_ports", diff --git a/stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json b/stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json index 230a6696c..1940601e8 100644 --- a/stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/vectra/stix_translation/json/to_stix_map.json @@ -62,7 +62,8 @@ }, "num_successes": { "key": "x-ibm-finding.x_num_successes", - "object": "detection" + "object": "detection", + "transformer": "ToString" }, "dst_ports": { "key": "x-ibm-finding.x_dst_ports", diff --git a/stix_shifter_modules/vectra/stix_translation/transformers.py b/stix_shifter_modules/vectra/stix_translation/transformers.py index a86371349..29c43e5f3 100644 --- a/stix_shifter_modules/vectra/stix_translation/transformers.py +++ b/stix_shifter_modules/vectra/stix_translation/transformers.py @@ -39,7 +39,7 @@ class ConvertToReal(ValueTransformer): def transform(obj): try: if not isinstance(obj, float): - obj = obj * 1.0 + obj = obj / 100 except ValueError: LOGGER.error('Cannot convert input %s to a float value between 0 to 1', obj) return obj diff --git a/stix_shifter_modules/vectra/stix_transmission/api_client.py b/stix_shifter_modules/vectra/stix_transmission/api_client.py index 01ddc7fcd..6930542b6 100644 --- a/stix_shifter_modules/vectra/stix_transmission/api_client.py +++ b/stix_shifter_modules/vectra/stix_transmission/api_client.py @@ -13,7 +13,7 @@ def __init__(self, connection, configuration): self.headers = {"Authorization": "Token " + self.auth["api_token"], 'Content-Type': "application/json", 'Cache-Control': "no-cache"} - self.client = RestApiClientAsync(connection.get('host'), port=None, headers=self.headers) + self.client = RestApiClientAsync(connection.get('host'), port=connection.get('port'), headers=self.headers) self.host = connection.get('host') async def ping_data_source(self): diff --git a/stix_shifter_modules/vectra/stix_transmission/connector.py b/stix_shifter_modules/vectra/stix_transmission/connector.py index b1a25a935..32f9b135c 100644 --- a/stix_shifter_modules/vectra/stix_transmission/connector.py +++ b/stix_shifter_modules/vectra/stix_transmission/connector.py @@ -223,6 +223,12 @@ def get_results_data(self, response_dict): detection_type = record.get('detection_type', '') + # if x-ibm-finding object event_count is not available, setting the default value to 1. + # if default value is not set, CP4S inserts NaN value for event_count which causes rendering issue in UI. + if record.get('summary') and \ + 'num_attempts' not in record['summary'] and 'num_sessions' not in record['summary']: + record['summary']['num_sessions'] = 1 + if 'Privilege' in detection_type: # Skip any preprocessing for these detections. continue diff --git a/stix_shifter_modules/vectra/test/stix_transmission/test_vectra.py b/stix_shifter_modules/vectra/test/stix_transmission/test_vectra.py index 8e79640f7..9cbb10273 100644 --- a/stix_shifter_modules/vectra/test/stix_transmission/test_vectra.py +++ b/stix_shifter_modules/vectra/test/stix_transmission/test_vectra.py @@ -333,7 +333,10 @@ class TestVectraConnection(unittest.TestCase, object): def connection(self): """format for connection""" - return {"host": "hostbla"} + return { + "host": "hostbla", + "port": 443 + } def configuration(self): """format for configuration""" From a26611ef03c82d94e0dc084b0b1cd2fd6d616c5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 09:45:03 -0300 Subject: [PATCH 07/13] Bump regex from 2023.8.8 to 2023.10.3 in /stix_shifter (#1598) Bumps [regex](https://github.com/mrabarnett/mrab-regex) from 2023.8.8 to 2023.10.3. - [Changelog](https://github.com/mrabarnett/mrab-regex/blob/hg/changelog.txt) - [Commits](https://github.com/mrabarnett/mrab-regex/compare/2023.8.8...2023.10.3) --- updated-dependencies: - dependency-name: regex dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Md Azam --- stix_shifter/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter/requirements.txt b/stix_shifter/requirements.txt index 67f51de5d..36ccc4899 100644 --- a/stix_shifter/requirements.txt +++ b/stix_shifter/requirements.txt @@ -18,4 +18,4 @@ stix2-matcher==3.0.0 stix2-patterns==1.3.2 xmltodict==0.13.0 urllib3==1.26.17 -regex==2023.8.8 \ No newline at end of file +regex==2023.10.3 \ No newline at end of file From 7b3ea929f906ba7c1cb43e660a2788ff6586866d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 11:46:12 -0300 Subject: [PATCH 08/13] Bump attrs from 22.2.0 to 23.1.0 in /stix_shifter (#1595) Bumps [attrs](https://github.com/python-attrs/attrs) from 22.2.0 to 23.1.0. - [Release notes](https://github.com/python-attrs/attrs/releases) - [Changelog](https://github.com/python-attrs/attrs/blob/main/CHANGELOG.md) - [Commits](https://github.com/python-attrs/attrs/compare/22.2.0...23.1.0) --- updated-dependencies: - dependency-name: attrs dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- stix_shifter/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter/requirements.txt b/stix_shifter/requirements.txt index 36ccc4899..e7586bbce 100644 --- a/stix_shifter/requirements.txt +++ b/stix_shifter/requirements.txt @@ -4,7 +4,7 @@ aiomysql==0.2.0 antlr4-python3-runtime==4.8 asyncio==3.4.3 asynctest==0.13.0 -attrs==22.2.0 +attrs==23.1.0 azure-identity==1.12.0 colorlog==6.7.0 flask==2.3.3 From 7774c0ae6cdf4e3e939593a56e1dbcd5587f5d1e Mon Sep 17 00:00:00 2001 From: thangaraj-ramesh <92723742+thangaraj-ramesh@users.noreply.github.com> Date: Thu, 12 Oct 2023 23:46:09 +0530 Subject: [PATCH 09/13] cisco secure email connector (#1579) --- .../cisco_secure_email/README.md | 414 +++++++++++++++ .../cisco_secure_email/__init__.py | 0 .../cisco_secure_email_supported_stix.md | 162 ++++++ .../configuration/config.json | 37 ++ .../configuration/lang_en.json | 32 ++ .../cisco_secure_email/entry_point.py | 11 + .../stix_translation/__init__.py | 0 .../stix_translation/json/config_map.json | 228 +++++++++ .../stix_translation/json/from_stix_map.json | 221 ++++++++ .../stix_translation/json/operators.json | 9 + .../json/stix_2_1/from_stix_map.json | 221 ++++++++ .../json/stix_2_1/to_stix_map.json | 159 ++++++ .../stix_translation/json/to_stix_map.json | 159 ++++++ .../stix_translation/query_constructor.py | 470 ++++++++++++++++++ .../stix_translation/query_translator.py | 27 + .../stix_translation/transformers.py | 69 +++ .../stix_transmission/__init__.py | 0 .../stix_transmission/api_client.py | 55 ++ .../stix_transmission/connector.py | 175 +++++++ .../stix_transmission/error_mapper.py | 40 ++ .../test_cisco_secure_email_json_to_stix.py | 157 ++++++ .../test_cisco_secure_email_stix_to_query.py | 295 +++++++++++ .../test_cisco_secure_email.py | 351 +++++++++++++ 23 files changed, 3292 insertions(+) create mode 100644 stix_shifter_modules/cisco_secure_email/README.md create mode 100644 stix_shifter_modules/cisco_secure_email/__init__.py create mode 100644 stix_shifter_modules/cisco_secure_email/cisco_secure_email_supported_stix.md create mode 100644 stix_shifter_modules/cisco_secure_email/configuration/config.json create mode 100644 stix_shifter_modules/cisco_secure_email/configuration/lang_en.json create mode 100644 stix_shifter_modules/cisco_secure_email/entry_point.py create mode 100644 stix_shifter_modules/cisco_secure_email/stix_translation/__init__.py create mode 100644 stix_shifter_modules/cisco_secure_email/stix_translation/json/config_map.json create mode 100644 stix_shifter_modules/cisco_secure_email/stix_translation/json/from_stix_map.json create mode 100644 stix_shifter_modules/cisco_secure_email/stix_translation/json/operators.json create mode 100644 stix_shifter_modules/cisco_secure_email/stix_translation/json/stix_2_1/from_stix_map.json create mode 100644 stix_shifter_modules/cisco_secure_email/stix_translation/json/stix_2_1/to_stix_map.json create mode 100644 stix_shifter_modules/cisco_secure_email/stix_translation/json/to_stix_map.json create mode 100644 stix_shifter_modules/cisco_secure_email/stix_translation/query_constructor.py create mode 100644 stix_shifter_modules/cisco_secure_email/stix_translation/query_translator.py create mode 100644 stix_shifter_modules/cisco_secure_email/stix_translation/transformers.py create mode 100644 stix_shifter_modules/cisco_secure_email/stix_transmission/__init__.py create mode 100644 stix_shifter_modules/cisco_secure_email/stix_transmission/api_client.py create mode 100644 stix_shifter_modules/cisco_secure_email/stix_transmission/connector.py create mode 100644 stix_shifter_modules/cisco_secure_email/stix_transmission/error_mapper.py create mode 100644 stix_shifter_modules/cisco_secure_email/test/stix_translation/test_cisco_secure_email_json_to_stix.py create mode 100644 stix_shifter_modules/cisco_secure_email/test/stix_translation/test_cisco_secure_email_stix_to_query.py create mode 100644 stix_shifter_modules/cisco_secure_email/test/stix_transmission/test_cisco_secure_email.py diff --git a/stix_shifter_modules/cisco_secure_email/README.md b/stix_shifter_modules/cisco_secure_email/README.md new file mode 100644 index 000000000..6ebf48e64 --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/README.md @@ -0,0 +1,414 @@ +# Cisco Secure Email + +**Table of Contents** + +- [Cisco Secure Email API Endpoints](#Cisco Secure Email-api-endpoints) +- [Pattern expression with STIX and CUSTOM attributes - Single Observation](#single-observation) +- [Pattern expression with STIX and CUSTOM attributes - Multiple Observation](#multiple-observation) +- [STIX Execute Query](#stix-execute-query) +- [Limitations](#limitations) +- [Observations](#observations) +- [References](#references) + +### Cisco Secure Email API Endpoints + + | Connector Method | Cisco Secure Email API Endpoint | Method | + |-----|------| ------| + | Ping Endpoint | https://< server >/esa/api/v2.0/login/privileges | GET | + | Token Endpoint | https://< server >/esa/api/v2.0/login | POST | + | Results Endpoint | https://< server >/esa/api/v2.0/message-tracking/messages | GET | + +### Format for calling stix-shifter from the command line +``` +python main.py `` `` `` `` +``` +### Pattern expression with STIX and CUSTOM attributes + +#### Single Observation + +#### STIX Translate query to fetch the messages from a specific ipaddress +```shell +translate cisco_secure_email query {} "[ipv4-addr:value='1.1.1.1'] START t'2023-08-01T11:00:00.000Z' STOP t'2023-08-31T21:00:00.003Z'" +``` +#### STIX Translate query - output +```json +{ + "queries": [ + "senderIp=1.1.1.1&startDate=2023-08-01T11:00:00.000Z&endDate=2023-08-31T21:00:00.000Z" + ] +} + +``` + +#### STIX Transmit results + +```shell +transmit +cisco_secure_email +"{\"host\":\"instance.cisco_secure_email\", \"port\":xxxx, \"selfSignedCert\":\"-----BEGIN CERTIFICATE-----xxxxx-----END CERTIFICATE-----\"}" +"{\"auth\":{\"username\":\"apiuser\", \"password\":\"xxxx\"}}" +results +"senderIp=1.1.1.1&startDate=2023-08-01T11:00:00.000Z&endDate=2023-08-31T21:00:00.000Z" +0 +1 +``` + +#### STIX Transmit results - output +```json +{ + "success": true, + "data": [ + { + "attributes": { + "hostName": "host", + "friendly_from": [ + "user1@dummydomain.com" + ], + "isCompleteData": "N/A", + "messageStatus": { + "1757": "Delivered" + }, + "recipientMap": { + "1756": [ + "user1@dummydomain.com" + ], + "1757": [ + "user1@dummydomain.com" + ] + }, + "senderIp": "1.1.1.1", + "mailPolicy": [ + "DEFAULT" + ], + "senderGroup": "UNKNOWNLIST", + "morInfo": { + "midVsState": { + "1757": "Delivered" + } + }, + "subject": "URL - https://www.indianexpress.com", + "mid": [ + 1756, + 1757 + ], + "senderDomain": "amazonses.com", + "finalSubject": { + "1757": "URL - https://www.indianexpress.com" + }, + "direction": "incoming", + "icid": 1625, + "morDetails": {}, + "replyTo": "user3@dummydomain.com", + "timestamp": "29 Aug 2023 10:00:48 (GMT +00:00)", + "messageID": { + "1756": "<0100018a404d1371-34997a47-abde-49df-bb90-d6c22aaee8ba-000000@email.amazonses.com>" + }, + "verdictChart": { + "1757": "01101110" + }, + "recipient": [ + "user1@dummydomain.com" + ], + "sender": "0100018a404d1371-34997a47-abde-49df-bb90-d6c22aaee8ba-000000@amazonses.com", + "serialNumber": "EC2CD1D95C273722A23A-CA0C47E74D1B", + "allIcid": [ + 1625 + ], + "sbrs": "3.5" + } + } + ] +} +``` + + +#### STIX Translate results + +```json +{ + "type": "bundle", + "id": "bundle--31b224bc-75a0-4598-b4a6-c12c0855c2e5", + "objects": [ + { + "type": "identity", + "id": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", + "name": "cisco secure email", + "identity_class": "events" + }, + { + "id": "observed-data--c9200bf6-57ef-49df-8a9c-40cbe88359fd", + "type": "observed-data", + "created_by_ref": "identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3", + "created": "2023-09-19T08:43:16.582Z", + "modified": "2023-09-19T08:43:16.582Z", + "objects": { + "0": { + "type": "x-oca-asset", + "hostname": "host" + }, + "1": { + "type": "email-message", + "x_cisco_host_ref": "0", + "from_ref": "2", + "is_multipart": true, + "x_sender_ip_ref": "4", + "x_sender_group": "UNKNOWNLIST", + "subject": "URL - https://www.indianexpress.com", + "x_cisco_mid": 1756, + "x_cisco_icid": 1625, + "date": "2023-08-29T10:00:00.000Z", + "x_message_id_header": "<0100018a404d1371-34997a47-abde-49df-bb90-d6c22aaee8ba-000000@email.amazonses.com>", + "to_refs": [ + "7" + ], + "sender_ref": "8", + "x_serial_number": "EC2CD1D95C273722A23A-CA0C47E74D1B" + }, + "2": { + "type": "email-addr", + "value": "user1@dummydomain.com" + }, + "3": { + "type": "x-cisco-email-msgevent", + "message_status": "Delivered", + "mail_policy": [ + "DEFAULT" + ], + "direction": "incoming", + "reply_to": "6", + "sbrs_score": "3.5" + }, + "4": { + "type": "ipv4-addr", + "value": "1.1.1.1" + }, + "5": { + "type": "domain-name", + "value": "amazonses.com", + "resolves_to_refs": [ + "4" + ] + }, + "6": { + "type": "email-addr", + "value": "user3@dummydomain.com" + }, + "7": { + "type": "email-addr", + "value": "user1@dummydomain.com" + }, + "8": { + "type": "email-addr", + "value": "0100018a404d1371-34997a47-abde-49df-bb90-d6c22aaee8ba-000000@amazonses.com" + } + }, + "first_observed": "2023-08-29T10:00:00.000Z", + "last_observed": "2023-08-29T10:00:00.000Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} +``` + +#### Multiple Observation + +```shell +translate +cisco_secure_email +query {} +"[ipv4-addr:value = '1.1.1.1' AND file:hashes.'SHA-256' = '271c0119ac4455fc8db4ef4a8caf8e2bfcfb8bbd3b8c894e117a9ae9f743894b' ] OR [x-cisco-email-msgevent:message_status = 'DELIVERED']START t'2023-07-19T01:56:00.000Z' STOP t'2023-09-01T01:57:00.003Z'" +``` + +#### STIX Multiple observation - output +```json +{ + "queries": [ + "fileSha256=271c0119ac4455fc8db4ef4a8caf8e2bfcfb8bbd3b8c894e117a9ae9f743894b&senderIp=1.1.1.1&startDate=2023-09-05T10:52:00.000Z&endDate=2023-09-05T10:57:00.000Z", + "deliveryStatus=DELIVERED&startDate=2023-07-19T01:56:00.000Z&endDate=2023-09-01T01:57:00.000Z" + ] +} +``` + +### STIX Execute query +```shell +execute +cisco_secure_email +cisco_secure_email +"{\"type\":\"identity\",\"id\":\"identity--f431f809-377b-45e0-aa1c-6a4751cae5ff\",\"name\":\"cisco_secure_email\",\"identity_class\":\"events\",\"created\":\"2023-02-23T13:22:50.336Z\",\"modified\":\"2022-02-23T13:22:50.336Z\"}" +"{\"host\":\"instance.cisco_secure_email\", \"port\":xxxx, \"selfSignedCert\":\"-----BEGIN CERTIFICATE-----xxxxx-----END CERTIFICATE-----\"}" +"{\"auth\":{\"username\":\"apiuser\", \"password\":\"xxxx\"}}" +"[ipv4-addr:value = '1.1.1.1' AND file:hashes.'SHA-256' = '271c0119ac4455fc8db4ef4a8caf8e2bfcfb8bbd3b8c894e117a9ae9f743894b' ] OR [x-cisco-email-msgevent:message_status = 'DELIVERED']START t'2023-07-19T01:56:00.000Z' STOP t'2023-09-01T01:57:00.003Z'" +``` + +#### STIX Execute query - output +```json +{ + "type": "bundle", + "id": "bundle--dc608fc4-f6e6-4d7e-8ea5-124441153e77", + "objects": [ + { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "cisco_secure_email", + "identity_class": "events", + "created": "2023-02-23T13:22:50.336Z", + "modified": "2022-02-23T13:22:50.336Z" + }, + { + "id": "observed-data--c0129c6d-9251-46b2-9b01-62bfbabeea45", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2023-09-05T11:06:27.543Z", + "modified": "2023-09-05T11:06:27.543Z", + "objects": { + "0": { + "type": "email-addr", + "value": "user1@dummydomain.com" + }, + "1": { + "type": "email-message", + "from_ref": "0", + "is_multipart": true, + "x_sender_ip_ref": "3", + "x_sender_group": "UNKNOWNLIST", + "subject": "new site test", + "x_cisco_mid": 1786, + "x_cisco_icid": 1697, + "date": "2023-08-31T09:54:00.000Z", + "x_message_id_header": "<0100018a4b044014-40f605c5-fb57-497c-b6d3-87e5cc7e661d-000000@email.amazonses.com>", + "to_refs": [ + "6" + ], + "sender_ref": "7", + "x_serial_number": "EC2CD1D95C273722A23A-CA0C47E74D1B" + }, + "2": { + "type": "x-cisco-email-msgevent", + "message_status": "Delivered", + "mail_policy": [ + "DEFAULT" + ], + "direction": "incoming", + "reply_to": "5", + "sbrs_score": "5.2" + }, + "3": { + "type": "ipv4-addr", + "value": "1.1.1.1" + }, + "4": { + "type": "domain-name", + "value": "amazonses.com", + "resolves_to_refs": [ + "3" + ] + }, + "5": { + "type": "email-addr", + "value": "user11@dummydomain.com" + }, + "6": { + "type": "email-addr", + "value": "user1@dummydomain.com" + }, + "7": { + "type": "email-addr", + "value": "0100018a4b044014-40f605c5-fb57-497c-b6d3-87e5cc7e661d-000000@amazonses.com" + } + }, + "first_observed": "2023-09-05T11:06:27.543Z", + "last_observed": "2023-09-05T11:06:27.543Z", + "number_observed": 1 + }, + { + "id": "observed-data--0affd03f-d76d-4729-a546-9f7a9c666bce", + "type": "observed-data", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2023-09-05T11:06:27.547Z", + "modified": "2023-09-05T11:06:27.547Z", + "objects": { + "0": { + "type": "email-addr", + "value": "user1@dummydomain.com" + }, + "1": { + "type": "email-message", + "from_ref": "0", + "is_multipart": true, + "x_sender_ip_ref": "3", + "x_sender_group": "UNKNOWNLISTTEST", + "subject": "Fwd: image", + "x_cisco_mid": 1783, + "x_cisco_icid": 1693, + "date": "2023-08-31T08:31:00.000Z", + "x_message_id_header": "<0100018a4ab7bcc1-2de5bed1-70f4-412c-aa19-478aca21b37c-000000@email.amazonses.com>", + "to_refs": [ + "6" + ], + "sender_ref": "7", + "x_serial_number": "EC2CD1D95C273722A23A-CA0C47E74D1B" + }, + "2": { + "type": "x-cisco-email-msgevent", + "message_status": "Delivered", + "mail_policy": [ + "DEFAULT" + ], + "direction": "incoming", + "reply_to": "5", + "sbrs_score": "3.5" + }, + "3": { + "type": "ipv4-addr", + "value": "1.1.1.1" + }, + "4": { + "type": "domain-name", + "value": "amazonses.com", + "resolves_to_refs": [ + "3" + ] + }, + "5": { + "type": "email-addr", + "value": "user11@dummydomain.com" + }, + "6": { + "type": "email-addr", + "value": "user3@dummydomain.com" + }, + "7": { + "type": "email-addr", + "value": "0100018a4ab7bcc1-2de5bed1-70f4-412c-aa19-478aca21b37c-000000@amazonses.com" + } + }, + "first_observed": "2023-09-05T11:06:27.547Z", + "last_observed": "2023-09-05T11:06:27.547Z", + "number_observed": 1 + } + ], + "spec_version": "2.0" +} +``` + +### Observations +- LIKE, IN operators are supported for certain fields only.(For example: envelopeRecipientfilterValue, + envelopeSenderfilterValue, subjectfilterValue, domainNameValue, replyToValue, attachmentNameValue, + contentFiltersDirection, quarantinedTo, urlCategories) +- AND isn’t supported within the attributes belonging to x-cisco-email-msgevent group. + Example: "[x-cisco-email-msgevent:spam_positive='true' AND x-cisco-email-msgevent:virus_positive='true']" + Wrong Parameter error is returned in such case. +- The user inactivity timeout settings in the email gateway applies to the validity of a JWT(5 - 1440 Minutes (24 hours)). + The email gateway checks every API query with a JWT, for its time validity. So based on the response code a new refresh JWT is generated, and to be used with API calls. + + +### Limitations +- Due to a Cisco API limitation, if timeout error occurs it is recommended increase the timeout configuration to higher value and/or configure smaller value for result limit. +- ‘NOT’, <, >, <=, >= operators are not supported. + +### References +- [Cisco Secure Email Product Overview](https://www.cisco.com/c/en/us/products/security/email-security/what-is-secure-email.html) +- [Overview Guide for Cisco Cloud/Hybrid Secure Email](https://www.cisco.com/c/dam/en/us/td/docs/security/ces/overview_guide/Cisco_Cloud_Hybrid_Email_Security_Overview_Guide.pdf) +- [AsyncOS 15.0 API for Cisco Secure Email](https://www.cisco.com/c/en/us/td/docs/security/esa/esa15-0/api_guide/b_Secure_Email_API_Guide_15-0/b_ESA_API_Guide_chapter_01.html) +- [AsyncOS 15.0 API - Addendum to the Getting Started Guide for Cisco Secure Email](https://www.cisco.com/c/en/us/support/security/email-security-appliance/products-programming-reference-guides-list.html) \ No newline at end of file diff --git a/stix_shifter_modules/cisco_secure_email/__init__.py b/stix_shifter_modules/cisco_secure_email/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/cisco_secure_email/cisco_secure_email_supported_stix.md b/stix_shifter_modules/cisco_secure_email/cisco_secure_email_supported_stix.md new file mode 100644 index 000000000..d6aa8c354 --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/cisco_secure_email_supported_stix.md @@ -0,0 +1,162 @@ +##### Updated on 14/09/23 +## Cisco Secure Email +### Results STIX Domain Objects +* Identity +* Observed Data +
+### Supported STIX Operators +*Comparison AND/OR operators are inside the observation while observation AND/OR operators are between observations (square brackets).* + +| STIX Operator | Data Cisco Secure Email Operator | +|--|--| +| AND (Comparision) | & | +| = | = | +| IN | = | +| LIKE | = | +| OR (Observation) | OR | +| AND (Observation) | OR | +|
| | +### Searchable STIX objects and properties +| STIX Object and Property | Mapped Data Source Fields | +|--|--| +| **email-addr**:value | envelopeRecipientfilterValue, envelopeSenderfilterValue | +| **email-message**:from_ref | envelopeSenderfilterValue | +| **email-message**:sender_ref | envelopeSenderfilterValue | +| **email-message**:to_refs | envelopeRecipientfilterValue | +| **email-message**:subject | subjectfilterValue | +| **email-message**:x_message_id_header | messageIdHeader | +| **email-message**:x_cisco_mid | ciscoMid | +| **email-message**:x_sender_ip_ref | senderIp | +| **file**:name | attachmentNameValue | +| **file**:hashes.'SHA-256' | fileSha256 | +| **ipv4-addr**:value | senderIp | +| **ipv6-addr**:value | senderIp | +| **domain-name**:value | domainNameValue | +| **x-oca-host**:hostname | ciscoHost | +| **x-cisco-email-msgevent**:advanced_malware_protection_mailflow_direction | advancedMalwareProtectionMailflowDirection | +| **x-cisco-email-msgevent**:advanced_malware_protection | advancedMalwareProtection | +| **x-cisco-email-msgevent**:app_forwarding | appForwarding | +| **x-cisco-email-msgevent**:content_filters_name | contentFiltersName | +| **x-cisco-email-msgevent**:content_filters_direction | contentFiltersDirection | +| **x-cisco-email-msgevent**:content_filters_action | contentFiltersAction | +| **x-cisco-email-msgevent**:dane_failure | daneFailure | +| **x-cisco-email-msgevent**:message_status | deliveryStatus | +| **x-cisco-email-msgevent**:message_delivered | message_delivered | +| **x-cisco-email-msgevent**:dlp_violations_names | dlpViolationsNames | +| **x-cisco-email-msgevent**:dlpViolationsSeverities | dlpViolationsSeverities | +| **x-cisco-email-msgevent**:dlp_action | dlpAction | +| **x-cisco-email-msgevent**:dmarc_from | dmarcFrom | +| **x-cisco-email-msgevent**:dmarc_action | dmarcAction | +| **x-cisco-email-msgevent**:etf_sources | etfSources | +| **x-cisco-email-msgevent**:etf_iocs | etfIocs | +| **x-cisco-email-msgevent**:forged_email_detection | forgedEmailDetection | +| **x-cisco-email-msgevent**:geo_location | geoLocation | +| **x-cisco-email-msgevent**:graymail | graymail | +| **x-cisco-email-msgevent**:hard_bounced | hardBounced | +| **x-cisco-email-msgevent**:ip_reputation | ipReputation | +| **x-cisco-email-msgevent**:macro_mailflow_direction | macroMailflowDirection | +| **x-cisco-email-msgevent**:macro_file_types_detected | macroFileTypesDetected | +| **x-cisco-email-msgevent**:message_filters | messageFilters | +| **x-cisco-email-msgevent**:message_direction | messageDirection | +| **x-cisco-email-msgevent**:contained_malicious_urls | containedMaliciousUrls | +| **x-cisco-email-msgevent**:contained_neutral_urls | containedNeutralUrls | +| **x-cisco-email-msgevent**:outbreak_filters_url_rewritten_byof | outbreakFiltersUrlRewrittenByOf | +| **x-cisco-email-msgevent**:outbreak_filtersVofThreatCategory | outbreakFiltersVofThreatCategory | +| **x-cisco-email-msgevent**:in_outbreak_quarantine | inOutbreakQuarantine | +| **x-cisco-email-msgevent**:quarantined_to | quarantinedTo | +| **x-cisco-email-msgevent**:reply_to | replyToValue | +| **x-cisco-email-msgevent**:s_mime | smime | +| **x-cisco-email-msgevent**:domain_categories | domainCategories | +| **x-cisco-email-msgevent**:sdr_categories | sdrCategories | +| **x-cisco-email-msgevent**:sdr_threat_levels | sdrThreatLevels | +| **x-cisco-email-msgevent**:soft_bounced | softBounced | +| **x-cisco-email-msgevent**:spam_positive | spamPositive | +| **x-cisco-email-msgevent**:quarantined_as_spam | quarantinedAsSpam | +| **x-cisco-email-msgevent**:quarantine_status | quarantineStatus | +| **x-cisco-email-msgevent**:threat_name | threatName | +| **x-cisco-email-msgevent**:suspect_spam | suspectSpam | +| **x-cisco-email-msgevent**:url_categories | urlCategories | +| **x-cisco-email-msgevent**:url_reputation | urlReputation | +| **x-cisco-email-msgevent**:safeprint_ext | safeprintExt | +| **x-cisco-email-msgevent**:virus_positive | virusPositive | +| **x-cisco-email-msgevent**:web_interaction_tracking_urls | webInteractionTrackingUrls | +| **x-cisco-email-msgevent**:web_interaction_tracking_mailflow_direction | webInteractionTrackingMailflowDirection | +| **x-cisco-email-msgevent**:mail_policy | mailPolicyName | +| **x-cisco-email-msgevent**:mail_policy_direction | mailPolicyDirection | +|
| | +### Supported STIX Objects and Properties for Query Results +| STIX Object | STIX Property | Data Source Field | +|--|--|--| +| email-addr | value | envelopeRecipientfilterValue | +| email-addr | value | envelopeSenderfilterValue | +|
| | | +| email-message | from_ref | envelopeSenderfilterValue | +| email-message | sender_ref | envelopeSenderfilterValue | +| email-message | to_refs | envelopeRecipientfilterValue | +| email-message | subject | subjectfilterValue | +| email-message | x_message_id_header | messageIdHeader | +| email-message | x_cisco_mid | ciscoMid | +| email-message | x_sender_ip_ref | senderIp | +|
| | | +| file | name | attachmentNameValue | +| file | hashes.'SHA-256' | fileSha256 | +|
| | | +| ipv4-addr | value | senderIp | +|
| | | +| ipv6-addr | value | senderIp | +|
| | | +| domain-name | value | domainNameValue | +|
| | | +| x-oca-host | hostname | ciscoHost | +|
| | | +| x-cisco-email-msgevent | advanced_malware_protection_mailflow_direction | advancedMalwareProtectionMailflowDirection | +| x-cisco-email-msgevent | advanced_malware_protection | advancedMalwareProtection | +| x-cisco-email-msgevent | app_forwarding | appForwarding | +| x-cisco-email-msgevent | content_filters_name | contentFiltersName | +| x-cisco-email-msgevent | content_filters_direction | contentFiltersDirection | +| x-cisco-email-msgevent | content_filters_action | contentFiltersAction | +| x-cisco-email-msgevent | dane_failure | daneFailure | +| x-cisco-email-msgevent | message_status | deliveryStatus | +| x-cisco-email-msgevent | message_delivered | message_delivered | +| x-cisco-email-msgevent | dlp_violations_names | dlpViolationsNames | +| x-cisco-email-msgevent | dlpViolationsSeverities | dlpViolationsSeverities | +| x-cisco-email-msgevent | dlp_action | dlpAction | +| x-cisco-email-msgevent | dmarc_from | dmarcFrom | +| x-cisco-email-msgevent | dmarc_action | dmarcAction | +| x-cisco-email-msgevent | etf_sources | etfSources | +| x-cisco-email-msgevent | etf_iocs | etfIocs | +| x-cisco-email-msgevent | forged_email_detection | forgedEmailDetection | +| x-cisco-email-msgevent | geo_location | geoLocation | +| x-cisco-email-msgevent | graymail | graymail | +| x-cisco-email-msgevent | hard_bounced | hardBounced | +| x-cisco-email-msgevent | ip_reputation | ipReputation | +| x-cisco-email-msgevent | macro_mailflow_direction | macroMailflowDirection | +| x-cisco-email-msgevent | macro_file_types_detected | macroFileTypesDetected | +| x-cisco-email-msgevent | message_filters | messageFilters | +| x-cisco-email-msgevent | message_direction | messageDirection | +| x-cisco-email-msgevent | contained_malicious_urls | containedMaliciousUrls | +| x-cisco-email-msgevent | contained_neutral_urls | containedNeutralUrls | +| x-cisco-email-msgevent | outbreak_filters_url_rewritten_byof | outbreakFiltersUrlRewrittenByOf | +| x-cisco-email-msgevent | outbreak_filtersVofThreatCategory | outbreakFiltersVofThreatCategory | +| x-cisco-email-msgevent | in_outbreak_quarantine | inOutbreakQuarantine | +| x-cisco-email-msgevent | quarantined_to | quarantinedTo | +| x-cisco-email-msgevent | reply_to | replyToValue | +| x-cisco-email-msgevent | s_mime | smime | +| x-cisco-email-msgevent | domain_categories | domainCategories | +| x-cisco-email-msgevent | sdr_categories | sdrCategories | +| x-cisco-email-msgevent | sdr_threat_levels | sdrThreatLevels | +| x-cisco-email-msgevent | soft_bounced | softBounced | +| x-cisco-email-msgevent | spam_positive | spamPositive | +| x-cisco-email-msgevent | quarantined_as_spam | quarantinedAsSpam | +| x-cisco-email-msgevent | quarantine_status | quarantineStatus | +| x-cisco-email-msgevent | threat_name | threatName | +| x-cisco-email-msgevent | suspect_spam | suspectSpam | +| x-cisco-email-msgevent | url_categories | urlCategories | +| x-cisco-email-msgevent | url_reputation | urlReputation | +| x-cisco-email-msgevent | safeprint_ext | safeprintExt | +| x-cisco-email-msgevent | virus_positive | virusPositive | +| x-cisco-email-msgevent | web_interaction_tracking_urls | webInteractionTrackingUrls | +| x-cisco-email-msgevent | web_interaction_tracking_mailflow_direction | webInteractionTrackingMailflowDirection | +| x-cisco-email-msgevent | mail_policy | mailPolicyName | +| x-cisco-email-msgevent | mail_policy_direction | mailPolicyDirection | +|
| | | diff --git a/stix_shifter_modules/cisco_secure_email/configuration/config.json b/stix_shifter_modules/cisco_secure_email/configuration/config.json new file mode 100644 index 000000000..014ddfb04 --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/configuration/config.json @@ -0,0 +1,37 @@ +{ + "connection": { + "type": { + "displayName": "Cisco Secure Email", + "group": "cisco" + }, + "host": { + "type": "text", + "regex": "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\\-]*[a-zA-Z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\\-]*[A-Za-z0-9])$" + }, + "port": { + "type": "number", + "default": 443, + "min": 1, + "max": 65535 + }, + "help": { + "type": "link", + "default": "data-sources.html" + }, + "selfSignedCert": { + "type": "password", + "optional": true + } + }, + "configuration": { + "auth": { + "type" : "fields", + "username": { + "type": "text" + }, + "password": { + "type": "password" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/cisco_secure_email/configuration/lang_en.json b/stix_shifter_modules/cisco_secure_email/configuration/lang_en.json new file mode 100644 index 000000000..cc6c3f156 --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/configuration/lang_en.json @@ -0,0 +1,32 @@ +{ + "connection": { + "host": { + "label": "Management IP address or hostname", + "description": "Specify the IP address or hostname of the data source" + }, + "port": { + "label": "Host port", + "description": "Set the port number that is associated with the hostname or IP address" + }, + "help": { + "label": "Need additional help?", + "description": "More details on the data source setting can be found in the specified link" + }, + "selfSignedCert": { + "label": "Cisco Secure Email HTTPS connection certificate", + "description": "Use self-signed Security Sockets Layer (SSL) or CA certificate for HTTPS services" + } + }, + "configuration": { + "auth": { + "username": { + "label": "Username", + "description": "Username with access to the tracking API" + }, + "password": { + "label": "Password", + "description": "Password of the user with access to the tracking API" + } + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/cisco_secure_email/entry_point.py b/stix_shifter_modules/cisco_secure_email/entry_point.py new file mode 100644 index 000000000..607486410 --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/entry_point.py @@ -0,0 +1,11 @@ +from stix_shifter_utils.utils.base_entry_point import BaseEntryPoint + + +class EntryPoint(BaseEntryPoint): + + def __init__(self, connection={}, configuration={}, options={}): + super().__init__(connection, configuration, options) + self.set_async(False) + if connection: + self.setup_transmission_basic(connection, configuration) + self.setup_translation_simple(dialect_default='default') diff --git a/stix_shifter_modules/cisco_secure_email/stix_translation/__init__.py b/stix_shifter_modules/cisco_secure_email/stix_translation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/cisco_secure_email/stix_translation/json/config_map.json b/stix_shifter_modules/cisco_secure_email/stix_translation/json/config_map.json new file mode 100644 index 000000000..402d0c111 --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/stix_translation/json/config_map.json @@ -0,0 +1,228 @@ +{ + "like_supported_fields": [ + "envelopeRecipientfilterValue", + "envelopeSenderfilterValue", + "subjectfilterValue", + "domainNameValue", + "replyToValue", + "attachmentNameValue" + ], + "in_supported_fields": [ + "contentFiltersDirection", + "quarantinedTo", + "urlCategories", + "advancedMalwareProtectionMailflowDirection", + "advancedMalwareProtection", + "macroMailflowDirection", + "macroFileTypesDetected", + "webInteractionTrackingMailflowDirection", + "mailPolicyDirection" + ], + "bool_supported_fields": [ + "message_delivered", + "daneFailure", + "graymail", + "hardBounced", + "containedMaliciousUrls", + "containedNeutralUrls", + "inOutbreakQuarantine", + "softBounced", + "spamPositive", + "quarantinedAsSpam", + "suspectSpam", + "safeprintExt", + "virusPositive" + ], + "int_supported_fields": [ + "ciscoMid", + "ipReputation", + "sdrCategories", + "sdrThreatLevels" + ], + "enum_supported_fields": [ + "deliveryStatus", + "appForwarding", + "dlpViolationsSeverities", + "ciscoHost", + "appForwarding", + "dlpAction", + "dmarcAction", + "etfIocs", + "smime", + "quarantineStatus", + "contentFiltersDirection", + "contentFiltersAction", + "advancedMalwareProtectionMailflowDirection", + "advancedMalwareProtection", + "macroMailflowDirection", + "webInteractionTrackingMailflowDirection", + "mailPolicyDirection" + ], + "enum_supported_values": { + "deliveryStatus": [ + "DELIVERED", + "DROPPED", + "ABORTED", + "BOUNCED" + ], + "appForwarding": [ + "app_success", + "app_failed" + ], + "dlpViolationsSeverities": [ + "critical", + "high", + "medium", + "low" + ], + "ciscoHost": [ + "All_Hosts" + ], + "dlpAction": [ + "delivered", + "encrypted", + "dropped" + ], + "dmarcAction": [ + "none", + "quarantine", + "reject", + "passed", + "failed" + ], + "etfIocs": [ + "file_hash", + "url", + "domain" + ], + "smime": [ + "smime_successful", + "smime_failed" + ], + "quarantineStatus": [ + "POLICY", + "AMP", + "AV", + "UNCLASSIFIED", + "DLP", + "OUTBREAK" + ], + "contentFiltersDirection": [ + "inbound", + "outbound" + ], + "contentFiltersAction": [ + "stopped" + ], + "advancedMalwareProtectionMailflowDirection": [ + "incoming", + "outgoing" + ], + "advancedMalwareProtection": [ + "amp_clean", + "amp_malicious", + "amp_unknown", + "amp_unscannable", + "amp_lowrisk" + ], + "macroMailflowDirection": [ + "inbound", + "outbound" + ], + "webInteractionTrackingMailflowDirection": [ + "inbound", + "outbound" + ], + "mailPolicyDirection": [ + "inbound", + "outbound" + ] + }, + "or_supported_values": [ + "advancedMalwareProtectionMailflowDirection", + "advancedMalwareProtection", + "appForwarding", + "contentFiltersName", + "contentFiltersDirection", + "contentFiltersAction", + "daneFailure", + "deliveryStatus", + "message_delivered", + "dlpViolationsNames", + "dlp_violations_severities", + "dlpAction", + "dmarcFrom", + "dmarcAction", + "etfSources", + "etfIocs", + "forgedEmailDetection", + "geoLocation", + "graymail", + "hardBounced", + "ipReputation", + "macroMailflowDirection", + "macroFileTypesDetected", + "messageFilters", + "containedMaliciousUrls", + "containedNeutralUrls", + "outbreakFiltersUrlRewrittenByOf", + "outbreakFiltersVofThreatCategory", + "inOutbreakQuarantine", + "quarantinedTo", + "smime", + "sdrCategories", + "sdrReputation", + "softBounced", + "spamPositive", + "quarantinedAsSpam", + "quarantineStatus", + "threatName", + "suspectSpam", + "urlCategories", + "urlReputation", + "safeprintExt", + "virusPositive", + "webInteractionTrackingUrls", + "webInteractionTrackingMailflowDirection", + "mailPolicyName", + "mailPolicyDirection" + ], + "grouped_fields": { + "contentFiltersName": [ + "contentFiltersDirection", + "contentFiltersAction" + ], + "contentFiltersDirection": [ + "contentFiltersName", + "contentFiltersAction" + ], + "contentFiltersAction": [ + "contentFiltersDirection", + "contentFiltersName" + ], + "advancedMalwareProtectionMailflowDirection": [ + "advancedMalwareProtection" + ], + "advancedMalwareProtection": [ + "advancedMalwareProtectionMailflowDirection" + ], + "macroMailflowDirection": [ + "macroFileTypesDetected" + ], + "macroFileTypesDetected": [ + "macroMailflowDirection" + ], + "webInteractionTrackingUrls": [ + "webInteractionTrackingMailflowDirection" + ], + "webInteractionTrackingMailflowDirection": [ + "webInteractionTrackingUrls" + ], + "mailPolicyDirection": [ + "mailPolicyName" + ], + "mailPolicyName": [ + "mailPolicyDirection" + ] + } +} \ No newline at end of file diff --git a/stix_shifter_modules/cisco_secure_email/stix_translation/json/from_stix_map.json b/stix_shifter_modules/cisco_secure_email/stix_translation/json/from_stix_map.json new file mode 100644 index 000000000..beec44d2d --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/stix_translation/json/from_stix_map.json @@ -0,0 +1,221 @@ +{ + "email-addr": { + "fields": { + "value": [ + "envelopeRecipientfilterValue", + "envelopeSenderfilterValue" + ] + } + }, + "email-message": { + "fields": { + "from_ref": [ + "envelopeSenderfilterValue" + ], + "sender_ref": [ + "envelopeSenderfilterValue" + ], + "to_refs": [ + "envelopeRecipientfilterValue" + ], + "subject": [ + "subjectfilterValue" + ], + "x_message_id_header": [ + "messageIdHeader" + ], + "x_cisco_mid": [ + "ciscoMid" + ], + "x_sender_ip_ref": [ + "senderIp" + ] + } + }, + "file": { + "fields": { + "name": [ + "attachmentNameValue" + ], + "hashes.'SHA-256'": [ + "fileSha256" + ] + } + }, + "ipv4-addr": { + "fields": { + "value": [ + "senderIp" + ] + } + }, + "ipv6-addr": { + "fields": { + "value": [ + "senderIp" + ] + } + }, + "domain-name": { + "fields": { + "value": [ + "domainNameValue" + ] + } + }, + "x-oca-host": { + "fields": { + "hostname": [ + "ciscoHost" + ] + } + }, + "x-cisco-email-msgevent": { + "fields": { + "advanced_malware_protection_mailflow_direction": [ + "advancedMalwareProtectionMailflowDirection" + ], + "advanced_malware_protection": [ + "advancedMalwareProtection" + ], + "app_forwarding": [ + "appForwarding" + ], + "content_filters_name": [ + "contentFiltersName" + ], + "content_filters_direction": [ + "contentFiltersDirection" + ], + "content_filters_action": [ + "contentFiltersAction" + ], + "dane_failure": [ + "daneFailure" + ], + "message_status": [ + "deliveryStatus" + ], + "message_delivered": [ + "message_delivered" + ], + "dlp_violations_names": [ + "dlpViolationsNames" + ], + "dlpViolationsSeverities": [ + "dlpViolationsSeverities" + ], + "dlp_action": [ + "dlpAction" + ], + "dmarc_from": [ + "dmarcFrom" + ], + "dmarc_action": [ + "dmarcAction" + ], + "etf_sources": [ + "etfSources" + ], + "etf_iocs": [ + "etfIocs" + ], + "forged_email_detection": [ + "forgedEmailDetection" + ], + "geo_location": [ + "geoLocation" + ], + "graymail": [ + "graymail" + ], + "hard_bounced": [ + "hardBounced" + ], + "ip_reputation": [ + "ipReputation" + ], + "macro_mailflow_direction": [ + "macroMailflowDirection" + ], + "macro_file_types_detected": [ + "macroFileTypesDetected" + ], + "message_filters": [ + "messageFilters" + ], + "contained_malicious_urls": [ + "containedMaliciousUrls" + ], + "contained_neutral_urls": [ + "containedNeutralUrls" + ], + "outbreak_filters_url_rewritten_byof": [ + "outbreakFiltersUrlRewrittenByOf" + ], + "outbreak_filtersVofThreatCategory": [ + "outbreakFiltersVofThreatCategory" + ], + "in_outbreak_quarantine": [ + "inOutbreakQuarantine" + ], + "quarantined_to": [ + "quarantinedTo" + ], + "reply_to": [ + "replyToValue" + ], + "s_mime": [ + "smime" + ], + "sdr_categories": [ + "sdrCategories" + ], + "sdr_threat_levels": [ + "sdrThreatLevels" + ], + "soft_bounced": [ + "softBounced" + ], + "spam_positive": [ + "spamPositive" + ], + "quarantined_as_spam": [ + "quarantinedAsSpam" + ], + "quarantine_status": [ + "quarantineStatus" + ], + "threat_name": [ + "threatName" + ], + "suspect_spam": [ + "suspectSpam" + ], + "url_categories": [ + "urlCategories" + ], + "url_reputation": [ + "urlReputation" + ], + "safeprint_ext": [ + "safeprintExt" + ], + "virus_positive": [ + "virusPositive" + ], + "web_interaction_tracking_urls": [ + "webInteractionTrackingUrls" + ], + "web_interaction_tracking_mailflow_direction": [ + "webInteractionTrackingMailflowDirection" + ], + "mail_policy": [ + "mailPolicyName" + ], + "mail_policy_direction": [ + "mailPolicyDirection" + ] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/cisco_secure_email/stix_translation/json/operators.json b/stix_shifter_modules/cisco_secure_email/stix_translation/json/operators.json new file mode 100644 index 000000000..200972177 --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/stix_translation/json/operators.json @@ -0,0 +1,9 @@ +{ + "ComparisonExpressionOperators.And": "&", + "ComparisonExpressionOperators.Or": " ", + "ComparisonComparators.Equal": "=", + "ComparisonComparators.In": "=", + "ComparisonComparators.Like": "=", + "ObservationOperators.Or": "", + "ObservationOperators.And": "" +} \ No newline at end of file diff --git a/stix_shifter_modules/cisco_secure_email/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/cisco_secure_email/stix_translation/json/stix_2_1/from_stix_map.json new file mode 100644 index 000000000..beec44d2d --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/stix_translation/json/stix_2_1/from_stix_map.json @@ -0,0 +1,221 @@ +{ + "email-addr": { + "fields": { + "value": [ + "envelopeRecipientfilterValue", + "envelopeSenderfilterValue" + ] + } + }, + "email-message": { + "fields": { + "from_ref": [ + "envelopeSenderfilterValue" + ], + "sender_ref": [ + "envelopeSenderfilterValue" + ], + "to_refs": [ + "envelopeRecipientfilterValue" + ], + "subject": [ + "subjectfilterValue" + ], + "x_message_id_header": [ + "messageIdHeader" + ], + "x_cisco_mid": [ + "ciscoMid" + ], + "x_sender_ip_ref": [ + "senderIp" + ] + } + }, + "file": { + "fields": { + "name": [ + "attachmentNameValue" + ], + "hashes.'SHA-256'": [ + "fileSha256" + ] + } + }, + "ipv4-addr": { + "fields": { + "value": [ + "senderIp" + ] + } + }, + "ipv6-addr": { + "fields": { + "value": [ + "senderIp" + ] + } + }, + "domain-name": { + "fields": { + "value": [ + "domainNameValue" + ] + } + }, + "x-oca-host": { + "fields": { + "hostname": [ + "ciscoHost" + ] + } + }, + "x-cisco-email-msgevent": { + "fields": { + "advanced_malware_protection_mailflow_direction": [ + "advancedMalwareProtectionMailflowDirection" + ], + "advanced_malware_protection": [ + "advancedMalwareProtection" + ], + "app_forwarding": [ + "appForwarding" + ], + "content_filters_name": [ + "contentFiltersName" + ], + "content_filters_direction": [ + "contentFiltersDirection" + ], + "content_filters_action": [ + "contentFiltersAction" + ], + "dane_failure": [ + "daneFailure" + ], + "message_status": [ + "deliveryStatus" + ], + "message_delivered": [ + "message_delivered" + ], + "dlp_violations_names": [ + "dlpViolationsNames" + ], + "dlpViolationsSeverities": [ + "dlpViolationsSeverities" + ], + "dlp_action": [ + "dlpAction" + ], + "dmarc_from": [ + "dmarcFrom" + ], + "dmarc_action": [ + "dmarcAction" + ], + "etf_sources": [ + "etfSources" + ], + "etf_iocs": [ + "etfIocs" + ], + "forged_email_detection": [ + "forgedEmailDetection" + ], + "geo_location": [ + "geoLocation" + ], + "graymail": [ + "graymail" + ], + "hard_bounced": [ + "hardBounced" + ], + "ip_reputation": [ + "ipReputation" + ], + "macro_mailflow_direction": [ + "macroMailflowDirection" + ], + "macro_file_types_detected": [ + "macroFileTypesDetected" + ], + "message_filters": [ + "messageFilters" + ], + "contained_malicious_urls": [ + "containedMaliciousUrls" + ], + "contained_neutral_urls": [ + "containedNeutralUrls" + ], + "outbreak_filters_url_rewritten_byof": [ + "outbreakFiltersUrlRewrittenByOf" + ], + "outbreak_filtersVofThreatCategory": [ + "outbreakFiltersVofThreatCategory" + ], + "in_outbreak_quarantine": [ + "inOutbreakQuarantine" + ], + "quarantined_to": [ + "quarantinedTo" + ], + "reply_to": [ + "replyToValue" + ], + "s_mime": [ + "smime" + ], + "sdr_categories": [ + "sdrCategories" + ], + "sdr_threat_levels": [ + "sdrThreatLevels" + ], + "soft_bounced": [ + "softBounced" + ], + "spam_positive": [ + "spamPositive" + ], + "quarantined_as_spam": [ + "quarantinedAsSpam" + ], + "quarantine_status": [ + "quarantineStatus" + ], + "threat_name": [ + "threatName" + ], + "suspect_spam": [ + "suspectSpam" + ], + "url_categories": [ + "urlCategories" + ], + "url_reputation": [ + "urlReputation" + ], + "safeprint_ext": [ + "safeprintExt" + ], + "virus_positive": [ + "virusPositive" + ], + "web_interaction_tracking_urls": [ + "webInteractionTrackingUrls" + ], + "web_interaction_tracking_mailflow_direction": [ + "webInteractionTrackingMailflowDirection" + ], + "mail_policy": [ + "mailPolicyName" + ], + "mail_policy_direction": [ + "mailPolicyDirection" + ] + } + } +} \ No newline at end of file diff --git a/stix_shifter_modules/cisco_secure_email/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/cisco_secure_email/stix_translation/json/stix_2_1/to_stix_map.json new file mode 100644 index 000000000..9d01f7247 --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/stix_translation/json/stix_2_1/to_stix_map.json @@ -0,0 +1,159 @@ +{ + "attributes": { + "hostName": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "email-message.x_cisco_host_ref", + "object": "email_message", + "references": "host" + } + ], + "timestamp": [ + { + "key": "email-message.date", + "object": "email_message", + "transformer": "FormatDateTimeObjectToTimestamp" + }, + { + "key": "first_observed", + "transformer": "FormatDateTimeObjectToTimestamp" + }, + { + "key": "last_observed", + "transformer": "FormatDateTimeObjectToTimestamp" + } + ], + "recipient": [ + { + "key": "email-addr.value", + "object": "recipient", + "unwrap": "true", + "transformer": "ValidateEmailTransformer" + }, + { + "key": "email-message.to_refs", + "object": "email_message", + "references": [ + "recipient" + ] + } + ], + "friendly_from": [ + { + "key": "email-addr.value", + "object": "from", + "unwrap": "true", + "transformer": "ValidateEmailTransformer" + }, + { + "key": "email-message.from_ref", + "object": "email_message", + "references": "from" + } + ], + "sender": [ + { + "key": "email-addr.value", + "object": "sender", + "transformer": "ValidateEmailTransformer" + }, + { + "key": "email-message.sender_ref", + "object": "email_message", + "references": "sender" + } + ], + "replyTo": [ + { + "key": "email-addr.value", + "object": "replyto", + "transformer": "ValidateEmailTransformer" + }, + { + "key": "x-cisco-email-msgevent.reply_to", + "object": "msgevent", + "references": "replyto" + } + ], + "subject": [ + { + "key": "email-message.subject", + "object": "email_message" + } + ], + "is_multipart": { + "key": "email-message.is_multipart", + "object": "email_message" + }, + "messageID": { + "key": "email-message.x_message_id_header", + "object": "email_message", + "transformer": "DictToValueTransformer" + }, + "senderIp": [ + { + "key": "ipv4-addr.value", + "object": "source_ip" + }, + { + "key": "ipv6-addr.value", + "object": "source_ip" + }, + { + "key": "email-message.x_sender_ip_ref", + "object": "email_message", + "references": "source_ip" + } + ], + "senderDomain": [ + { + "key": "domain-name.value", + "object": "domain_name" + }, + { + "key": "domain-name.resolves_to_refs", + "object": "domain_name", + "references": [ + "source_ip" + ] + } + ], + "senderGroup": { + "key": "email-message.x_sender_group", + "object": "email_message" + }, + "mid": { + "key": "email-message.x_cisco_mid", + "object": "email_message", + "transformer": "ListToIDTransformer" + }, + "icid": { + "key": "email-message.x_cisco_icid", + "object": "email_message" + }, + "serialNumber": { + "key": "email-message.x_serial_number", + "object": "email_message" + }, + "mailPolicy": { + "key": "x-cisco-email-msgevent.mail_policy", + "object": "msgevent" + }, + "direction": { + "key": "x-cisco-email-msgevent.direction", + "object": "msgevent" + }, + "messageStatus": { + "key": "x-cisco-email-msgevent.message_status", + "object": "msgevent", + "transformer": "DictToValueTransformer" + }, + "sbrs": { + "key": "x-cisco-email-msgevent.sbrs_score", + "object": "msgevent" + } + } +} diff --git a/stix_shifter_modules/cisco_secure_email/stix_translation/json/to_stix_map.json b/stix_shifter_modules/cisco_secure_email/stix_translation/json/to_stix_map.json new file mode 100644 index 000000000..9d01f7247 --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/stix_translation/json/to_stix_map.json @@ -0,0 +1,159 @@ +{ + "attributes": { + "hostName": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "email-message.x_cisco_host_ref", + "object": "email_message", + "references": "host" + } + ], + "timestamp": [ + { + "key": "email-message.date", + "object": "email_message", + "transformer": "FormatDateTimeObjectToTimestamp" + }, + { + "key": "first_observed", + "transformer": "FormatDateTimeObjectToTimestamp" + }, + { + "key": "last_observed", + "transformer": "FormatDateTimeObjectToTimestamp" + } + ], + "recipient": [ + { + "key": "email-addr.value", + "object": "recipient", + "unwrap": "true", + "transformer": "ValidateEmailTransformer" + }, + { + "key": "email-message.to_refs", + "object": "email_message", + "references": [ + "recipient" + ] + } + ], + "friendly_from": [ + { + "key": "email-addr.value", + "object": "from", + "unwrap": "true", + "transformer": "ValidateEmailTransformer" + }, + { + "key": "email-message.from_ref", + "object": "email_message", + "references": "from" + } + ], + "sender": [ + { + "key": "email-addr.value", + "object": "sender", + "transformer": "ValidateEmailTransformer" + }, + { + "key": "email-message.sender_ref", + "object": "email_message", + "references": "sender" + } + ], + "replyTo": [ + { + "key": "email-addr.value", + "object": "replyto", + "transformer": "ValidateEmailTransformer" + }, + { + "key": "x-cisco-email-msgevent.reply_to", + "object": "msgevent", + "references": "replyto" + } + ], + "subject": [ + { + "key": "email-message.subject", + "object": "email_message" + } + ], + "is_multipart": { + "key": "email-message.is_multipart", + "object": "email_message" + }, + "messageID": { + "key": "email-message.x_message_id_header", + "object": "email_message", + "transformer": "DictToValueTransformer" + }, + "senderIp": [ + { + "key": "ipv4-addr.value", + "object": "source_ip" + }, + { + "key": "ipv6-addr.value", + "object": "source_ip" + }, + { + "key": "email-message.x_sender_ip_ref", + "object": "email_message", + "references": "source_ip" + } + ], + "senderDomain": [ + { + "key": "domain-name.value", + "object": "domain_name" + }, + { + "key": "domain-name.resolves_to_refs", + "object": "domain_name", + "references": [ + "source_ip" + ] + } + ], + "senderGroup": { + "key": "email-message.x_sender_group", + "object": "email_message" + }, + "mid": { + "key": "email-message.x_cisco_mid", + "object": "email_message", + "transformer": "ListToIDTransformer" + }, + "icid": { + "key": "email-message.x_cisco_icid", + "object": "email_message" + }, + "serialNumber": { + "key": "email-message.x_serial_number", + "object": "email_message" + }, + "mailPolicy": { + "key": "x-cisco-email-msgevent.mail_policy", + "object": "msgevent" + }, + "direction": { + "key": "x-cisco-email-msgevent.direction", + "object": "msgevent" + }, + "messageStatus": { + "key": "x-cisco-email-msgevent.message_status", + "object": "msgevent", + "transformer": "DictToValueTransformer" + }, + "sbrs": { + "key": "x-cisco-email-msgevent.sbrs_score", + "object": "msgevent" + } + } +} diff --git a/stix_shifter_modules/cisco_secure_email/stix_translation/query_constructor.py b/stix_shifter_modules/cisco_secure_email/stix_translation/query_constructor.py new file mode 100644 index 000000000..62f83e2ad --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/stix_translation/query_constructor.py @@ -0,0 +1,470 @@ +import copy +import json +import re +from os import path +from datetime import datetime, timedelta +from stix_shifter_utils.utils import logger +from stix_shifter_utils.stix_translation.src.patterns.pattern_objects import ObservationExpression, \ + ComparisonExpression, ComparisonComparators, Pattern, CombinedComparisonExpression, CombinedObservationExpression, \ + ComparisonExpressionOperators + +logger = logger.set_logger(__name__) + +START_STOP_PATTERN = r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z)" +STOP_TIME = datetime.utcnow() +CONFIG_MAP_PATH = "json/config_map.json" + + +class FileNotFoundException(Exception): + pass + + +class QueryStringPatternTranslator: + """ + comparator values to match with supported data source operators + """ + + def __init__(self, pattern: Pattern, data_model_mapper, options): + + logger.info("Cisco Secure Email Connector") + self.dmm = data_model_mapper + self.comparator_lookup = self.dmm.map_comparator() + self.config_map = self.load_json(CONFIG_MAP_PATH) + self.pattern = pattern + self.options = options + self.qualified_queries = [] + self.parse_expression(pattern) + + @staticmethod + def load_json(rel_path_of_file): + """ Consumes a json file and returns a dictionary + :param rel_path_of_file: str + :return: dict """ + _json_path = path.dirname(path.realpath(__file__)) + "/" + rel_path_of_file + try: + if path.exists(_json_path): + with open(_json_path, encoding='utf-8') as f_obj: + return json.load(f_obj) + raise FileNotFoundException + except FileNotFoundException as e: + raise FileNotFoundError(f'{rel_path_of_file} not found') from e + + def _format_set(self, values, mapped_field_type, expression, mapped_fields_array) -> str: + """ + Formats value in the event of set operation + :param values: list + :param mapped_field_type: str + :param expression: object + :param mapped_fields_array: object + :return formatted value + """ + gen = values.element_iterator() + formatted_list = [] + for value in gen: + value = self._check_value_comparator_support(value, expression.comparator, mapped_field_type, + mapped_fields_array) + formatted_list.append(value) + formatted_values = ','.join(formatted_list) + return formatted_values + + @staticmethod + def _format_equality(value) -> str: + """ + Formats value in the event of equality operation + :param value + :return formatted value + """ + return str(value) + + @staticmethod + def _format_like(value) -> str: + """ + Formatting value in the event of like operation + :param value + :return: list + """ + return str(value) + + @staticmethod + def _format_datetime(value) -> str: + """ + format the date + :param: value: str + :return: converted_time: str + """ + try: + time_pattern = '%Y-%m-%dT%H:%M:%S.%fZ' + if re.search(r"\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}Z", str(value)): # without milli seconds + time_pattern = '%Y-%m-%dT%H:%M:%SZ' + converted_time = datetime.strptime(value, time_pattern).strftime('%Y-%m-%dT%H:%M:00.000Z') + return converted_time + except ValueError: + pass + raise NotImplementedError(f'cannot format the timestamp {value}') + + @staticmethod + def _parse_time_range(qualifier, time_range): + """ + Converts qualifier to timestamp format + :param qualifier: str + :param time_range: int + return: list of formatted timestamps + """ + try: + compile_timestamp_regex = re.compile(START_STOP_PATTERN) + if qualifier and compile_timestamp_regex.search(qualifier): + time_range_iterator = compile_timestamp_regex.finditer(qualifier) + time_range_list = [each.group() for each in time_range_iterator] + else: + start_time = STOP_TIME - timedelta(minutes=time_range) + converted_start_time = start_time.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + # limit 3 digit value for millisecond + converted_stop_time = STOP_TIME.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + time_range_list = [converted_start_time, converted_stop_time] + for index, value in enumerate(time_range_list): + time_range_list[index] = QueryStringPatternTranslator._format_datetime(value) + return time_range_list + except (KeyError, IndexError, TypeError) as e: + raise e + + def _get_mapped_field_type(self, mapped_field_array) -> str: + """ + Returns the type of mapped field array + :param mapped_field_array: list + :return: str + """ + mapped_field = mapped_field_array[0] + mapped_field_type = "string" + for key, value in self.config_map.items(): + if mapped_field in value and key in ["like_supported_fields", "bool_supported_fields", + "int_supported_fields", "enum_supported_fields"]: + mapped_field_type = key.split('_')[0] + break + return mapped_field_type + + def _check_value_comparator_support(self, value, comparator, mapped_field_type, mapped_fields_array) -> str: + """ + checks the comparator and value support + :param value + :param comparator + :param mapped_field_type: str + :param mapped_fields_array: list + :return value: str + """ + if mapped_field_type == "int" and not str(value).replace('-', '').isdigit(): + raise NotImplementedError(f"String type input {value} is not supported for integer type field") + + if mapped_field_type == "enum": + supported_values = self.config_map['enum_supported_values'].get(mapped_fields_array[0], []) + if value not in supported_values: + raise NotImplementedError(f'Unsupported ENUM values provided. {mapped_fields_array[0]} possible ' + f"supported enum values are '{','.join(supported_values)}'") + + if mapped_field_type == "bool": + if comparator != ComparisonComparators.Equal: + raise NotImplementedError('Boolean fields supports only for Equal operator') + if value.lower() == 'true': + value = True + else: + raise NotImplementedError('Boolean field supported value is only True') + + if comparator == ComparisonComparators.In and \ + mapped_fields_array[0] not in self.config_map['in_supported_fields']: + raise NotImplementedError("IN operator is not supported for this field") + + if comparator == ComparisonComparators.Like and mapped_field_type != "like": + raise NotImplementedError("LIKE operator is not supported for this field") + + return value + + def _lookup_comparison_operator(self, expression_operator) -> str: + """ + lookup operators support in cisco secure email + :param expression_operator:object + :return str + """ + if str(expression_operator) not in self.comparator_lookup: + raise NotImplementedError( + f'Comparison operator {expression_operator.name} unsupported for cisco secure email connector') + + return self.comparator_lookup[str(expression_operator)] + + def _eval_comparison_value(self, expression, mapped_field_type, mapped_fields_array) -> str: + """ + Function for parsing comparison expression value + :param expression: expression object + :param mapped_field_type: str + :param mapped_fields_array: list object + :return: formatted expression value + """ + if expression.negated or expression.comparator == ComparisonComparators.NotEqual: + raise NotImplementedError('Not operator is unsupported for cisco secure email') + + if expression.comparator == ComparisonComparators.Like: + value = self._check_value_comparator_support(expression.value, expression.comparator, mapped_field_type, + mapped_fields_array) + value = self._format_like(value) + elif expression.comparator == ComparisonComparators.In: + value = self._format_set(expression.value, mapped_field_type, expression, mapped_fields_array) + elif expression.comparator == ComparisonComparators.Equal: + value = self._check_value_comparator_support(expression.value, expression.comparator, mapped_field_type, + mapped_fields_array) + value = self._format_equality(value) + else: + raise NotImplementedError('Unknown comparator expression operator') + return value + + def _add_qualifier(self, query, qualifier) -> list: + """ + Convert the qualifier into epoch time and append in the query. + params: query : list + params: qualifier + return: query : list + """ + query_qualifier = [] + time_range = QueryStringPatternTranslator._parse_time_range(qualifier, self.options['time_range']) + for row in query: + query_qualifier.append(f"{row}&startDate={time_range[0]}&endDate={time_range[1]}") + return query_qualifier + + def _parse_mapped_fields(self, formatted_value, mapped_fields_array, mapped_field_type, expression) -> list: + """ + parse mapped fields into boolean expression + :param formatted_value: str + :param mapped_fields_array: list + :param mapped_field_type:str + :param expression: expression object + :return: list + """ + comparator = self._lookup_comparison_operator(expression.comparator) + comparison_list = [] + for field_name in mapped_fields_array: + if expression.comparator == ComparisonComparators.Like: + comparison_string_new = f'{field_name.replace("Value", "Operator")}{comparator}contains&' \ + f'{field_name}{comparator}{formatted_value}' + elif expression.comparator == ComparisonComparators.Equal and mapped_field_type == 'like': + comparison_string_new = f'{field_name.replace("Value", "Operator")}{comparator}is&' \ + f'{field_name}{comparator}{formatted_value}' + else: + comparison_string_new = f'{field_name}{comparator}{formatted_value}' + comparison_list.append(comparison_string_new) + return comparison_list + + def combine_or_queries(self, expression_01, expression_02) -> list: + """ + combine the queries using or operator + :param expression_01: expression object + :param expression_02: expression object + :return query list + """ + query_list = [] + if len(expression_01) == 1 and len(expression_02) == 1: + field_01, field_02 = None, None + if len(expression_01[0].split('=')) == 2 or len(expression_02[0].split('=')) == 2: + field_01 = expression_01[0].split('=')[0] + field_02 = expression_02[0].split('=')[0] + if field_01 in self.config_map['or_supported_values'] and \ + field_02 in self.config_map['or_supported_values']: + query_list.append(expression_01[0] + '&' + expression_02[0]) + # combining same date queries + elif 'startDate' in expression_01[0] and 'startDate' in expression_02[0]: + splitted_query_01 = expression_01[0].split('&startDate') + splitted_query_02 = expression_02[0].split('&startDate') + if splitted_query_01[-1] == splitted_query_02[-1]: + field_01 = splitted_query_01[0].split('=')[0] + field_02 = splitted_query_02[0].split('=')[0] + if field_01 in self.config_map['or_supported_values'] and \ + field_02 in self.config_map['or_supported_values']: + query_list.append(expression_01[0] + '&' + + expression_02[0].replace('&startDate'+splitted_query_02[-1], '')) + + # for or operator [query1 , query2] + if not query_list: + query_list = expression_01 + expression_02 + + return query_list + + def combine_and_queries(self, expression_01, expression_02, operator) -> list: + """ + combine the queries using and operator + :param expression_01: expression object + :param expression_02: expression object + :param operator: string + :return query list + """ + query_list = [] + + if len(expression_01) == 1 and len(expression_02) == 1: + if len(expression_01[0].split('=')) == 2 or len(expression_02[0].split('=')) == 2: + field_01 = expression_01[0].split('=')[0] + field_02 = expression_02[0].split('=')[0] + if field_01 in self.config_map['or_supported_values'] and \ + field_02 in self.config_map['or_supported_values']: + if not (field_01 in self.config_map['grouped_fields'].keys() and + field_02 in self.config_map['grouped_fields'].keys()): + raise NotImplementedError(f"AND operator is not supported for {field_01, field_02}" + f" message event fields") + + # for and operator [query1&query2] + for row_01 in expression_01: + for row_02 in expression_02: + query_list.append(f'{row_01}{operator}{row_02}') + + return query_list + + def _eval_combined_comparison_exp(self, expression) -> str: + """ + Function for parsing combined comparison expression + :param expression: expression object + """ + query = [] + operator = self._lookup_comparison_operator(expression.operator) + expression_01 = self._parse_expression(expression.expr1) + expression_02 = self._parse_expression(expression.expr2) + + if not expression_01 or not expression_02: + return query + + if expression.operator == ComparisonExpressionOperators.Or: + query = self.combine_or_queries(expression_01, expression_02) + elif expression.operator == ComparisonExpressionOperators.And: + query = self.combine_and_queries(expression_01, expression_02, operator) + return query + + def _eval_combined_observation_exp(self, expression, qualifier=None) -> str: + """ + Function for parsing combined observation expression + :param expression: expression object + :param qualifier: qualifier + """ + expression_01 = self._parse_expression(expression.expr1, qualifier) + expression_02 = self._parse_expression(expression.expr2, qualifier) + query = [] + if expression_01 and expression_02: + query = self.combine_or_queries(expression_01, expression_02) + elif expression_01: + query = expression_01 + elif expression_02: + query = expression_02 + return query + + def _parse_expression(self, expression, qualifier=None) -> list: + """ + Formation of cisco secure email query from ANTLR parsing expression + :param expression: expression object, ANTLR parsed expression object + :param qualifier: str, default in None + :return :None or list + """ + if isinstance(expression, ComparisonExpression): # Base Case + stix_objects = expression.object_path.split(':') + mapped_fields_array = self.dmm.map_field(stix_objects[0], stix_objects[1]) + mapped_field_type = self._get_mapped_field_type(mapped_fields_array) + value = self._eval_comparison_value(expression, mapped_field_type, mapped_fields_array) + query = self._parse_mapped_fields(value, mapped_fields_array, mapped_field_type, expression) + return query + + elif isinstance(expression, CombinedComparisonExpression): + return self._eval_combined_comparison_exp(expression) + + elif isinstance(expression, ObservationExpression): + query = self._parse_expression(expression.comparison_expression) + return self._add_qualifier(query, qualifier) + + elif hasattr(expression, 'qualifier') and hasattr(expression, 'observation_expression'): + if isinstance(expression.observation_expression, CombinedObservationExpression): + expression_01 = self._parse_expression(expression.observation_expression.expr1, expression.qualifier) + expression_02 = self._parse_expression(expression.observation_expression.expr2, expression.qualifier) + query = [] + if expression_01 and expression_02: + query = self.combine_or_queries(expression_01, expression_02) + else: + query = self._parse_expression(expression.observation_expression, expression.qualifier) + if qualifier is not None: + query = self._add_qualifier(query, qualifier) + return query + + elif isinstance(expression, CombinedObservationExpression): + return self._eval_combined_observation_exp(expression, qualifier) + elif isinstance(expression, Pattern): + return self._parse_expression(expression.expression) + else: + raise RuntimeError(f'Unknown Recursion Case for expression={expression}, ' + f'type(expression)={type(expression)}') + + def combine_in_supported_queries(self, query_list) -> list: + """ + Combine IN supported fields + params: query: list + return: query list + """ + copy_query_list = copy.deepcopy(query_list) + for query in copy_query_list: + for in_field in self.config_map['in_supported_fields']: + in_field += '=' + if query.count(in_field) > 1: + split_query = query.split('&') + copy_split_query = copy.deepcopy(split_query) + values = [] + for row in split_query: + if in_field in row: + values.append(row.replace(in_field, '')) + copy_split_query.remove(row) + if values: + copy_split_query.append(in_field + ','.join(values)) + query_list.remove(query) + query_list.append('&'.join(copy_split_query)) + return query_list + + @staticmethod + def validate_repeated_fields(query_list) -> list: + """ + Validate repeated fields in query + :param query_list: query list + """ + for row in query_list: + split_queries = row.split('&') + for statement in split_queries: + # get the field name + if '=' in statement: + field = statement.split('=')[0] + # duplicate fields cannot be allowed + if row.count(field + '=') > 1: + raise NotImplementedError(f"{field} cannot be allowed more than once in query") + else: + continue + return query_list + + def validate_dependant_fields(self, query): + """ + Validate any dependant fields are required + params: query : list + """ + for row in query: + for key, values in self.config_map['grouped_fields'].items(): + if key in row and [value for value in values if value + '=' not in row]: + raise NotImplementedError(f'Add dependant {",".join(values)} fields in pattern') + + def parse_expression(self, pattern: Pattern) -> list: + """ + Formation of cisco secure email query from ANTLR parsing expression. + :param pattern: expression object, ANTLR parsed expression object + """ + query_list = self._parse_expression(pattern) + query_list = self.combine_in_supported_queries(query_list) + self.validate_repeated_fields(query_list) + self.validate_dependant_fields(query_list) + + self.qualified_queries = query_list + + +def translate_pattern(pattern: Pattern, data_model_mapping, options) -> list: + """ + Conversion of ANTLR pattern to cisco secure email query + :param pattern: expression object, ANTLR parsed expression object + :param data_model_mapping: DataMapper object, mapping object obtained by parsing json + :param options: dict, time_range defaults to 5 + :return: list, cisco secure email queries + """ + translated_query_strings = QueryStringPatternTranslator(pattern, data_model_mapping, options) + queries = translated_query_strings.qualified_queries + return queries diff --git a/stix_shifter_modules/cisco_secure_email/stix_translation/query_translator.py b/stix_shifter_modules/cisco_secure_email/stix_translation/query_translator.py new file mode 100644 index 000000000..45ff86188 --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/stix_translation/query_translator.py @@ -0,0 +1,27 @@ +import logging + +from stix_shifter_utils.modules.base.stix_translation.base_query_translator import BaseQueryTranslator +from . import query_constructor + +logger = logging.getLogger(__name__) + + +class QueryTranslator(BaseQueryTranslator): + + def transform_antlr(self, data, antlr_parsing_object): + """ + Transforms STIX pattern into a different query format. Based on a mapping file + :param antlr_parsing_object: Antlr parsing objects for the STIX pattern + :type antlr_parsing_object: object + :param mapping: The mapping file path to use as instructions on how to transform the given STIX query into + another format. This should default to something if one isn't passed in + :type mapping: str (filepath) + :return: transformed query string + :rtype: str + """ + + logger.info("Converting STIX2 Pattern to data source query") + + query_string = query_constructor.translate_pattern( + antlr_parsing_object, self, self.options) + return query_string diff --git a/stix_shifter_modules/cisco_secure_email/stix_translation/transformers.py b/stix_shifter_modules/cisco_secure_email/stix_translation/transformers.py new file mode 100644 index 000000000..0c5c33437 --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/stix_translation/transformers.py @@ -0,0 +1,69 @@ +import re +from datetime import datetime +from stix_shifter_utils.utils import logger +from stix_shifter_utils.stix_translation.src.utils.transformers import ValueTransformer + + +LOGGER = logger.set_logger(__name__) + + +class FormatDateTimeObjectToTimestamp(ValueTransformer): + """A value transformer to convert local datetime object to UTC timestamp""" + + @staticmethod + def transform(obj): + try: + pattern = r'\d{2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} \(GMT \+\d{2}:\d{2}\)' + if re.match(pattern, str(obj)): + obj = datetime.strptime(obj, "%d %b %Y %H:%M:%S (%Z %z)").strftime('%Y-%m-%dT%H:%M:00.000Z') + return obj + except ValueError: + LOGGER.error("Cannot convert value to timestamp format") + + +class ListToIDTransformer(ValueTransformer): + """A value transformer to convert list to first index item""" + + @staticmethod + def transform(obj): + try: + if isinstance(obj, list) and obj: + val = obj[0] + return val + return int(obj) + except ValueError: + LOGGER.error(f"Cannot convert data value {obj} to ID value") + + +class DictToValueTransformer(ValueTransformer): + """A value transformer to convert dict to first item value""" + + @staticmethod + def transform(obj): + try: + if isinstance(obj, dict) and obj: + val = list(obj.values())[0] + return val + return obj + except ValueError: + LOGGER.error(f"Cannot convert data value {obj} to item value") + + +class ValidateEmailTransformer(ValueTransformer): + """ Validate email address format """ + + @staticmethod + def transform(obj): + try: + pattern = re.compile(r'(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)') + if isinstance(obj, list): + result = [] + for val in obj: + if pattern.match(str(val)): + result.append(val) + return result + elif pattern.match(str(obj)): + return obj + return None + except ValueError: + LOGGER.error(f"Cannot validate the email {obj}") diff --git a/stix_shifter_modules/cisco_secure_email/stix_transmission/__init__.py b/stix_shifter_modules/cisco_secure_email/stix_transmission/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/stix_shifter_modules/cisco_secure_email/stix_transmission/api_client.py b/stix_shifter_modules/cisco_secure_email/stix_transmission/api_client.py new file mode 100644 index 000000000..bd4f3919c --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/stix_transmission/api_client.py @@ -0,0 +1,55 @@ +import base64 +import json +from urllib import parse +from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync + + +class APIClient: + QUERY_ENDPOINT = "esa/api/v2.0/message-tracking/messages?searchOption=messages&" + PING_ENDPOINT = "esa/api/v2.0/login/privileges" + TOKEN_ENDPOINT = "esa/api/v2.0/login" + + def __init__(self, connection, configuration): + self.auth = configuration.get('auth') + self.headers = {'Content-Type': 'application/json'} + self.client = RestApiClientAsync(connection.get('host'), connection.get('port'), headers=self.headers, + cert_verify=connection.get('selfSignedCert', True)) + self.result_limit = connection['options'].get('result_limit') + self.timeout = connection['options'].get('timeout') + + async def ping_data_source(self, token): + """ + Ping the Data Source + :return: Response object + """ + self.headers['jwtToken'] = token + return await self.client.call_api(self.PING_ENDPOINT, 'GET', headers=self.headers, data={}, + timeout=self.timeout) + + async def get_search_results(self, query, token): + """ + Get results from Data Source + :param query: Data Source Query + :param token: Authentication token + :return: Response Object + """ + self.headers['jwtToken'] = token + query = self.QUERY_ENDPOINT + parse.quote(query, safe='&,=') + + return await self.client.call_api(query, 'GET', headers=self.headers, data={}, timeout=self.timeout) + + async def generate_token(self): + """Get Authorization token""" + username = f"{base64.b64encode(self.auth['username'].encode('utf-8')).decode('utf-8')}" + password = f"{base64.b64encode(self.auth['password'].encode('utf-8')).decode('utf-8')}" + data = { + "data": { + "userName": username, + "passphrase": password + } + } + if 'jwtToken' in self.headers: + self.headers.pop('jwtToken') + payload = json.dumps(data) + return await self.client.call_api(self.TOKEN_ENDPOINT, 'POST', headers=self.headers, data=payload, + timeout=self.timeout) diff --git a/stix_shifter_modules/cisco_secure_email/stix_transmission/connector.py b/stix_shifter_modules/cisco_secure_email/stix_transmission/connector.py new file mode 100644 index 000000000..f677c4c84 --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/stix_transmission/connector.py @@ -0,0 +1,175 @@ +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector +from stix_shifter_utils.utils.error_response import ErrorResponder +from stix_shifter_utils.utils import logger +from .api_client import APIClient +import json + + +class InvalidMetadataException(Exception): + pass + + +class Connector(BaseJsonSyncConnector): + EMAIL_MAX_PAGE_SIZE = 100 + + def __init__(self, connection, configuration): + self.api_client = APIClient(connection, configuration) + self.logger = logger.set_logger(__name__) + self.connector = __name__.split('.')[1] + + async def create_results_connection(self, query, offset, length, metadata=None): + """ + Fetching the results using query, offset and length + :param query: str, Data Source query + :param offset: str, Offset value + :param length: str, Length value + :param metadata: dict + :return: return_obj, dict + """ + return_obj = {} + data = [] + result_count = 0 + response_dict = {} + jwt_token = None + try: + if metadata: + if isinstance(metadata, dict) and metadata.get('jwtToken'): + jwt_token = metadata.get('jwtToken') + else: + # raise exception when metadata doesnt contain jwtToken token + raise InvalidMetadataException(metadata) + total_records = int(offset) + int(length) + if self.api_client.result_limit < total_records: + total_records = self.api_client.result_limit + if total_records <= Connector.EMAIL_MAX_PAGE_SIZE: + page_size = total_records + else: + page_size = Connector.EMAIL_MAX_PAGE_SIZE + if not jwt_token: + jwt_token = await self.get_token() + result_offset = int(offset) + while result_count < total_records: + limit = f'&limit={page_size}&offset={result_offset}' + response_wrapper = await self.api_client.get_search_results(query + limit, jwt_token) + response_dict = json.loads(response_wrapper.read().decode('utf-8')) + if response_wrapper.code == 200: + return_obj['success'] = True + data += response_dict['data'] + result_count += len(response_dict['data']) + remaining_records = total_records - result_count + if remaining_records > Connector.EMAIL_MAX_PAGE_SIZE: + page_size = Connector.EMAIL_MAX_PAGE_SIZE + else: + page_size = remaining_records + # Exit the loop if there are no more records to fetch + # if no more records to fetch API returns empty data + if len(response_dict['data']) == 0: + break + result_offset += len(response_dict['data']) + elif response_wrapper.code == 401 and response_dict['error']['message'] in ['ExpiredSignatureError.', + 'InvalidTokenError.']: + jwt_token = await self.get_token() + continue + else: + return_obj = self.exception_response(response_wrapper.code, response_dict['error']) + data = [] + break + if data: + data = self.add_is_multipart(data) + if metadata: + return_obj['data'] = data if data else [] + else: + return_obj['data'] = data[int(offset): total_records] if data else [] + if jwt_token: + return_obj['metadata'] = {"jwtToken": jwt_token} + else: + if not return_obj.get('error') and return_obj.get('success') is not False: + return_obj['success'] = True + return_obj['data'] = [] + + except InvalidMetadataException as ex: + response_dict['code'] = 100 + response_dict['message'] = f'Invalid metadata: {str(ex)}' + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + + except Exception as ex: + if "server timeout_error" in str(ex): + response_dict['code'] = 503 + elif "timeout_error" in str(ex): + response_dict['code'] = 408 + elif "X509" in str(ex): + response_dict['code'] = 101 + response_dict['message'] = "Invalid Self Signed Certificate: "+str(ex) + if not response_dict.get('message'): + response_dict['message'] = str(ex) + self.logger.error('error while fetching results: %s', ex) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + return return_obj + + async def ping_connection(self): + """ + Ping the endpoint + :return: return_object, dict + """ + return_obj = {} + response_dict = {} + try: + token = await self.get_token() + response = await self.api_client.ping_data_source(token) + response_code = response.code + response_dict = json.loads(response.read().decode('utf-8')) + if response_code == 200: + return_obj['success'] = True + else: + return_obj = self.exception_response(response_code, response_dict.get('errorSummary', '')) + + except Exception as ex: + if "server timeout_error" in str(ex): + response_dict['code'] = 503 + elif "timeout_error" in str(ex): + response_dict['code'] = 408 + elif "X509" in str(ex): + response_dict['code'] = 101 + response_dict['message'] = "Invalid Self Signed Certificate: "+str(ex) + if not response_dict.get('message'): + response_dict['message'] = str(ex) + self.logger.error('error while pinging: %s', ex) + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + return return_obj + + async def get_token(self): + """ Generate new token""" + response = await self.api_client.generate_token() + response_code = response.code + response_txt = response.read().decode('utf-8') + response_json = json.loads(response_txt) + if response_code == 200: + if 'data' in response_json and 'jwtToken' in response_json['data']: + token = response_json['data'].get('jwtToken') + return token + raise Exception(response_json['error']) + + def exception_response(self, code, response_txt): + """ + create the exception response + :param code, int + :param response_txt, dict + :return: return_obj, dict + """ + return_obj = {} + response_dict = {'code': code, 'message': str(response_txt)} + ErrorResponder.fill_error(return_obj, response_dict, ['message'], connector=self.connector) + return return_obj + + @staticmethod + def add_is_multipart(data): + """ + add is_multipart field + :param data, list + :return: data, list + """ + for row in data: + if row['attributes'].get('recipient') or row['attributes'].get('friendly_from') or \ + row['attributes'].get('sender') or row['attributes'].get('subject'): + row['attributes']['is_multipart'] = True + return data diff --git a/stix_shifter_modules/cisco_secure_email/stix_transmission/error_mapper.py b/stix_shifter_modules/cisco_secure_email/stix_transmission/error_mapper.py new file mode 100644 index 000000000..331c44232 --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/stix_transmission/error_mapper.py @@ -0,0 +1,40 @@ +from stix_shifter_utils.utils.error_mapper_base import ErrorMapperBase +from stix_shifter_utils.utils.error_response import ErrorCode +from stix_shifter_utils.utils import logger + +error_mapping = { + 100: ErrorCode.TRANSMISSION_INVALID_PARAMETER, + 101: ErrorCode.TRANSMISSION_CERT_ERROR, + 406: ErrorCode.TRANSMISSION_CONNECT, + 408: ErrorCode.TRANSMISSION_CONNECT, + 503: ErrorCode.TRANSMISSION_CONNECT, + 400: ErrorCode.TRANSMISSION_QUERY_PARSING_ERROR, + 401: ErrorCode.TRANSMISSION_AUTH_CREDENTIALS, + 403: ErrorCode.TRANSMISSION_FORBIDDEN, + 404: ErrorCode.TRANSMISSION_QUERY_PARSING_ERROR, + 410: ErrorCode.TRANSMISSION_REMOTE_SYSTEM_IS_UNAVAILABLE, + 500: ErrorCode.TRANSMISSION_REMOTE_SYSTEM_IS_UNAVAILABLE +} + + +class ErrorMapper: + logger = logger.set_logger(__name__) + DEFAULT_ERROR = ErrorCode.TRANSMISSION_MODULE_DEFAULT_ERROR + + @staticmethod + def set_error_code(json_data, return_obj, connector=None): + code = None + try: + code = int(json_data['code']) + except Exception: + pass + + error_code = ErrorMapper.DEFAULT_ERROR + + if code in error_mapping: + error_code = error_mapping.get(code) + + if error_code == ErrorMapper.DEFAULT_ERROR: + ErrorMapper.logger.error("failed to map: %s", str(json_data)) + + ErrorMapperBase.set_error_code(return_obj, error_code, connector=connector) diff --git a/stix_shifter_modules/cisco_secure_email/test/stix_translation/test_cisco_secure_email_json_to_stix.py b/stix_shifter_modules/cisco_secure_email/test/stix_translation/test_cisco_secure_email_json_to_stix.py new file mode 100644 index 000000000..923627f9f --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/test/stix_translation/test_cisco_secure_email_json_to_stix.py @@ -0,0 +1,157 @@ +""" test script to perform unit test case for cisco_secure_email translate results """ +import unittest +from stix_shifter_modules.cisco_secure_email.entry_point import EntryPoint +from stix_shifter_utils.stix_translation.src.json_to_stix import json_to_stix_translator +from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers + +MODULE = "cisco_secure_email" +entry_point = EntryPoint() +map_data = entry_point.get_results_translator().map_data +data_source = { + "type": "identity", + "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "name": "cisco_secure_email", + "identity_class": "events" +} +options = {} + +cisco_secure_email_sample_response = [ + { + "attributes": { + "hostName": "", + "friendly_from": [ + "testuser123@email.com" + ], + "isCompleteData": "N/A", + "messageStatus": { + "1637": "Delivered" + }, + "recipientMap": { + "1637": [ + "testuser13@email.com" + ] + }, + "senderIp": "11.111.11.11", + "mailPolicy": [ + "DEFAULT" + ], + "senderGroup": "UNKNOWNLIST", + "morInfo": { + "midVsState": { + "1637": "Delivered" + } + }, + "subject": "Test ReplyTo message", + "mid": [ + 1637 + ], + "senderDomain": "amazonses.com", + "finalSubject": { + "1637": "Test ReplyTo message" + }, + "direction": "incoming", + "icid": 1418, + "morDetails": {}, + "replyTo": "testuser123@email.com", + "timestamp": "04 Aug 2023 12:11:41 (GMT +00:00)", + "messageID": { + "1637": "<01000189c075eb9f-5fc5923c-5c03-4d1b-ad00-f75cc2c28fbf-000000@email.amazonses.com>" + }, + "verdictChart": { + "1637": "01141210" + }, + "recipient": [ + "testuser13@email.com" + ], + "sender": "01000189c075eb9f-5fc5923c-5c03-4d1b-ad00-f75cc2c28fbf-000000@amazonses.com", + "serialNumber": "EC2CD1D95C273722A23A-CA0C47E74D1B", + "allIcid": [ + 1418 + ], + "sbrs": "3.4", + "is_multipart": True + } + } +] + + +class TestCiscoSecureEmailResultsToStix(unittest.TestCase): + """ + class to perform unit test case for cisco_secure_email translate results + """ + + @staticmethod + def get_first(itr, constraint): + """ return the obj in the itr if constraint is true """ + return next((obj for obj in itr if constraint(obj)), None) + + @staticmethod + def get_first_of_type(itr, typ): + """ check whether the object belongs to respective stix object """ + return TestCiscoSecureEmailResultsToStix.get_first(itr, lambda o: isinstance(o, dict) and o.get('type') == typ) + + @staticmethod + def get_observed_data_objects(data): + result_bundle = json_to_stix_translator.convert_to_stix( + data_source, map_data, data, get_module_transformers(MODULE), options) + result_bundle_objects = result_bundle['objects'] + + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + + assert 'objects' in observed_data + return observed_data['objects'] + + def test_ipv4_addr_json_to_stix(self): + """test ipv4-addr stix object properties""" + objects = TestCiscoSecureEmailResultsToStix.get_observed_data_objects(cisco_secure_email_sample_response) + ipv4_obj = TestCiscoSecureEmailResultsToStix.get_first_of_type(objects.values(), 'ipv4-addr') + assert ipv4_obj is not None + assert ipv4_obj['type'] == 'ipv4-addr' + assert ipv4_obj['value'] == '11.111.11.11' + + def test_email_addr_json_to_stix(self): + """test email-addr stix object properties""" + objects = TestCiscoSecureEmailResultsToStix.get_observed_data_objects(cisco_secure_email_sample_response) + email_obj = TestCiscoSecureEmailResultsToStix.get_first_of_type(objects.values(), 'email-addr') + assert email_obj is not None + assert email_obj['type'] == 'email-addr' + assert email_obj['value'] == 'testuser123@email.com' + + def test_email_message_json_to_stix(self): + """test email-message stix object properties""" + objects = TestCiscoSecureEmailResultsToStix.get_observed_data_objects(cisco_secure_email_sample_response) + email_msg_obj = TestCiscoSecureEmailResultsToStix.get_first_of_type(objects.values(), 'email-message') + assert email_msg_obj is not None + assert (email_msg_obj.keys() == {'type', 'from_ref', 'is_multipart', 'x_sender_ip_ref', 'x_sender_group', 'subject', + 'x_cisco_mid', 'x_cisco_icid', 'date', 'x_message_id_header', 'to_refs', + 'sender_ref', 'x_serial_number'}) + assert email_msg_obj['type'] == 'email-message' + assert email_msg_obj['from_ref'] == '0' + assert email_msg_obj['is_multipart'] == True + assert email_msg_obj['subject'] == 'Test ReplyTo message' + + def test_cisco_email_msgevent_json_to_stix(self): + """test x-cisco-email-msgevent stix object properties""" + objects = TestCiscoSecureEmailResultsToStix.get_observed_data_objects(cisco_secure_email_sample_response) + cisco_email_obj = TestCiscoSecureEmailResultsToStix.get_first_of_type(objects.values(), + 'x-cisco-email-msgevent') + assert cisco_email_obj is not None + assert (cisco_email_obj.keys() == {'type', 'message_status', 'mail_policy', 'direction', 'reply_to', + 'sbrs_score'}) + assert cisco_email_obj['type'] == 'x-cisco-email-msgevent' + assert cisco_email_obj['message_status'] == 'Delivered' + assert cisco_email_obj['mail_policy'] == ['DEFAULT'] + assert cisco_email_obj['direction'] == 'incoming' + + + def test_domain_name_json_to_stix(self): + """test domain-name stix object properties""" + objects = TestCiscoSecureEmailResultsToStix.get_observed_data_objects(cisco_secure_email_sample_response) + domain_obj = TestCiscoSecureEmailResultsToStix.get_first_of_type(objects.values(), 'domain-name') + assert domain_obj is not None + assert (domain_obj.keys() == {'type', 'value', 'resolves_to_refs'}) + assert domain_obj['type'] == 'domain-name' + assert domain_obj['value'] == 'amazonses.com' + assert domain_obj['resolves_to_refs'] == ['3'] diff --git a/stix_shifter_modules/cisco_secure_email/test/stix_translation/test_cisco_secure_email_stix_to_query.py b/stix_shifter_modules/cisco_secure_email/test/stix_translation/test_cisco_secure_email_stix_to_query.py new file mode 100644 index 000000000..d99e84055 --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/test/stix_translation/test_cisco_secure_email_stix_to_query.py @@ -0,0 +1,295 @@ +from stix_shifter.stix_translation import stix_translation +import unittest +import re + +translation = stix_translation.StixTranslation() + + +def _remove_timestamp_from_query(queries): + pattern = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z' + if isinstance(queries, list): + return [re.sub(pattern, '', str(query)) for query in queries] + elif isinstance(queries, str): + return re.sub(pattern, '', queries) + + +class TestQueryTranslator(unittest.TestCase): + """ + class to perform unit test case cisco secure email translate query + """ + if __name__ == "__main__": + unittest.main() + + def _test_query_assertions(self, query, queries): + """ + to assert the each query in the list against expected result + """ + self.assertIsInstance(queries, list) + self.assertIsInstance(query, dict) + self.assertIsInstance(query['queries'], list) + for index, each_query in enumerate(query.get('queries'), start=0): + self.assertEqual(each_query, queries[index]) + + def test_equal_operator(self): + stix_pattern = "[ipv4-addr:value = '1.1.1.1'] START t'2023-02-15T16:43:00.000Z' STOP " \ + "t'2023-08-18T16:43:00.000Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['senderIp=1.1.1.1&startDate=2023-02-15T16:43:00.000Z&endDate=2023-08-18T16:43:00.000Z'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_IN_operator(self): + stix_pattern = "[x-cisco-email-msgevent:quarantined_to IN ('test','test2')] START t'2023-02-15T16:43:26.000Z' " \ + "STOP t'2023-08-18T16:43:26.003Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['quarantinedTo=test,test2&startDate=2023-02-15T16:43:00.000Z&endDate=2023-08-18T16:43:00.000Z'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_LIKE_operator(self): + stix_pattern = "[email-message:subject LIKE 'Reply'] START t'2023-02-15T16:43:26.000Z' STOP " \ + "t'2023-03-25T16:43:26.003Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['subjectfilterOperator=contains&subjectfilterValue=Reply&startDate=2023-02-15T16:43:00.000Z' + '&endDate=2023-03-25T16:43:00.000Z'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_bool_operator(self): + stix_pattern = "[x-cisco-email-msgevent:message_delivered = 'true'] START t'2023-02-15T16:43:26.000Z' STOP " \ + "t'2023-03-25T16:43:26.003Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['message_delivered=True&startDate=2023-02-15T16:43:00.000Z&endDate=2023-03-25T16:43:00.000Z'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_enum_operator(self): + stix_pattern = "[x-cisco-email-msgevent:message_status = 'DELIVERED'] START t'2023-02-15T16:43:26.000Z' " \ + "STOP t'2023-03-25T16:43:26.003Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['deliveryStatus=DELIVERED&startDate=2023-02-15T16:43:00.000Z&endDate=2023-03-25T16:43:00.000Z'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_file_attribute(self): + stix_pattern = "[file:name LIKE 'MicrosoftEdgeSetup'] START t'2023-02-15T16:43:26.000Z' STOP " \ + "t'2023-03-25T16:43:26.003Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['attachmentNameOperator=contains&attachmentNameValue=MicrosoftEdgeSetup&startDate=2023-02-15T16:43' + ':00.000Z&endDate=2023-03-25T16:43:00.000Z'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_file_hashes(self): + stix_pattern = "[file:hashes.'SHA-256' = '271c0119ac4455fc8db4ef4a8caf8e2bfcfb8bbd3b8c894e117a9ae9f743894b'] " \ + "START t'2023-02-15T16:43:26.000Z' STOP " \ + "t'2023-03-25T16:43:26.003Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['fileSha256=271c0119ac4455fc8db4ef4a8caf8e2bfcfb8bbd3b8c894e117a9ae9f743894b&startDate=2023-02' + '-15T16:43:00.000Z&endDate=2023-03-25T16:43:00.000Z'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_combined_comparison_AND_operator(self): + stix_pattern = "[ipv4-addr:value = '1.1.1.1' AND domain-name:value LIKE 'amazonses']START " \ + "t'2022-10-01T00:00:00.000Z' STOP t'2022-11-07T11:00:00.000Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['domainNameOperator=contains&domainNameValue=amazonses&senderIp=1.1.1.1&startDate=2022-10-01T00:00' + ':00.000Z&endDate=2022-11-07T11:00:00.000Z'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_comparison_OR_operator_for_basic_attributes(self): + stix_pattern = "[email-message:x_sender_ip_ref = '3.87.209.25' OR email-message:subject = 'important']START " \ + "t'2022-10-01T00:00:00.000Z' STOP t'2022-11-07T11:00:00.000Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['subjectfilterOperator=is&subjectfilterValue=important&startDate=2022-10-01T00:00:00.000Z&endDate' + '=2022-11-07T11:00:00.000Z', + 'senderIp=3.87.209.25&startDate=2022-10-01T00:00:00.000Z&endDate=2022-11-07T11:00:00.000Z'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_combined_comparison_OR_AND_operator_for_basic_attributes(self): + stix_pattern = "[(ipv4-addr:value = '1.1.1.1' OR email-message:from_ref = 'user1@.com') AND (file:name LIKE " \ + "'Microsoft' OR file:hashes.'SHA-256' = " \ + "'271c0119ac4455fc8db4ef4a8caf8e2bfcfb8bbd3b8c894e117a9ae9f743894b')]START " \ + "t'2023-07-19T01:56:00.000Z' STOP " \ + "t'2023-09-01T01:57:00.000Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['fileSha256=271c0119ac4455fc8db4ef4a8caf8e2bfcfb8bbd3b8c894e117a9ae9f743894b' + '&envelopeSenderfilterOperator=is&envelopeSenderfilterValue=user1@.com&startDate=2023-07-19T01:56' + ':00.000Z&endDate=2023-09-01T01:57:00.000Z', + 'fileSha256=271c0119ac4455fc8db4ef4a8caf8e2bfcfb8bbd3b8c894e117a9ae9f743894b&senderIp=1.1.1.1' + '&startDate=2023-07-19T01:56:00.000Z&endDate=2023-09-01T01:57:00.000Z', + 'attachmentNameOperator=contains&attachmentNameValue=Microsoft&envelopeSenderfilterOperator=is' + '&envelopeSenderfilterValue=user1@.com&startDate=2023-07-19T01:56:00.000Z&endDate=2023-09-01T01:57' + ':00.000Z', 'attachmentNameOperator=contains&attachmentNameValue=Microsoft&senderIp=1.1.1.1' + '&startDate=2023-07-19T01:56:00.000Z&endDate=2023-09-01T01:57:00.000Z'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_comparison_OR_operator_for_basic_and_event_attributes(self): + stix_pattern = "[ipv4-addr:value = '1.1.1.1' OR x-cisco-email-msgevent:spam_positive = 'true']" \ + "START t'2023-07-19T01:56:00.000Z' STOP t'2023-09-01T01:57:00.000Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['spamPositive=True&startDate=2023-07-19T01:56:00.000Z&endDate=2023-09-01T01:57:00.000Z', + 'senderIp=1.1.1.1&startDate=2023-07-19T01:56:00.000Z&endDate=2023-09-01T01:57:00.000Z'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_multiple_observation_AND_operator(self): + stix_pattern = "[ipv4-addr:value = '1.1.1.1'] AND [file:name LIKE 'Microsoft']START " \ + "t'2023-07-19T01:56:00.000Z' STOP t'2023-09-01T01:57:00.003Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['senderIp=1.1.1.1&startDate=2023-09-04T07:08:00.000Z&endDate=2023-09-04T07:13:00.000Z', + 'attachmentNameOperator=contains&attachmentNameValue=Microsoft&startDate=2023-07-19T01:56:00.000Z' + '&endDate=2023-09-01T01:57:00.000Z'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_multiple_observation_OR_operator(self): + stix_pattern = "[email-message:from_ref LIKE 'user1'] OR [email-message:x_sender_ip_ref = '3.87.209.25']START " \ + "t'2023-07-19T01:56:00.000Z' STOP t'2023-09-01T01:57:00.003Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['envelopeSenderfilterOperator=contains&envelopeSenderfilterValue=user1&startDate=2023-09-04T07:12' + ':00.000Z&endDate=2023-09-04T07:17:00.000Z', + 'senderIp=3.87.209.25&startDate=2023-07-19T01:56:00.000Z&endDate=2023-09-01T01:57:00.000Z'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_multiple_observation_with_combined_comparison(self): + stix_pattern = "[ipv4-addr:value = '1.1.1.1' AND file:hashes.'SHA-256' = " \ + "'271c0119ac4455fc8db4ef4a8caf8e2bfcfb8bbd3b8c894e117a9ae9f743894b' ] OR [" \ + "x-cisco-email-msgevent:message_status = 'DELIVERED']START t'2023-07-19T01:56:00.000Z' STOP " \ + "t'2023-09-01T01:57:00.003Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['fileSha256=271c0119ac4455fc8db4ef4a8caf8e2bfcfb8bbd3b8c894e117a9ae9f743894b&senderIp=1.1.1.1' + '&startDate=2023-09-08T05:53:00.000Z&endDate=2023-09-08T05:58:00.000Z', + 'deliveryStatus=DELIVERED&startDate=2023-07-19T01:56:00.000Z&endDate=2023-09-01T01:57:00.000Z'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_combine_multiple_observation_with_same_date(self): + stix_pattern = "([(x-cisco-email-msgevent:virus_positive = 'true')] OR [(x-cisco-email-msgevent:spam_positive " \ + "= 'true')]) START t'2023-07-19T01:56:00.000Z' STOP t'2023-09-01T01:57:00.003Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['virusPositive=True&startDate=2023-07-19T01:56:00.000Z&endDate=2023-09-01T01:57:00.000Z' + '&spamPositive=True'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_combine_IN_supported_fields(self): + stix_pattern = "[x-cisco-email-msgevent:quarantined_to = 'test1' OR x-cisco-email-msgevent:quarantined_to = " \ + "'test2'OR x-cisco-email-msgevent:quarantined_to = 'test3'] START t'2023-02-15T16:43:26.000Z' " \ + "STOP t'2023-08-18T16:43:26.003Z'" + query = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + queries = ['startDate=2023-02-15T16:43:00.000Z&endDate=2023-08-18T16:43:00.000Z&quarantinedTo=test3,test2,test1'] + queries = _remove_timestamp_from_query(queries) + self._test_query_assertions(query, queries) + + def test_invalid_boolean_operator(self): + stix_pattern = "[x-cisco-email-msgevent:message_delivered LIKE 'true'] START t'2023-02-15T16:43:26.000Z' STOP " \ + "t'2023-03-25T16:43:26.003Z'" + result = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'cisco_secure_email connector error => wrong parameter : Boolean fields supports ' \ + 'only for Equal operator' + + def test_invalid_boolean_value(self): + stix_pattern = "[x-cisco-email-msgevent:message_delivered = 'False'] START t'2023-02-15T16:43:26.000Z' STOP " \ + "t'2023-03-25T16:43:26.003Z'" + result = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'cisco_secure_email connector error => wrong parameter : Boolean field supported' \ + ' value is only True' + + def test_invalid_IN_operator(self): + stix_pattern = "[x-cisco-email-msgevent:ip_reputation IN ('1', '-1')] START t'2023-02-15T16:43:26.000Z' STOP " \ + "t'2023-03-25T16:43:26.003Z'" + result = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'cisco_secure_email connector error => wrong parameter : IN operator is not ' \ + 'supported for this field' + + def test_invalid_AND_operator(self): + stix_pattern = "[x-cisco-email-msgevent:message_delivered = 'true' AND x-cisco-email-msgevent:message_status " \ + "= 'DELIVERED']START t'2023-02-15T16:43:26.000Z' STOP t'2023-03-25T16:43:26.003Z'" + result = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == "cisco_secure_email connector error => wrong parameter : AND operator is not " \ + "supported for ('deliveryStatus', 'message_delivered') message event fields" + + def test_invalid_string_input(self): + stix_pattern = "[x-cisco-email-msgevent:ip_reputation = 'delivered'] START t'2023-02-15T16:43:26.000Z' STOP " \ + "t'2023-08-18T16:43:26.003Z'" + result = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'cisco_secure_email connector error => wrong parameter : String type input ' \ + 'delivered is not supported for integer type field' + + def test_invalid_enum_value(self): + stix_pattern = "[x-cisco-email-msgevent:message_status = 'DELIV'] START t'2023-02-15T16:43:26.000Z' STOP " \ + "t'2023-08-25T16:43:26.003Z'" + result = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == "cisco_secure_email connector error => wrong parameter : Unsupported ENUM values " \ + "provided. deliveryStatus possible supported enum values are 'DELIVERED,DROPPED," \ + "ABORTED,BOUNCED'" + + def test_invalid_LIKE_input(self): + stix_pattern = "[ipv4-addr:value LIKE '3.87.209.25'] START t'2023-02-15T16:43:00.000Z' STOP " \ + "t'2023-08-18T16:43:00.000Z'" + result = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'cisco_secure_email connector error => wrong parameter : LIKE operator is not ' \ + 'supported for this field' + + def test_invalid_timestamp(self): + stix_pattern = "[x-cisco-email-msgevent:ip_reputation = -1] START t'0000-03-01T11:00:00.003Z' STOP " \ + "t'2023-03-13T11:00:00.003Z'" + result = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'cisco_secure_email connector error => wrong parameter : cannot format the ' \ + 'timestamp 0000-03-01T11:00:00.003Z' + + def test_invalid_duplicate_value(self): + stix_pattern = "[x-cisco-email-msgevent:message_status = 'DELIVERED' OR x-cisco-email-msgevent:message_status " \ + "='DROPPED']START t'2023-07-19T01:56:00.000Z' STOP t'2023-09-01T01:57:00.003Z'" + result = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'cisco_secure_email connector error => wrong parameter : deliveryStatus cannot be ' \ + 'allowed more than once in query' + + def test_invalid_NOT_operator(self): + stix_pattern = "[ipv4-addr:value NOT LIKE '1.1.1.1'] START t'2023-07-19T01:56:00.000Z' STOP " \ + "t'2023-09-01T01:57:00.003Z'" + result = translation.translate('cisco_secure_email', 'query', '{}', stix_pattern) + assert False is result['success'] + assert 'not_implemented' == result['code'] + assert result['error'] == 'cisco_secure_email connector error => wrong parameter : Not operator is ' \ + 'unsupported for cisco secure email' diff --git a/stix_shifter_modules/cisco_secure_email/test/stix_transmission/test_cisco_secure_email.py b/stix_shifter_modules/cisco_secure_email/test/stix_transmission/test_cisco_secure_email.py new file mode 100644 index 000000000..8deb16ffe --- /dev/null +++ b/stix_shifter_modules/cisco_secure_email/test/stix_transmission/test_cisco_secure_email.py @@ -0,0 +1,351 @@ +from stix_shifter_modules.cisco_secure_email.entry_point import EntryPoint +import unittest +from unittest.mock import patch +from stix_shifter.stix_transmission import stix_transmission +import json +from stix_shifter.stix_transmission.stix_transmission import run_in_thread +from tests.utils.async_utils import get_mock_response + + +class CiscoEmailMockResponse: + """ class for Cisco Secure Email mock response""" + + def __init__(self, code, data, headers): + self.code = code + self.content = data + self.headers = headers + + def read(self): + return bytearray(self.content, 'utf-8') + + +class TestCiscoEmailConnection(unittest.TestCase, object): + mocked_ping_response = {"data": ["e_message_tracking_messages", + "e_message_tracking_detail", + "e_message_tracking_amp_details", + "e_message_tracking_connection_details"] + } + mock_token_response = {"data": {"userName": "u", + "is2FactorRedirectRequired": "false", + "role": "HelpDeskUser", + "jwtToken": "eyJhxxxxxxxxxxB7as" + } + } + mock_email_result = { + "meta": { + "num_bad_records": 0, + "totalCount": 1 + }, + "data": [ + { + "attributes": { + "hostName": "", + "mid": [ + 1619 + ], + "isCompleteData": "N/A", + "messageStatus": { + "1619": "Dropped By Anti-Virus" + }, + "recipientMap": { + "1619": [ + "user1@isc.com" + ] + }, + "senderIp": "1.1.1.1", + "mailPolicy": [ + "DEFAULT" + ], + "senderGroup": "UNKNOWNLIST", + "subject": "virus mail", + "friendly_from": [ + "user1@isc.com" + ], + "senderDomain": "isc.com", + "direction": "incoming", + "icid": 1350, + "morDetails": {}, + "replyTo": "N/A", + "timestamp": "31 Jul 2023 15:20:45 (GMT +00:00)", + "messageID": {}, + "verdictChart": { + "1619": "01500000" + }, + "recipient": [ + "user1@isc.com" + ], + "sender": "user2@isc.com", + "serialNumber": "EC2Cxxxx4D1B", + "allIcid": [ + 1350 + ], + "sbrs": "None" + } + } + ] + } + + mock_email_result2 = { + "meta": { + "num_bad_records": 0, + "totalCount": 1 + }, + "data": [ + { + "attributes": { + "hostName": "", + "mid": [ + 1620 + ], + "isCompleteData": "N/A", + "messageStatus": { + "1620": "Dropped By Anti-Virus" + }, + "recipientMap": { + "1620": [ + "user1@isc.com" + ] + }, + "senderIp": "1.1.1.1", + "mailPolicy": [ + "DEFAULT" + ], + "senderGroup": "UNKNOWNLIST", + "subject": "virus mail", + "friendly_from": [ + "user1@isc.com" + ], + "senderDomain": "isc.com", + "direction": "incoming", + "icid": 1351, + "morDetails": {}, + "replyTo": "N/A", + "timestamp": "31 Jul 2023 15:20:50 (GMT +00:00)", + "messageID": {}, + "verdictChart": { + "1620": "01500000" + }, + "recipient": [ + "user1@isc.com" + ], + "sender": "user2@isc.com", + "serialNumber": "EC2Cxxxx4D1C", + "allIcid": [ + 1351 + ], + "sbrs": "None" + } + } + ] + } + + def connection(self): + """format for connection""" + return { + "host": "hostbla", + "port": 443 + } + + def configuration(self): + """format for configuration""" + return { + "auth": { + "username": "u", + "password": "p" + } + } + + def test_is_async(self): + """check for synchronous or asynchronous""" + entry_point = EntryPoint(self.connection(), self.configuration()) + check_async = entry_point.is_async() + assert check_async is False + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_get_ping_results(self, mock_ping_response): + """test ping connection""" + mock_ping_response.side_effect = [ + get_mock_response(200, json.dumps(TestCiscoEmailConnection.mock_token_response), 'byte'), + get_mock_response(200, json.dumps(TestCiscoEmailConnection.mocked_ping_response), 'byte')] + entry_point = EntryPoint(self.connection(), self.configuration()) + ping_response = run_in_thread(entry_point.ping_connection) + assert ping_response is not None + assert ping_response['success'] is True + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_success_query_results(self, mock_result_response): + """ test success result response""" + query = "senderIP=1.1.1.1" + mock_result_response.side_effect = [ + get_mock_response(200, json.dumps(TestCiscoEmailConnection.mock_token_response), 'byte'), + get_mock_response(200, json.dumps(TestCiscoEmailConnection.mock_email_result), 'byte')] + transmission = stix_transmission.StixTransmission('cisco_secure_email', self.connection(), self.configuration()) + offset = 0 + length = 1 + result_response = transmission.results(query, offset, length) + assert result_response is not None + assert result_response['success'] is True + assert 'data' in result_response + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_success_query_results_pagination(self, mock_result_response): + """ test success result response""" + query = "senderIP=1.1.1.1" + mock_result_response.side_effect = [ + get_mock_response(200, json.dumps(TestCiscoEmailConnection.mock_token_response), 'byte'), + get_mock_response(200, json.dumps(TestCiscoEmailConnection.mock_email_result), 'byte'), + get_mock_response(200, json.dumps(TestCiscoEmailConnection.mock_email_result2), 'byte')] + transmission = stix_transmission.StixTransmission('cisco_secure_email', self.connection(), self.configuration()) + offset = 0 + length = 2 + result_response = transmission.results(query, offset, length) + assert result_response is not None + assert result_response['success'] is True + assert 'data' in result_response + assert result_response['data'][0] is not None + assert result_response['data'][1] is not None + assert 'metadata' in result_response + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_ping_invalid_host(self, mock_result_response): + """Test Invalid host for ping""" + mock_result_response.side_effect = Exception("client_connector_error") + transmission = stix_transmission.StixTransmission('cisco_secure_email', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert "client_connector_error" in ping_response['error'] + assert ping_response['code'] == "service_unavailable" + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_ping_invalid_auth(self, mock_results_response): + """Test invalid authentication for ping""" + error = json.dumps({"error": {"message": "Invalid username or passphrase.", + "code": "401", + "explanation": "401 = No permission -- see authorization schemes."}}) + mock_results_response.return_value = get_mock_response(401, error, 'byte') + transmission = stix_transmission.StixTransmission('cisco_secure_email', self.connection(), self.configuration()) + ping_response = transmission.ping() + assert ping_response is not None + assert ping_response['success'] is False + assert ping_response['code'] == "authentication_fail" + assert "Invalid username" in ping_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_results_invalid_auth(self, mock_results_response): + """Test invalid authentication for results""" + error = json.dumps({"error": {"message": "Invalid username or passphrase.", + "code": "401", + "explanation": "401 = No permission -- see authorization schemes."}}) + query = "senderIP=1.1.1.1" + mock_results_response.return_value = get_mock_response(401, error, 'byte') + transmission = stix_transmission.StixTransmission('cisco_secure_email', self.connection(), self.configuration()) + offset = 0 + length = 1 + result_response = transmission.results(query, offset, length) + assert result_response is not None + assert result_response['success'] is False + assert result_response['code'] == "authentication_fail" + assert 'Invalid username' in result_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_time_out_exception_for_results(self, mock_result_response): + """Test timeout exception for results""" + mock_result_response.side_effect = Exception("timeout_error") + query = "senderIP=1.1.1.1" + transmission = stix_transmission.StixTransmission('cisco_secure_email', self.connection(), self.configuration()) + offset = 0 + length = 1 + result_response = transmission.results(query, offset, length) + assert result_response is not None + assert result_response['success'] is False + assert 'error' in result_response + assert 'timeout_error' in result_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_server_time_out_exception_for_results(self, mock_result_response): + """Test timeout exception for results""" + mock_result_response.side_effect = Exception("server timeout_error (2 sec)") + query = "senderIP=1.1.1.1" + transmission = stix_transmission.StixTransmission('cisco_secure_email', self.connection(), self.configuration()) + offset = 0 + length = 1 + result_response = transmission.results(query, offset, length) + assert result_response is not None + assert result_response['success'] is False + assert 'error' in result_response + assert 'server timeout_error (2 sec)' in result_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_time_out_exception_for_ping(self, mock_ping_response): + """Test timeout exception for ping""" + mock_ping_response.side_effect = Exception("timeout_error") + transmission = stix_transmission.StixTransmission('cisco_secure_email', self.connection(), self.configuration()) + result_response = transmission.ping() + assert result_response is not None + assert result_response['success'] is False + assert 'error' in result_response + assert 'timeout_error' in result_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_success_query_with_metadata_parameter(self, mock_result_response): + """ test success result response with metadata parameter""" + metadata = {'jwtToken': 'abcdxxxxxxdefg'} + query = "senderIP=1.1.1.1" + mock_result_response.side_effect = [ + get_mock_response(200, json.dumps(TestCiscoEmailConnection.mock_email_result), 'byte')] + transmission = stix_transmission.StixTransmission('cisco_secure_email', self.connection(), self.configuration()) + offset = 0 + length = 1 + result_response = transmission.results(query, offset, length, metadata) + assert result_response is not None + assert result_response['success'] is True + assert 'data' in result_response + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_with_invalid_metadata_parameter(self, mock_result_response): + """ test invalid metadata parameter""" + metadata = {'next_page_token': '123a'} + query = "senderIP=1.1.1.1" + mock_result_response.side_effect = [ + get_mock_response(200, json.dumps(TestCiscoEmailConnection.mock_token_response), 'byte'), + get_mock_response(200, json.dumps(TestCiscoEmailConnection.mock_email_result), 'byte')] + transmission = stix_transmission.StixTransmission('cisco_secure_email', self.connection(), self.configuration()) + offset = 0 + length = 1 + result_response = transmission.results(query, offset, length, metadata) + assert result_response is not None + assert result_response['success'] is False + assert "Invalid metadata" in result_response['error'] + assert result_response['code'] == "invalid_parameter" + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_results_invalid_query(self, mock_results_response): + """Test invalid authentication for results""" + error = json.dumps({'error': {'message': 'Missing attribute - startDate or endDate.', 'code': '404', + 'explanation': '404 = Nothing matches the given URI.'}}) + query = "senderIP=1.1.1.1" + mock_results_response.side_effect = [ + get_mock_response(200, json.dumps(TestCiscoEmailConnection.mock_token_response), 'byte'), + get_mock_response(404, error, 'byte')] + transmission = stix_transmission.StixTransmission('cisco_secure_email', self.connection(), self.configuration()) + offset = 0 + length = 1 + result_response = transmission.results(query, offset, length) + assert result_response is not None + assert result_response['success'] is False + assert result_response['code'] == "invalid_query" + assert 'Nothing matches the given URI' in result_response['error'] + + @patch('stix_shifter_utils.stix_transmission.utils.RestApiClientAsync.RestApiClientAsync.call_api') + def test_invalid_certificate(self, mock_results_response): + """Test invalid certificate for results""" + mock_results_response.side_effect = Exception("[X509] PEM lib (_ssl.c:4293)") + transmission = stix_transmission.StixTransmission('cisco_secure_email', self.connection(), self.configuration()) + offset = 0 + length = 1 + result_response = transmission.results('', offset, length) + assert result_response is not None + assert result_response['success'] is False + assert result_response['code'] == "certificate_fail" + assert "Invalid Self Signed Certificate" in result_response['error'] From 8d327e8f7fe42835b07188febc8f95ca3421d2dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 15:49:11 -0300 Subject: [PATCH 10/13] Bump azure-identity from 1.12.0 to 1.14.1 in /stix_shifter (#1599) Bumps [azure-identity](https://github.com/Azure/azure-sdk-for-python) from 1.12.0 to 1.14.1. - [Release notes](https://github.com/Azure/azure-sdk-for-python/releases) - [Changelog](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/esrp_release.md) - [Commits](https://github.com/Azure/azure-sdk-for-python/compare/azure-identity_1.12.0...azure-identity_1.14.1) --- updated-dependencies: - dependency-name: azure-identity dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Md Azam --- stix_shifter/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter/requirements.txt b/stix_shifter/requirements.txt index e7586bbce..a37095ff0 100644 --- a/stix_shifter/requirements.txt +++ b/stix_shifter/requirements.txt @@ -5,7 +5,7 @@ antlr4-python3-runtime==4.8 asyncio==3.4.3 asynctest==0.13.0 attrs==23.1.0 -azure-identity==1.12.0 +azure-identity==1.14.1 colorlog==6.7.0 flask==2.3.3 flatten_json==0.1.13 From ba4d7d64b6ece2632b217613ab56776a11ab441f Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Thu, 19 Oct 2023 08:59:44 -0400 Subject: [PATCH 11/13] fix: case-insensitive bug for items in brackets for elastic_ecs --- .../stix_translation/query_constructor.py | 58 ++++++++++++++----- .../test_elastic_ecs_stix_to_query.py | 38 +++++++----- 2 files changed, 67 insertions(+), 29 deletions(-) diff --git a/stix_shifter_modules/elastic_ecs/stix_translation/query_constructor.py b/stix_shifter_modules/elastic_ecs/stix_translation/query_constructor.py index 658b7d2f2..d3dbdc3e9 100644 --- a/stix_shifter_modules/elastic_ecs/stix_translation/query_constructor.py +++ b/stix_shifter_modules/elastic_ecs/stix_translation/query_constructor.py @@ -335,15 +335,15 @@ def _unfold_case_insensitive_regex(regex_pattern): # xsb: xs inside bracket xsb = xs[1::2] - xsb[ci_index//2:] = ["[" + _unfold_ci_chars(x, True) + "]" for x in xsb[ci_index//2:]] + xsb[ci_index//2:] = [_unfold_ci_chars_in_bracket(x) for x in xsb[ci_index//2:]] xs[1::2] = xsb # xsob: xs outside bracket xsob = xs[0::2] xsob_s_h, *xsob_s_t = xsob[ci_index//2].split("(?i)") - xsob_s_t = [_unfold_ci_chars(x, False) for x in xsob_s_t] + xsob_s_t = [_unfold_plaintext_ci_chars(x) for x in xsob_s_t] xsob[ci_index//2] = "".join([xsob_s_h] + xsob_s_t) - xsob[ci_index//2+1:] = [_unfold_ci_chars(x.replace("(?i)", ""), False) for x in xsob[ci_index//2+1:]] + xsob[ci_index//2+1:] = [_unfold_plaintext_ci_chars(x.replace("(?i)", "")) for x in xsob[ci_index//2+1:]] xs[0::2] = xsob p_unfolded = "".join(xs) @@ -361,19 +361,47 @@ def _unfold_case_insensitive_regex(regex_pattern): return regex_pattern -def _unfold_ci_chars(regex_pattern_segment, if_set): - # if_set: if all chars are in the square bracket of regex - def char_mapper(c): - if if_set: - return c.lower() + c.upper() +def _unfold_plaintext_ci_chars(regex_pattern_segment): + return "".join([f"[{x.lower()}{x.upper()}]" if x.isascii() and x.isalpha() else x for x in regex_pattern_segment]) + + +def _unfold_ci_chars_in_bracket(regex_pattern_in_bracket): + # split segments + segs = [""] + # effective i that knows skipped indexes/chars + ie = 0 + for i, x in enumerate(regex_pattern_in_bracket): + if i < ie: + continue + if i < len(regex_pattern_in_bracket)-2: + if x.isascii(): + ahead1 = regex_pattern_in_bracket[i+1] + ahead2 = regex_pattern_in_bracket[i+2] + if ahead1 == "-" and ahead2.isascii(): + segs.append(regex_pattern_in_bracket[i:i+3]) + segs.append("") + ie = i+3 + else: + segs[-1] = segs[-1] + x + else: + segs[-1] = segs[-1] + x + else: + segs.append(regex_pattern_in_bracket[i:len(regex_pattern_in_bracket)]) + break + segs_new = [] + for seg in segs: + if len(seg) == 3 and seg[1] == "-" and seg[0].isascii() and seg[0].isalpha() and seg[2].isascii() and seg[2].isalpha(): + lower = f"{seg[0].lower()}-{seg[2].lower()}" + if lower not in segs_new: + segs_new.append(lower) + upper = f"{seg[0].upper()}-{seg[2].upper()}" + if upper not in segs_new: + segs_new.append(upper) else: - return f"[{c.lower()}{c.upper()}]" - xs = list(regex_pattern_segment) - s = "".join([char_mapper(x) if x.isascii() and x.isalpha() else x for x in xs]) - if if_set: - # dedup for items inside square bracket - s = "".join(sorted(set(s))) - return s + new = "".join([x.lower()+x.upper() if x.isascii() and x.isalpha() else x for x in seg]) + if new not in segs_new: + segs_new.append(new) + return "[" + "".join(segs_new) + "]" def translate_pattern(pattern: Pattern, data_model_mapping, options): diff --git a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_stix_to_query.py b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_stix_to_query.py index 9002631df..33b8aeb4e 100644 --- a/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_stix_to_query.py +++ b/stix_shifter_modules/elastic_ecs/tests/stix_translation/test_elastic_ecs_stix_to_query.py @@ -1,7 +1,8 @@ from stix_shifter.stix_translation import stix_translation from stix_shifter_utils.utils.error_response import ErrorCode from stix_shifter_modules.elastic_ecs.stix_translation.query_constructor import ( - _unfold_ci_chars, + _unfold_plaintext_ci_chars, + _unfold_ci_chars_in_bracket, _unfold_case_insensitive_regex, ) import unittest @@ -41,26 +42,35 @@ def _remove_timestamp_from_query(queries): class TestStixtoQuery(unittest.TestCase, object): def test_case_insensitive_unfold_chars(self): - input_output_pairs = [ ("a", False, "[aA]") - , ("ab", False, "[aA][bB]") - , ("ab7#*c((D))", False, "[aA][bB]7#*[cC](([dD]))") - , ("aba", True, "ABab") - , ("ab7#BBeE", True, "#7ABEabe") + input_output_pairs = [ ("a", "[aA]") + , ("ab", "[aA][bB]") + , ("ab7#*c((D))", "[aA][bB]7#*[cC](([dD]))") ] - for (x,y,z) in input_output_pairs: - assert z == _unfold_ci_chars(x, y) - + for (x,y) in input_output_pairs: + assert y == _unfold_plaintext_ci_chars(x) + + def test_unfold_ci_chars_in_bracket(self): + iopairs = [ ("abD", "[aAbBdD]") + , ("a-z0-9", "[a-zA-Z0-9]") + , ("-ef-z", "[-eEf-zF-Z]") + , ("ab-", "[aAbB-]") + , ("a-zA-Z0-9", "[a-zA-Z0-9]") + ] + for (x,y) in iopairs: + assert y == _unfold_ci_chars_in_bracket(x) def test_case_insensitive_unfold_regex(self): iopairs = [ ("http://z[abc]83m li", "http://z[abc]83m li") , ("(?i)virus", "[vV][iI][rR][uU][sS]") - , ("(?i)virus[ s]", "[vV][iI][rR][uU][sS][ Ss]") - , ("(?i)virus[ s] bin [c3b]", "[vV][iI][rR][uU][sS][ Ss] [bB][iI][nN] [3BCbc]") + , ("(?i)virus[ s]", "[vV][iI][rR][uU][sS][ sS]") + , ("(?i)virus[ s] bin [c3b]", "[vV][iI][rR][uU][sS][ sS] [bB][iI][nN] [cC3bB]") , (r"(?i)virus\[ s\]", r"[vV][iI][rR][uU][sS]\[ [sS]\]") - , (r"(?i)virus\\[ s\\]", r"[vV][iI][rR][uU][sS]\\[ Ss\\]") + , (r"(?i)virus\\[ s\\]", r"[vV][iI][rR][uU][sS]\\[ sS\\]") , ("(?i)http://z83m li", "[hH][tT][tT][pP]://[zZ]83[mM] [lL][iI]") - , ("(?i)http://z[abc]83m li", "[hH][tT][tT][pP]://[zZ][ABCabc]83[mM] [lL][iI]") - , ("http://(?i)z[abc]83m li", "http://[zZ][ABCabc]83[mM] [lL][iI]") + , ("(?i)http://z[abc]83m li", "[hH][tT][tT][pP]://[zZ][aAbBcC]83[mM] [lL][iI]") + , ("http://(?i)z[abc]83m li", "http://[zZ][aAbBcC]83[mM] [lL][iI]") + , ("http://(?i)z[a-z]83m li", "http://[zZ][a-zA-Z]83[mM] [lL][iI]") + , ("http://(?i)z[a-z0-9A-Z]83m li", "http://[zZ][a-zA-Z0-9]83[mM] [lL][iI]") ] for (x,y) in iopairs: assert y == _unfold_case_insensitive_regex(x) From b7b3ffbe7b99461ce8c1ace2ccd74da0684279c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Oct 2023 10:38:12 -0300 Subject: [PATCH 12/13] Bump flask from 2.3.3 to 3.0.0 in /stix_shifter (#1600) Bumps [flask](https://github.com/pallets/flask) from 2.3.3 to 3.0.0. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/2.3.3...3.0.0) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- stix_shifter/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter/requirements.txt b/stix_shifter/requirements.txt index a37095ff0..ff2169313 100644 --- a/stix_shifter/requirements.txt +++ b/stix_shifter/requirements.txt @@ -7,7 +7,7 @@ asynctest==0.13.0 attrs==23.1.0 azure-identity==1.14.1 colorlog==6.7.0 -flask==2.3.3 +flask==3.0.0 flatten_json==0.1.13 json-fix==0.5.2 jsonmerge==1.9.2 From d75520d4a623f117719232358d053a9a5e66a111 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Oct 2023 15:50:43 -0300 Subject: [PATCH 13/13] Bump aioboto3 from 11.3.0 to 11.3.1 in /stix_shifter (#1607) --- stix_shifter/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stix_shifter/requirements.txt b/stix_shifter/requirements.txt index ff2169313..547bec42e 100644 --- a/stix_shifter/requirements.txt +++ b/stix_shifter/requirements.txt @@ -1,4 +1,4 @@ -aioboto3==11.3.0 +aioboto3==11.3.1 aiohttp-retry==2.8.3 aiomysql==0.2.0 antlr4-python3-runtime==4.8