Resolve #11741, bbot update based on testing using BBOT v2.1.2

packages/bbot/_dev/build/docs/README.md

@@ -4,24 +4,68 @@ The Bighuge BLS OSINT Tool (BBOT) integration is intended for [BBOT](https://www
 
 Once the BBOT scan is complete, the integration will ingest the results into Elastic.
 
-This tool is used to enhance your external knowledge of your environment. This is done through the integration of many tools into BBOT providing a overview of your attack surface. Here is [how it works](https://www.blacklanternsecurity.com/bbot/Stable/how_it_works/).
+This integration requires the external use of BBOT. You will have to download and run the tool apart from this integration. Once your scan is complete, this integration can ingest the results into Elastic using the BBOT `json` output module or the `http` output module aka. HTTP POST/webhooks.
+
+As of version `0.3.0` of this integration both BBOT `1.x` and `2.x` are supported. However all examples provided by the module documentation now assume use of BBOT `2.x`.
+
+In order to support the conflicting BBOT NDJSON event formats the following occurs during ingest of BBOT `2.x` SCAN events:
+1. The `bbot.data.scan` field, if received as a Map aka. object, is renamed to `bbot.data.scan_config`
+2. `bbot.data.scan` is set using the ID and name of the scan in order to replicate the field type and content generated by BBOT `1.x`
+
+This tool is used to enhance your external knowledge of your environment. This is done through the integration of many tools into BBOT providing a overview of your attack surface. Here is [how it works](https://www.blacklanternsecurity.com/bbot/how_it_works/).
+
+**Important Note**
+
+You will have to provide the following parameter in your BBOT scan for your output.json to be formatted correctly.
+
+`-c modules.json.siem_friendly=true`
+
+Or if using the HTTP output modules,
+
+`-c modules.http.siem_friendly=true`
+
+We also strongly recommend ensuring you filter, at minimum, `HTTP_RESPONSE` and `RAW_TEXT` events from reaching the SIEM as these events can contain extreme field data which may not be consumable.
+
+Ensure the configuration option is left at the default or is similar to the following:
 
-**Important Note** - You will have to provide the following parameter in your BBOT scan for your output.ndjson to be formatted correctly.
 ```
--c output_modules.json.siem_friendly=true
+omit_event_types:
+- HTTP_RESPONSE
+- RAW_TEXT
 ```
+
 **Example BBOT Scan**
-```
-bbot -t elastic.co --strict-scope -f safe passive -c output_modules.json.siem_friendly=true -om json
-```
+
+If using the integration to collect log files from disk, you can simply use the json output module.
+
+`bbot -t example.com -p subdomain-enum -c modules.json.siem_friendly=true -om json`
+
+
+If using the integration to receive events via HTTP endpoint listener, you can use the the http output module.
+
+`bbot -t example.com -p subdomain-enum --config modules.http.url=http://your.elastic.agent:8080/bbot/asm_intel modules.http.username=bbot modules.http.password=P@55w0rd modules.http.siem_friendly=true -om http`
 
 You will have to configure the path for the output file within the integration settings. A common and popular path that could work here is:
 
-**Example BBOT Path**
+The output modules can also be configured as part of the bbot.yml or preset files.
+
 ```
-/home/<user>/.bbot/scans/*/output.ndjson
+config:
+  modules:
+    http:
+      url: 'https://your.elastic.agent:8443'
+      method: POST
+      username: 'bbot'
+      password: 'P@55w0rd'
+      siem_friendly: true
+    json:
+      siem_friendly: true
 ```
 
+**Example BBOT Path**
+
+`/home/<user>/.bbot/scans/*/output.json`
+
 BBOT Scanning [Documentation](https://www.blacklanternsecurity.com/bbot/scanning/).
 
 ## Data streams

packages/bbot/_dev/deploy/docker/docker-compose.yml

@@ -1,7 +1,26 @@
 services:
-  bbot-logfile:
+  bbot-all-logfile:
     image: alpine
     volumes:
       - ./sample_logs:/sample_logs:ro
       - ${SERVICE_LOGS_DIR}:/var/log
-    command: /bin/sh -c "cp /sample_logs/* /var/log/"
+    command: /bin/sh -c "cp -v /sample_logs/bbot-v*-logfile*.log /var/log/"
+  bbot-all-http:
+    image: docker.elastic.co/observability/stream:v0.17.1
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    environment:
+      - STREAM_PROTOCOL=webhook
+      - STREAM_ADDR=http://elastic-agent:8381/bbot/ndjson
+      - STREAM_WEBHOOK_HEADER=Authorization=abc123
+    command: log --start-signal=SIGHUP --delay=5s /sample_logs/bbot-v*-webhook*.log
+  bbot-all-https:
+    image: docker.elastic.co/observability/stream:v0.17.1
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    environment:
+      - STREAM_PROTOCOL=webhook
+      - STREAM_ADDR=https://elastic-agent:8382/bbot/ndjson
+      - STREAM_WEBHOOK_HEADER=Authorization=abc123
+      - STREAM_INSECURE=true
+    command: log --start-signal=SIGHUP --delay=5s /sample_logs/bbot-v*-webhook*.log

packages/bbot/_dev/deploy/docker/sample_logs/log.log → packages/bbot/_dev/deploy/docker/sample_logs/bbot-v1-logfile.log

@@ -1,7 +1,7 @@
 {"type": "SCAN", "id": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "data": {"SCAN": "heinous_hermione (SCAN:725368977d3a680e579707504e59428a7e3acc9d)"}, "scope_distance": 0, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170919.403808, "source": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "tags": ["in-scope"], "module": "TARGET", "module_sequence": "TARGET"}
-{"type": "DNS_NAME", "id": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "data": {"DNS_NAME": "example.com"}, "scope_distance": 0, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170907.779394, "resolved_hosts": ["123.123.123.123"], "source": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "tags": ["resolved", "a-record", "target", "in-scope", "subdomain"], "module": "TARGET", "module_sequence": "TARGET"}
+{"type": "DNS_NAME", "id": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "data": {"DNS_NAME": "example.com"}, "scope_distance": 0, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170907.779394, "resolved_hosts": ["1.128.0.0"], "source": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "tags": ["resolved", "a-record", "target", "in-scope", "subdomain"], "module": "TARGET", "module_sequence": "TARGET"}
 {"type": "ORG_STUB", "id": "ORG_STUB:4452d71687b6bc2c9389c3349fdc17fbd73b833b", "data": {"ORG_STUB": "ex"}, "scope_distance": 1, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170932.070682, "source": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "tags": ["distance-1"], "module": "speculate", "module_sequence": "speculate"}
-{"type": "DNS_NAME", "id": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "data": {"DNS_NAME": "example.com"}, "scope_distance": 0, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170932.40485, "resolved_hosts": ["123.123.123.123"], "source": "IP_ADDRESS:605bb7cd2537bcf0fd94ce6b120bfb56cf3e0753", "tags": ["resolved", "a-record", "in-scope", "subdomain"], "module": "PTR", "module_sequence": "PTR"}
-{"type": "IP_ADDRESS", "id": "IP_ADDRESS:605bb7cd2537bcf0fd94ce6b120bfb56cf3e0753", "data": {"IP_ADDRESS": "123.123.123.123"}, "scope_distance": 1, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170932.033501, "resolved_hosts": ["123.123.123.123"], "source": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "tags": ["ipv4", "resolved", "distance-1", "internal", "ptr-record"], "module": "A", "module_sequence": "A"}
-{"type": "DNS_NAME", "id": "DNS_NAME:32102fa4a314358e17370aaec3b0f32641dd9ab3", "data": {"DNS_NAME": "example.com"}, "scope_distance": 1, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170933.584077, "resolved_hosts": ["123.123.123.123"], "source": "OPEN_TCP_PORT:62743609c2dcda87da01a4faa9ddd1b7b362939f", "tags": ["soa-record", "affiliate", "resolved", "distance-1", "domain", "a-record", "txt-record", "mx-record", "ns-record"], "module": "sslcert", "module_sequence": "sslcert"}
+{"type": "DNS_NAME", "id": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "data": {"DNS_NAME": "example.com"}, "scope_distance": 0, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170932.40485, "resolved_hosts": ["1.128.0.0"], "source": "IP_ADDRESS:605bb7cd2537bcf0fd94ce6b120bfb56cf3e0753", "tags": ["resolved", "a-record", "in-scope", "subdomain"], "module": "PTR", "module_sequence": "PTR"}
+{"type": "IP_ADDRESS", "id": "IP_ADDRESS:605bb7cd2537bcf0fd94ce6b120bfb56cf3e0753", "data": {"IP_ADDRESS": "1.128.0.0"}, "scope_distance": 1, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170932.033501, "resolved_hosts": ["1.128.0.0"], "source": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "tags": ["ipv4", "resolved", "distance-1", "internal", "ptr-record"], "module": "A", "module_sequence": "A"}
+{"type": "DNS_NAME", "id": "DNS_NAME:32102fa4a314358e17370aaec3b0f32641dd9ab3", "data": {"DNS_NAME": "example.com"}, "scope_distance": 1, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170933.584077, "resolved_hosts": ["1.128.0.0"], "source": "OPEN_TCP_PORT:62743609c2dcda87da01a4faa9ddd1b7b362939f", "tags": ["soa-record", "affiliate", "resolved", "distance-1", "domain", "a-record", "txt-record", "mx-record", "ns-record"], "module": "sslcert", "module_sequence": "sslcert"}
 {"type": "AZURE_TENANT", "id": "AZURE_TENANT:db22e942c8a51294e1abc874daa076caecccbf01", "data": {"AZURE_TENANT": {"tenant-names": ["exampletenant"], "domains": ["example.smtp.online", "example.com", "example1.com", "example.org", "example2.com", "example.gov", "example3.com"], "tenant-id": "70af547c-69ab-416d-b4a6-543b5ce52b99"}}, "scope_distance": 1, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170932.947014, "source": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "tags": ["distance-1"], "module": "azure_tenant", "module_sequence": "azure_tenant"}

packages/bbot/_dev/deploy/docker/sample_logs/bbot-v1-webhook.log

@@ -0,0 +1,7 @@
+{"type": "SCAN", "id": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "data": {"SCAN": "heinous_hermione (SCAN:725368977d3a680e579707504e59428a7e3acc9d)"}, "scope_distance": 0, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170919.403808, "source": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "tags": ["in-scope"], "module": "TARGET", "module_sequence": "TARGET"}
+{"type": "DNS_NAME", "id": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "data": {"DNS_NAME": "example.com"}, "scope_distance": 0, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170907.779394, "resolved_hosts": ["1.128.0.0"], "source": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "tags": ["resolved", "a-record", "target", "in-scope", "subdomain"], "module": "TARGET", "module_sequence": "TARGET"}
+{"type": "ORG_STUB", "id": "ORG_STUB:4452d71687b6bc2c9389c3349fdc17fbd73b833b", "data": {"ORG_STUB": "ex"}, "scope_distance": 1, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170932.070682, "source": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "tags": ["distance-1"], "module": "speculate", "module_sequence": "speculate"}
+{"type": "DNS_NAME", "id": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "data": {"DNS_NAME": "example.com"}, "scope_distance": 0, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170932.40485, "resolved_hosts": ["1.128.0.0"], "source": "IP_ADDRESS:605bb7cd2537bcf0fd94ce6b120bfb56cf3e0753", "tags": ["resolved", "a-record", "in-scope", "subdomain"], "module": "PTR", "module_sequence": "PTR"}
+{"type": "IP_ADDRESS", "id": "IP_ADDRESS:605bb7cd2537bcf0fd94ce6b120bfb56cf3e0753", "data": {"IP_ADDRESS": "1.128.0.0"}, "scope_distance": 1, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170932.033501, "resolved_hosts": ["1.128.0.0"], "source": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "tags": ["ipv4", "resolved", "distance-1", "internal", "ptr-record"], "module": "A", "module_sequence": "A"}
+{"type": "DNS_NAME", "id": "DNS_NAME:32102fa4a314358e17370aaec3b0f32641dd9ab3", "data": {"DNS_NAME": "example.com"}, "scope_distance": 1, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170933.584077, "resolved_hosts": ["1.128.0.0"], "source": "OPEN_TCP_PORT:62743609c2dcda87da01a4faa9ddd1b7b362939f", "tags": ["soa-record", "affiliate", "resolved", "distance-1", "domain", "a-record", "txt-record", "mx-record", "ns-record"], "module": "sslcert", "module_sequence": "sslcert"}
+{"type": "AZURE_TENANT", "id": "AZURE_TENANT:db22e942c8a51294e1abc874daa076caecccbf01", "data": {"AZURE_TENANT": {"tenant-names": ["exampletenant"], "domains": ["example.smtp.online", "example.com", "example1.com", "example.org", "example2.com", "example.gov", "example3.com"], "tenant-id": "70af547c-69ab-416d-b4a6-543b5ce52b99"}}, "scope_distance": 1, "scan": "SCAN:725368977d3a680e579707504e59428a7e3acc9d", "timestamp": 1709170932.947014, "source": "DNS_NAME:f57ba0828becd7bf94faa616db081ed06f31bd3d", "tags": ["distance-1"], "module": "azure_tenant", "module_sequence": "azure_tenant"}