diff --git a/bin/rock b/bin/rock index 55fab25b4..53bb709b1 100755 --- a/bin/rock +++ b/bin/rock @@ -356,4 +356,4 @@ EOF (*) usage exit 0 ;; -esac \ No newline at end of file +esac diff --git a/playbooks/deploy-rock.yml b/playbooks/deploy-rock.yml index 374ad665a..2e8d62246 100644 --- a/playbooks/deploy-rock.yml +++ b/playbooks/deploy-rock.yml @@ -22,7 +22,7 @@ become: true roles: - role: elasticsearch - when: "rock_services | selectattr('name', 'equalto', 'elasticsearch') | map(attribute='installed') | list | first | bool" + when: "'elasticsearch' in installed_services" - hosts: elasticsearch tags: @@ -38,7 +38,7 @@ become: true roles: - role: elasticsearch - when: "rock_services | selectattr('name', 'equalto', 'elasticsearch') | map(attribute='installed') | list | first | bool and (es_restart is defined and es_restart)" + when: "'elasticsearch' in installed_services and (es_restart is defined and es_restart)" - hosts: elasticsearch tags: @@ -53,7 +53,7 @@ become: true roles: - role: elasticsearch - when: "rock_services | selectattr('name', 'equalto', 'elasticsearch') | map(attribute='installed') | list | first | bool" + when: "'elasticsearch' in installed_services" - hosts: zookeeper tags: @@ -64,7 +64,7 @@ become: true roles: - role: zookeeper - when: "rock_services | selectattr('name', 'equalto', 'zookeeper') | map(attribute='installed') | list | first | bool" + when: "'zookeeper' in installed_services" - hosts: kafka tags: @@ -75,7 +75,7 @@ become: true roles: - role: kafka - when: "rock_services | selectattr('name', 'equalto', 'kafka') | map(attribute='installed') | list | first | bool" + when: "'kafka' in installed_services" - hosts: stenographer tags: @@ -87,7 +87,7 @@ become: true roles: - role: stenographer - when: "rock_services | selectattr('name', 'equalto', 'stenographer') | map(attribute='installed') | list | first | bool" + when: "'stenographer' in installed_services" stenographer_monitor_interfaces: "{{ rock_monifs }}" - hosts: zeek @@ -100,7 +100,7 @@ become: true roles: - role: zeek - when: "rock_services | selectattr('name', 'equalto', 'zeek') | map(attribute='installed') | list | first | bool" + when: "'zeek' in installed_services" - hosts: suricata tags: @@ -112,7 +112,7 @@ become: true roles: - role: suricata - when: "rock_services | selectattr('name', 'equalto', 'suricata') | map(attribute='installed') | list | first | bool" + when: "'suricata' in installed_services" - hosts: fsf tags: @@ -124,7 +124,7 @@ become: true roles: - role: fsf - when: "rock_services | selectattr('name', 'equalto', 'fsf') | map(attribute='installed') | list | first | bool" + when: "'fsf' in installed_services" - hosts: - docket @@ -140,7 +140,7 @@ become: true roles: - role: lighttpd - when: "rock_services | selectattr('name', 'equalto', 'lighttpd') | map(attribute='installed') | list | first | bool" + when: "'lighttpd' in installed_services" - hosts: - docket @@ -154,8 +154,8 @@ become: true roles: - role: docket - when: "rock_services | selectattr('name', 'equalto', 'docket') | map(attribute='installed') | list | first | bool" - docket_enable: "{{ local_services | selectattr('name', 'equalto', 'docket') | map(attribute='enabled') | first | bool }}" + when: "'docket' in installed_services" + docket_enable: "{{ 'docket' in enabled_servicesl }}" - hosts: kibana tags: @@ -167,4 +167,4 @@ become: true roles: - role: kibana - when: "rock_services | selectattr('name', 'equalto', 'kibana') | map(attribute='installed') | list | first | bool" + when: "'kibana' in installed_services" diff --git a/playbooks/group_vars/all.yml b/playbooks/group_vars/all.yml index 26c79f85c..fd5a2ced3 100644 --- a/playbooks/group_vars/all.yml +++ b/playbooks/group_vars/all.yml @@ -39,50 +39,57 @@ rock_services: quota_weight: 1 installed: true enabled: true + version: 3.1.5 - name: stenographer quota_weight: 8 installed: true enabled: true + version: 1:0-2 - name: docket quota_weight: 0 installed: true enabled: true + version: 1.0.3 - name: suricata quota_weight: 2 installed: true enabled: true + version: 5.0.5 - name: elasticsearch quota_weight: 4 installed: true enabled: true + version: 7.11.2 - name: kibana quota_weight: 0 installed: true enabled: true + version: 7.11.2 - name: zookeeper quota_weight: 0 installed: true enabled: true + version: 3.4.14 - name: kafka quota_weight: 4 installed: true enabled: true + version: 2.3.0 - name: lighttpd quota_weight: 0 installed: true enabled: true - - name: fsf - quota_weight: 1 - installed: true - enabled: true + version: 1.4.54 - name: filebeat quota_weight: 0 installed: true enabled: true + version: 7.11.2 - name: logstash quota_weight: 0 installed: true enabled: true + version: 7.11.2 rocknsm_package_list: - jq diff --git a/playbooks/manage-services.yml b/playbooks/manage-services.yml index 5ed192868..caa531e37 100644 --- a/playbooks/manage-services.yml +++ b/playbooks/manage-services.yml @@ -13,7 +13,7 @@ - name: collect enabled services set_fact: - enabled_services: "{{ rock_services | selectattr('enabled', 'equalto', True) | map(attribute='name') | list }}" + enabled_services: "{{ rock_services | rejectattr('enabled', 'equalto', False) | map(attribute='name') | list }}" - name: Collect ROCK services from facts set_fact: diff --git a/roles/common/tasks/configure.yml b/roles/common/tasks/configure.yml index e37e851b2..9d03502bd 100644 --- a/roles/common/tasks/configure.yml +++ b/roles/common/tasks/configure.yml @@ -236,7 +236,7 @@ # The "base" repo only goes up to 0.13, so we need to exclude it for this task - name: Install pyopenssl package yum: - name: python2-pyOpenSSL + name: pyOpenSSL state: installed when: "'docket' in group_names or 'stenographer' in group_names or 'kibana' in group_names" diff --git a/roles/docket/templates/docket_prod.yaml.j2 b/roles/docket/templates/docket_prod.yaml.j2 index e72d210db..47e49218a 100644 --- a/roles/docket/templates/docket_prod.yaml.j2 +++ b/roles/docket/templates/docket_prod.yaml.j2 @@ -1,128 +1,128 @@ ---- -# Configure your stenographer instances here. sensor is a -# just an arbitrary unique name -STENOGRAPHER_INSTANCES: - {{ docket_steno_instances | to_json }} - -# ==== Flask ==== -DEBUG: {{ docket_debug }} -TESTING: {{ docket_testing }} -SECRET_KEY: {{ docket_secret }} -SESSION_COOKIE_NAME: {{ docket_session_cookie }} - - -# ==== Celery ==== -# CELERY_URL: the broker for sharing celery queues across processes -CELERY_URL: {{ docket_celery_url }} - -# ==== Basic Docket Configuration ==== -# SPOOL_DIR where captures are saved and served -# configure the webserver to serve SPOOL_DIR/[a-z0-9-]{32,}/MERGED_NAME.pcap -SPOOL_DIR: {{ docket_spool_dir }} - -# WEB_ROOT the base url for all docket requests: default is /api -# examples: -# http://HOST:PORT/api/stats/ -# http://HOST:PORT/api/uri/host/1.2.3.4/ -WEB_ROOT: {{ docket_url_apppath }}/api -UI_WEB_ROOT: {{ docket_url_apppath }} -# WEB_ROOT the base url for PCAP requests: default is /results -# example: -# http://HOST:PORT/results//merged.pcap -PCAP_WEB_ROOT: {{ docket_url_resultspath }} - -# ==== Logging ==== -LOGGER_NAME: {{ docket_logger }} -# LOG_LEVEL (critical, error, warning, info, debug) any other value will be ignored -LOG_LEVEL: info - -# Python log format strings: -LOG_MSG_FORMAT: "[%(processName)-8s:%(thread)d] %(message)s" -LOG_DATE_FORMAT: '%FT%T.%fZ' -LOG_DATE_UTC: true -# LOG_FILE a secondary logging handler - this file will be rotated at midnight -LOG_FILE: /var/log/docket/docket.log -LOG_FILE_LEVEL: info -LOG_FILE_MSG_FORMAT: "%(asctime)s[%(processName)-8s:%(thread)d] %(message)s" -LOG_FILE_DATE_FORMAT: '%FT%T' - -# ==== Query ==== -# TIME_WINDOW Integer: specifies the size of a time 'chunk' in seconds for request de-duplication -# Requested times are widened to a multiple of this value. -TIME_WINDOW: 60 -# EMPTY_RESULT capacity: the size of a pcap that contains packets (detects 'no matching packets') -EMPTY_RESULT: 25B - -# MERGED_NAME - base name (no extension) of the result capture file. Just a string. -MERGED_NAME: merged - -# QUERY_FILE - base name of the query meta-data save file. Just a string. -QUERY_FILE: query - -#DOCKET_NO_REDIS - if True, docket will use files (instead of redis) to maintain query meta-data -DOCKET_NO_REDIS: {{ docket_no_redis }} - -# LONG_AGO - The default 'start-time' aka 'after' clause isn't 1970, but this long before now() -LONG_AGO: {{ docket_long_ago }} - -# ==== Timeout ==== -# TIMEOUTs - Docket queries will fail if stenoboxes are unresponsive. -# For each stenobox instance: -# Docket remembers the last time it was idle. IDLE_TIME -# If it is not Docket waits IDLE_SLEEP, then requests stats, repeat if not idle -# If a stenobox had not become idle after QUERY_TIMEOUT, this instance FAILs. -# -# Once idle: a stenobox is queried and results are written to disk. -# If the 'Requests' module times-out (QUERY_TIMEOUT), this instance FAILs. -# -# Results are shared with other Docket processes (Web, celery queues...) -# This is subject to LOCK_TIMEOUT, which is never expected to happen -# -# IDLE_TIME - 5.0, assume stenoboxes remain IDLE for 5 seconds and check again after that. -# IDLE_SLEEP - 2.0, wait time between IDLE queries. will occur at least once every IDLE_TIME. -# STAT_TIMEOUT - 3.0, assume a stenobox is broken if we can't get stats within this many seconds -# QUERY_TIMEOUT - 720, seconds. A query request will fail if stenobox doesn't complete this quickly -# LOCK_TIMEOUT - 2.0, seconds to wait on a IPC shared data (file) lock before giving up. -IDLE_TIME: 5.0 -IDLE_SLEEP: 2.0 -STAT_TIMEOUT: 3.0 -QUERY_TIMEOUT: 720.0 -LOCK_TIMEOUT: 2.0 - -# WEIGHTS 'Fat Finger' protection -# - request 'weight' can be estimated to prevent clogging up the system. -# - Estimate the amount of data in the system (total buffered packet data) -# - Pick a threshold for a single request 1% * (TOTAL / 8 hours / 60 minutes / 5 requests per minute) -# - IPs - the number of (active) IPs captured -# - NETS - the number of (active) subnets captured -# - PORTs- the number of (active) Ports per IP -# - HOURS- the number of hours in the buffer (I know variance is likely above 50%) -# - 'weight' = (TOTAL / (HOURS * 3600) * QUERY_SECONDS) / ( SUM(IPs + NETs) * SUM( PORTs ) -# ex: a 5TB/8hour system, query: 1 IP and 1 port for 10 seconds == 381774 == 380KB -# 5TB / (8 * 3600) * 10seconds / ( (50.0 * 1ip + 2 * 0net) * 100.0 * 1port ) -# NOTE: Raw queries are not weighed or widened so only identical raw queries are deduplicated. -#WEIGHT_THRESHOLD: 22MB # valid quantifiers( B KB MB GB TB PB ) -#WEIGHT_TOTAL: 5TB # valid quantifiers( B KB MB GB TB PB ) -#WEIGHT_IPS: 50.0 -#WEIGHT_NETS: 2.0 -#WEIGHT_PORTS: 100.0 -#WEIGHT_HOURS: 8.0 - -# ==== EXPIRATION ==== -# EXPIRE_SPACE - if set, the oldest items will be removed until this much space becomes available. -# EXPIRE_TIME - how long after last 'touch' does a request expire ? -# CLEANUP_PERIOD - delete expired queries will run every CLEANUP_PERIOD seconds -EXPIRE_SPACE: 500MB -EXPIRE_TIME: 48h -CLEANUP_PERIOD: 1h - -# ==== Formatting ==== -# DATE_FORMAT strftime format for showing a datetime to a user -DATE_FORMAT: '%FT%T' - -# ID_FORMAT: if True, IDs will contain dashes: https://en.wikipedia.org/wiki/Universally_unique_identifier -UUID_FORMAT: false - -# Stop accepting requests if SPOOL_DIR's free space or nodes go below these values: -FREE_BYTES: 200MB -FREE_NODES: 10111 +--- +# Configure your stenographer instances here. sensor is a +# just an arbitrary unique name +STENOGRAPHER_INSTANCES: + {{ docket_steno_instances | to_json }} + +# ==== Flask ==== +DEBUG: {{ docket_debug }} +TESTING: {{ docket_testing }} +SECRET_KEY: {{ docket_secret }} +SESSION_COOKIE_NAME: {{ docket_session_cookie }} + + +# ==== Celery ==== +# CELERY_URL: the broker for sharing celery queues across processes +CELERY_URL: {{ docket_celery_url }} + +# ==== Basic Docket Configuration ==== +# SPOOL_DIR where captures are saved and served +# configure the webserver to serve SPOOL_DIR/[a-z0-9-]{32,}/MERGED_NAME.pcap +SPOOL_DIR: {{ docket_spool_dir }} + +# WEB_ROOT the base url for all docket requests: default is /api +# examples: +# http://HOST:PORT/api/stats/ +# http://HOST:PORT/api/uri/host/1.2.3.4/ +WEB_ROOT: {{ docket_url_apppath }}/api +UI_WEB_ROOT: {{ docket_url_apppath }} +# WEB_ROOT the base url for PCAP requests: default is /results +# example: +# http://HOST:PORT/results//merged.pcap +PCAP_WEB_ROOT: {{ docket_url_resultspath }} + +# ==== Logging ==== +LOGGER_NAME: {{ docket_logger }} +# LOG_LEVEL (critical, error, warning, info, debug) any other value will be ignored +LOG_LEVEL: info + +# Python log format strings: +LOG_MSG_FORMAT: "[%(processName)-8s:%(thread)d] %(message)s" +LOG_DATE_FORMAT: '%FT%T.%fZ' +LOG_DATE_UTC: true +# LOG_FILE a secondary logging handler - this file will be rotated at midnight +LOG_FILE: /var/log/docket/docket.log +LOG_FILE_LEVEL: info +LOG_FILE_MSG_FORMAT: "%(asctime)s[%(processName)-8s:%(thread)d] %(message)s" +LOG_FILE_DATE_FORMAT: '%FT%T' + +# ==== Query ==== +# TIME_WINDOW Integer: specifies the size of a time 'chunk' in seconds for request de-duplication +# Requested times are widened to a multiple of this value. +TIME_WINDOW: 60 +# EMPTY_RESULT capacity: the size of a pcap that contains packets (detects 'no matching packets') +EMPTY_RESULT: 25B + +# MERGED_NAME - base name (no extension) of the result capture file. Just a string. +MERGED_NAME: merged + +# QUERY_FILE - base name of the query meta-data save file. Just a string. +QUERY_FILE: query + +#DOCKET_NO_REDIS - if True, docket will use files (instead of redis) to maintain query meta-data +DOCKET_NO_REDIS: {{ docket_no_redis }} + +# LONG_AGO - The default 'start-time' aka 'after' clause isn't 1970, but this long before now() +LONG_AGO: {{ docket_long_ago }} + +# ==== Timeout ==== +# TIMEOUTs - Docket queries will fail if stenoboxes are unresponsive. +# For each stenobox instance: +# Docket remembers the last time it was idle. IDLE_TIME +# If it is not Docket waits IDLE_SLEEP, then requests stats, repeat if not idle +# If a stenobox had not become idle after QUERY_TIMEOUT, this instance FAILs. +# +# Once idle: a stenobox is queried and results are written to disk. +# If the 'Requests' module times-out (QUERY_TIMEOUT), this instance FAILs. +# +# Results are shared with other Docket processes (Web, celery queues...) +# This is subject to LOCK_TIMEOUT, which is never expected to happen +# +# IDLE_TIME - 5.0, assume stenoboxes remain IDLE for 5 seconds and check again after that. +# IDLE_SLEEP - 2.0, wait time between IDLE queries. will occur at least once every IDLE_TIME. +# STAT_TIMEOUT - 3.0, assume a stenobox is broken if we can't get stats within this many seconds +# QUERY_TIMEOUT - 720, seconds. A query request will fail if stenobox doesn't complete this quickly +# LOCK_TIMEOUT - 2.0, seconds to wait on a IPC shared data (file) lock before giving up. +IDLE_TIME: 5.0 +IDLE_SLEEP: 2.0 +STAT_TIMEOUT: 3.0 +QUERY_TIMEOUT: 720.0 +LOCK_TIMEOUT: 2.0 + +# WEIGHTS 'Fat Finger' protection +# - request 'weight' can be estimated to prevent clogging up the system. +# - Estimate the amount of data in the system (total buffered packet data) +# - Pick a threshold for a single request 1% * (TOTAL / 8 hours / 60 minutes / 5 requests per minute) +# - IPs - the number of (active) IPs captured +# - NETS - the number of (active) subnets captured +# - PORTs- the number of (active) Ports per IP +# - HOURS- the number of hours in the buffer (I know variance is likely above 50%) +# - 'weight' = (TOTAL / (HOURS * 3600) * QUERY_SECONDS) / ( SUM(IPs + NETs) * SUM( PORTs ) +# ex: a 5TB/8hour system, query: 1 IP and 1 port for 10 seconds == 381774 == 380KB +# 5TB / (8 * 3600) * 10seconds / ( (50.0 * 1ip + 2 * 0net) * 100.0 * 1port ) +# NOTE: Raw queries are not weighed or widened so only identical raw queries are deduplicated. +#WEIGHT_THRESHOLD: 22MB # valid quantifiers( B KB MB GB TB PB ) +#WEIGHT_TOTAL: 5TB # valid quantifiers( B KB MB GB TB PB ) +#WEIGHT_IPS: 50.0 +#WEIGHT_NETS: 2.0 +#WEIGHT_PORTS: 100.0 +#WEIGHT_HOURS: 8.0 + +# ==== EXPIRATION ==== +# EXPIRE_SPACE - if set, the oldest items will be removed until this much space becomes available. +# EXPIRE_TIME - how long after last 'touch' does a request expire ? +# CLEANUP_PERIOD - delete expired queries will run every CLEANUP_PERIOD seconds +EXPIRE_SPACE: 500MB +EXPIRE_TIME: 48h +CLEANUP_PERIOD: 1h + +# ==== Formatting ==== +# DATE_FORMAT strftime format for showing a datetime to a user +DATE_FORMAT: '%FT%T' + +# ID_FORMAT: if True, IDs will contain dashes: https://en.wikipedia.org/wiki/Universally_unique_identifier +UUID_FORMAT: false + +# Stop accepting requests if SPOOL_DIR's free space or nodes go below these values: +FREE_BYTES: 200MB +FREE_NODES: 10111