Merge pull request #536 from splunk/develop

Release/806

ansible.cfg

@@ -21,6 +21,10 @@ action_warnings = False
 allow_world_readable_tmpfiles = True
 stderr_callback=debug
 stdout_callback=debug
+localhost_warning = False
+
+[inventory]
+unparsed_is_failed = True
 
 [privilege_escalation]
 become = true

docs/ADVANCED.md

@@ -47,6 +47,7 @@ Splunk-Ansible ships with an inventory script in `inventory/environ.py`. The scr
 | SPLUNK_CERT_PREFIX | HTTP scheme used when making API requests to Splunk management endpoint. Default: `https` | no | no | no |
 | SPLUNK_ROOT_ENDPOINT | Allow SplunkWeb to be accessed behind a given route (ex. reverse proxy usage) | no | no | no |
 | SPLUNK_PASSWORD* | Default password of the admin user | yes | yes | yes |
+| SPLUNK_DECLARATVE_ADMIN_PASSWORD | When `true`, admin password will be fixed to the value defined through Ansible | no | no | no |
 | SPLUNK_PASS4SYMMKEY | Used to overwrite default `pass4SymmKey` for Splunk secrets | no | no | no |
 | SPLUNK_HEC_TOKEN | HEC (HTTP Event Collector) token when enabled | no | no | no |
 | SPLUNK_SHC_SECRET | Search Head Clustering shared secret (deprecated in favor of `SPLUNK_SHC_PASS4SYMMKEY`) | no | no | no |
@@ -79,6 +80,8 @@ Splunk-Ansible ships with an inventory script in `inventory/environ.py`. The scr
 | SPLUNK_HTTP_ENABLESSL_CERT | Path to SSL certificate used for SplunkWeb, if HTTPS is enabled | no | no | no |
 | SPLUNK_HTTP_ENABLESSL_PRIVKEY | Path to SSL private key used for SplunkWeb, if HTTPS is enabled | no | no | no |
 | SPLUNK_HTTP_ENABLESSL_PRIVKEY_PASSWORD | SSL certificate private key password used with SplunkWeb, if HTTPS is enabled | no | no | no |
+| SPLUNK_KVSTORE_PORT | Port to run Splunk KVStore. Default: `8191` | no | no | no |
+| SPLUNK_APPSERVER_PORT | Port to run Splunk appserver. Default: `8065` | no | no | no |
 | SPLUNK_SET_SEARCH_PEERS | Boolean to configure whether search heads should connect to search peers. Default: `True`. Not recommended to change | no | no | no |
 | SPLUNK_SITE | For multisite topologies, define the site of this particular Splunk Enterprise instance | no | no | no |
 | SPLUNK_ALL_SITES | For multisite topologies, define all sites of the topology | no | no | no |
@@ -91,10 +94,19 @@ Splunk-Ansible ships with an inventory script in `inventory/environ.py`. The scr
 | NO_HEALTHCHECK | Disable the Splunk health check script | no | no | yes |
 | STEPDOWN_ANSIBLE_USER | Removes Ansible user from the sudo group when set to true. This means that no other users than root will have root access. | no | no | no |
 | SPLUNK_HOME_OWNERSHIP_ENFORCEMENT | Recursively enforces `${SPLUNK_HOME}` to be owned by the user "splunk". Default: `True` | no | no | no |
+| SPLUNK_DISABLE_POPUPS | Disable pop-ups from login on home page and search app. Default: `False` | no | no | no |
 | HIDE_PASSWORD | Hide all Ansible task logs containing Splunk password to secure output to `stdout`. | no | no | no |
 | JAVA_VERSION | Supply `"oracle:8"`, `"openjdk:8"`, or `"openjdk:11"` to install a respective Java distribution. | no | no | no |
 | JAVA_DOWNLOAD_URL | Provide a custom URL where the Java installation will be fetched| no | no | no |
 | SPLUNK_TAIL_FILE | Determine which file gets written to the container's stdout (default: `splunkd_stderr.log`) | no | no | no |
+| SPLUNK_ES_SSL_ENABLEMENT | When running Enterprise Security version >= 6.3.0, define ssl_enablement installation option | no | no | no |
+| SPLUNK_DSP_ENABLE | Enable DSP forwarding. Default: `false` | no | no | no |
+| SPLUNK_DSP_SERVER | DSP S2S forwarding endpoint | no | no | no |
+| SPLUNK_DSP_CERT | DSP certificate used when forwarding | no | no | no |
+| SPLUNK_DSP_VERIFY | Enable cert verification when forwarding to DSP. Default: `false` | no | no | no |
+| SPLUNK_DSP_PIPELINE_NAME | Name of DSP pipeline to create/update | no | no | no |
+| SPLUNK_DSP_PIPELINE_DESC | Description of DSP pipeline to create/update | no | no | no |
+| SPLUNK_DSP_PIPELINE_SPEC | SPL2 specification of DSP pipeline to create/update | no | no | no |
 | SPLUNK_ENABLE_DFS | Enable [Data Fabric Search (DFS)](https://docs.splunk.com/Documentation/DFS/latest/DFS/Overview) | no | no | no |
 | SPLUNK_DFW_NUM_SLOTS | Maximum number of concurrent DFS searches that run on a search head cluster | no | no | no |
 | SPLUNK_DFC_NUM_SLOTS | Maximum number of concurrent DFS searches that run on each search head | no | no | no |

docs/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Navigation
 
+* [8.0.6](#806)
+* [8.0.5.1](#8051)
 * [8.0.5](#805)
 * [8.0.4.1](#8041)
 * [8.0.4](#804)
@@ -35,6 +37,33 @@
 
 ---
 
+## 8.0.6
+
+#### What's New?
+* Support for declarative admin password, enabling password updates and rotations. `splunk.password` will always be the password for the admin user and changes to `splunk.password` will drive password reconciliation.
+    * `splunk.declarative_admin_password` in `default.yml`
+    * `SPLUNK_DECLARATVE_ADMIN_PASSWORD` environment variable
+* Added flag to disable pop-ups and new user tour
+    * `splunk.disable_popups` in `default.yml`
+    * `SPLUNK_DISABLE_POPUPS` environment variable
+
+#### Changes
+* Fixed default variable propagation order 
+* ASan v5 is dynamically linked to builds
+* Bugfixes and security updates
+
+---
+
+## 8.0.5.1
+
+#### What's New?
+Syncing with latest codebase - currently up to sync with 8.0.6.
+
+#### Changes
+* See [8.0.6](#806) changes above.
+
+---
+
 ## 8.0.5
 
 #### What's New?

docs/SETUP.md

@@ -51,7 +51,13 @@ $ docker run -it splunk/splunk:latest create-defaults > default.yml
 ```
 Alternatively, you can download the example `default.yml` supplied [here](advanced/default.yml.spec.md#sample).
 
-3. Inspect your newly-created `default.yml` and tweak options as you see fit. For a full list of parameters, please see the [`default.yml.spec`](advanced/default.yml.spec.md#spec).
+3. Define a few key variables in your `default.yml`:
+* `splunk.role`: the role this instance will play in the Splunk Enterprise deployment
+* `splunk.build_location`: URL to dynamically fetch the Splunk Enterprise build and install it at run time
+* `splunk.build_remote_src`: this wll be `true` when `splunk.build_location` above is a URL
+* `splunk.password`: default `admin` user password that Splunk will be provisioned with on first-time run
+
+4. Inspect your newly-created `default.yml` and tweak options as you see fit. For a full list of parameters, please see the [`default.yml.spec`](advanced/default.yml.spec.md#spec).
 
 ## Execute playbooks
 In order to get your container to run Ansible, it needs a copy of all the playbooks. 
@@ -63,7 +69,7 @@ $ docker cp . splcontainer:/tmp/splunk-ansible/
 
 2. Run the following command
 ```
-$ docker exec -it splcontainer bash -c 'cd /tmp/splunk-ansible; ansible-playbook --connection local site.yml --extra-vars "@default.yml"'
+$ docker exec -it splcontainer bash -c 'cd /tmp/splunk-ansible; ansible-playbook --inventory localhost, --connection local site.yml --extra-vars "@default.yml"'
 ```
 You should see streaming Ansible output in your terminal. Here is what is happening when you run the above command:
 * `ansible-playbook` command is invoked using the playbook `site.yml`

docs/advanced/default.yml.spec.md

@@ -182,6 +182,10 @@ splunk:
   * NOTE: This is being deprecated in favor of `splunk.search_head_captain_url`.
   * Default: null
 
+  disable_popups: <bool>
+  * When set to true, pop-ups/modals will be disabled from login on the homescreen and search app.
+  * Default: false
+
   preferred_captaincy: <bool>
   * Boolean to determine whether splunk should set a preferred captain.  This can have an effect on day 2 operations if the search heads need to be restarted 
   * Default: true
@@ -214,6 +218,10 @@ splunk:
   * Default Splunk admin user password. This is REQUIRED when starting Splunk, and can only be set during the first-time run of the playbooks. If changes are required to the admin password, they should be done through SplunkWeb/CLI and the new value should be re-entered here.
   * Default: null
 
+  declarative_admin_password: <bool>
+  * When set to true, the playbooks will always enforce that the admin password is set to the value of `password` above. Any changes to the admin password outside of splunk-ansible will be reverted.
+  * Default: false
+
   user: <str>
   * Host user under which Splunk will run
   * Default: splunk

execute_adhoc_plays.yml

@@ -4,6 +4,7 @@
     url: "{{ playbook }}"
     dest: "{{ '/opt/container_artifact/' + playbook|basename }}"
     force: yes
+    mode: 0666
   ignore_errors: yes
   register: downloaded_plays
   when:

inventory/environ.py

@@ -111,6 +111,7 @@ def getDefaultVars():
     overrideEnvironmentVars(defaultVars)
     getAnsibleContext(defaultVars)
     getASan(defaultVars)
+    getDisablePopups(defaultVars)
     getHEC(defaultVars)
     getSecrets(defaultVars)
     getSplunkPaths(defaultVars)
@@ -365,6 +366,11 @@ def getSecrets(vars_scope):
             vars_scope["splunk"]["password"] = f.read().strip()
             if not vars_scope["splunk"]["password"]:
                 raise Exception("Splunk password supplied is empty/null")
+    dpw = os.environ.get("SPLUNK_DECLARATIVE_ADMIN_PASSWORD", "")
+    if dpw.lower() == "true":
+        vars_scope["splunk"]["declarative_admin_password"] = True
+    else:
+        vars_scope["splunk"]["declarative_admin_password"] = bool(vars_scope["splunk"].get("declarative_admin_password"))
     vars_scope["splunk"]["pass4SymmKey"] = os.environ.get('SPLUNK_PASS4SYMMKEY', vars_scope["splunk"].get("pass4SymmKey"))
     vars_scope["splunk"]["secret"] = os.environ.get('SPLUNK_SECRET', vars_scope["splunk"].get("secret"))
 
@@ -403,6 +409,17 @@ def getASan(vars_scope):
     if vars_scope["splunk"]["asan"]:
         vars_scope["ansible_environment"].update({"ASAN_OPTIONS": "detect_leaks=0"})
 
+def getDisablePopups(vars_scope):
+    """
+    Configure pop-up settings
+    """
+    vars_scope["splunk"]["disable_popups"] = bool(vars_scope["splunk"].get("disable_popups"))
+    popups_disabled = os.environ.get("SPLUNK_DISABLE_POPUPS", "")
+    if popups_disabled.lower() == "true":
+        vars_scope["splunk"]["disable_popups"] = True
+    elif popups_disabled.lower() == "false":
+        vars_scope["splunk"]["disable_popups"] = False
+
 def getHEC(vars_scope):
     """
     Configure HEC settings
@@ -613,19 +630,16 @@ def loadDefaults():
     """
     # Load base defaults from splunk-ansible repository
     base = loadBaseDefaults()
-    # Build an array of new defaults to override the base
-    ymls = []
-    config = base.get("config")
-    if not config:
+    if not base.get("config"):
         return base
     # Add "baked" files to array
-    ymls.extend(loadBakedDefaults(config))
-    # Add "env" URLs to array
-    ymls.extend(loadEnvDefaults(config))
-    # Add "host" URLs to array
-    ymls.extend(loadHostDefaults(config))
-    # For each new YAML discovered, merge them with base in order so values get superseded
-    for yml in ymls:
+    for yml in loadBakedDefaults(base.get("config")):
+        base = mergeDefaults(base, yml["key"], yml["src"])
+    # Add "env" files to array
+    for yml in loadEnvDefaults(base.get("config")):
+        base = mergeDefaults(base, yml["key"], yml["src"])
+    # Add "host" files to array
+    for yml in loadHostDefaults(base.get("config")):
         base = mergeDefaults(base, yml["key"], yml["src"])
     return base
 

inventory/splunk_defaults_linux.yml

@@ -35,6 +35,7 @@ splunk:
     admin_user: "admin"
     root_endpoint:
     password:
+    declarative_admin_password: False
     secret:
     pass4SymmKey:
     svc_port: 8089
@@ -110,6 +111,7 @@ splunk:
     deployer_url:
     connection_timeout: 0
     enable_service: False
+    disable_popups: False
     service_name:
     smartstore:
     app_paths:

inventory/splunk_defaults_windows.yml

@@ -35,6 +35,7 @@ splunk:
     admin_user: "admin"
     root_endpoint:
     password:
+    declarative_admin_password: False
     secret:
     pass4SymmKey:
     svc_port: 8089
@@ -102,6 +103,7 @@ splunk:
     deployer_url:
     connection_timeout: 180
     enable_service: False
+    disable_popups: False
     service_name:
     smartstore:
     app_paths:

inventory/splunkforwarder_defaults_linux.yml

@@ -35,6 +35,7 @@ splunk:
     admin_user: "admin"
     root_endpoint:
     password:
+    declarative_admin_password: False
     secret:
     pass4SymmKey:
     svc_port: 8089
@@ -102,6 +103,7 @@ splunk:
     deployer_url:
     connection_timeout: 0
     enable_service: False
+    disable_popups: False
     service_name:
     smartstore:
     app_paths:

inventory/splunkforwarder_defaults_windows.yml

@@ -35,6 +35,7 @@ splunk:
     admin_user: "admin"
     root_endpoint:
     password:
+    declarative_admin_password: False
     secret:
     pass4SymmKey:
     svc_port: 8089
@@ -102,6 +103,7 @@ splunk:
     deployer_url:
     connection_timeout: 180
     enable_service: False
+    disable_popups: False
     service_name:
     smartstore:
     app_paths:

roles/splunk_common/tasks/add_forward_server.yml

@@ -1,6 +1,6 @@
 ---
 - name: "Enable forwarding to {{ forward_servers }}"
-  command: "{{ splunk.exec }} add forward-server {{ item }}:{{ splunk.s2s.port if splunk.s2s.port is defined else splunk.s2s_port }} -auth {{ splunk.admin_user }}:{{ splunk.password }}"
+  command: "{{ splunk.exec }} add forward-server {{ item }}:{{ splunk.s2s.port if splunk.s2s.port is defined else splunk.s2s_port }} -auth {{ splunk.admin_user }}:{{ splunk.password }} --accept-license --answer-yes --no-prompt"
   become: yes
   become_user: "{{ splunk.user }}"
   with_items: "{{ forward_servers }}"
@@ -39,18 +39,6 @@
   no_log: "{{ hide_password }}"
   register: enable_ssl_forwarding
 
-- name: "Get Splunk status"
-  command: "{{ splunk.exec }} status --accept-license --answer-yes --no-prompt"
-  become: yes
-  become_user: "{{ splunk.user }}"
-  register: splunk_status
-  changed_when: False
-  failed_when: False
-  ignore_errors: yes
-
-# We want to restart only when Splunk is currently running and when any of the above have changed
-- name: Trigger restart
-  command: ls
-  changed_when: splunk_status.rc == 0 and (forward_status is changed or enable_ssl_forwarding is changed)
-  notify:
-    - Restart the splunkd service
+# Restart only when Splunk is running and when any of the above have changed
+- include_tasks: trigger_restart.yml
+  when: forward_status is changed or enable_ssl_forwarding is changed

roles/splunk_common/tasks/disable_popups.yml

@@ -0,0 +1,27 @@
+---
+- name: GET OptInVersion
+  uri:
+    url: "{{ cert_prefix }}://127.0.0.1:{{ splunk.svc_port }}/servicesNS/nobody/splunk_instrumentation/admin/telemetry/general?output_mode=json"
+    method: GET
+    user: "{{ splunk.admin_user }}"
+    password: "{{ splunk.password }}"
+    validate_certs: false
+    status_code: 200
+    timeout: 10
+    return_content: yes
+  register: telemetry
+  no_log: "{{ hide_password }}"
+
+- name: Disable Popups
+  uri:
+    url: "{{ cert_prefix }}://127.0.0.1:{{ splunk.svc_port }}/{{ item.key }}"
+    method: POST
+    user: "{{ splunk.admin_user }}"
+    password: "{{ splunk.password }}"
+    body: "{{ item.value }}"
+    validate_certs: false
+    status_code: 200,201,409
+  with_items:
+    - { key: "servicesNS/admin/user-prefs/data/user-prefs/general", value: "hideInstrumentationOptInModal=1&notification_python_3_impact=false&showWhatsNew=0" }
+    - { key: "servicesNS/nobody/splunk_instrumentation/admin/telemetry/general", value: "showOptInModal=0&optInVersionAcknowledged={{ telemetry['json']['entry'][0]['content']['optInVersion'] }}" }
+    - { key: "servicesNS/admin/search/data/ui/ui-tour/search-tour", value: "tourPage=search&viewed=1" }

roles/splunk_common/tasks/enable_admin_auth.yml

@@ -0,0 +1,56 @@
+---
+- name: Set admin access via seed
+  when: first_run | bool
+  block:
+  - name: "Hash the password"
+    command: "{{ splunk.exec }} hash-passwd {{ splunk.password }}"
+    register: hashed_pwd
+    changed_when: hashed_pwd.rc == 0
+    become: yes
+    become_user: "{{ splunk.user }}"
+    no_log: "{{ hide_password }}"
+
+  - name: "Generate user-seed.conf (Linux)"
+    ini_file:
+      owner: "{{ splunk.user }}"
+      group: "{{ splunk.group }}"
+      dest: "{{ splunk.home }}/etc/system/local/user-seed.conf"
+      section: user_info
+      option: "{{ item.opt }}"
+      value: "{{ item.val }}"
+    with_items:
+      - { opt: 'USERNAME', val: '{{ splunk.admin_user }}' }
+      - { opt: 'HASHED_PASSWORD', val: '{{ hashed_pwd.stdout }}' }
+    loop_control:
+      label: "{{ item.opt }}"
+    when: ansible_system is match("Linux")
+    no_log: "{{ hide_password }}"
+
+  - name: "Generate user-seed.conf (Windows)"
+    ini_file:
+      dest: "{{ splunk.home }}/etc/system/local/user-seed.conf"
+      section: user_info
+      option: "{{ item.opt }}"
+      value: "{{ item.val }}"
+    with_items:
+      - { opt: 'USERNAME', val: '{{ splunk.admin_user }}' }
+      - { opt: 'HASHED_PASSWORD', val: '{{ hashed_pwd.stdout }}' }
+    loop_control:
+      label: "{{ item.opt }}"
+    when: ansible_system is match("CYGWIN*|Win32NT")
+    no_log: "{{ hide_password }}"
+
+- name: Establish declarative admin access
+  when:
+    - "'declarative_admin_password' in splunk and splunk.declarative_admin_password | bool"
+    - not first_run | bool
+  block:
+  - name: Apply admin password
+    command: "{{ splunk.exec }} cmd splunkd rest --noauth POST /services/admin/users/{{ splunk.admin_user }} 'password={{ splunk.password }}'"
+    register: declarative_pw
+    changed_when: False
+    become: yes
+    become_user: "{{ splunk.user }}"
+    no_log: "{{ hide_password }}"
+
+  - include_tasks: trigger_restart.yml