Merge pull request #599 from splunk/develop

Release/812

.circleci/config.yml

@@ -1,8 +1,9 @@
-version: 2
+version: 2.1
+
 jobs:
   splunk-ansible-test:
     machine:
-      image: circleci/classic:latest
+      image: ubuntu-1604:202007-01
     steps:
       - checkout
       - run:
@@ -24,12 +25,17 @@ jobs:
           path: tests/results
   py3k-splunk-ansible-test:
     machine:
-      image: circleci/classic:latest
+      image: ubuntu-1604:202007-01
     steps:
       - checkout
       - run:
           name: Setup Python3
-          command: pyenv global 2.7.12 3.5.2
+          command: |
+            pyenv global 2.7.18 3.7.8
+            python --version
+            pip --version
+            python3 --version
+            pip3 --version
       - run:
           name: Run small tests
           command: make py3k-small-tests

CODEOWNERS

@@ -7,10 +7,10 @@
 
 # Default owners for everything in docker-splunk:
 # *       @splunk/if-01
-*       @nwang92 @alishamayor @arctan5x @jrigassio-splunk @jmeixensperger @hendolim @bb03 @sarahotis @martinluo22 @Devmalion
+*       @nwang92 @alishamayor @jrigassio-splunk @jmeixensperger @hendolim @bb03 @sarahotis @martinluo22 @msuthar-splunk @ttwd80 @matt-sm
 
 # Docs-only pull requests:
 /docs/  @alishamayor @nwang92 @bb03
 
 # Release changelog
-docs/CHANGELOG.md  @nwang92 @alishamayor @arctan5x @jrigassio-splunk @jmeixensperger @hendolim @bb03 @sarahotis @martinluo22 @Devmalion
+docs/CHANGELOG.md  @nwang92 @alishamayor @jrigassio-splunk @jmeixensperger @hendolim @bb03 @sarahotis @martinluo22 @msuthar-splunk @ttwd80 @matt-sm

Makefile

@@ -4,7 +4,7 @@ SHELL := /bin/bash
 
 test-setup:
 	@echo 'Install test requirements'
-	pip install --upgrade pip
+	pip install pip==20.3.3
 	pip install -r $(shell pwd)/tests/requirements.txt --upgrade
 
 py3k-test-setup:
@@ -15,7 +15,7 @@ lint: test-setup
 	ansible-lint -v -c ./tests/ansible-lint.cfg site.yml roles/**/**/*.yml roles/**/**/**/*.yml
 
 py3k-lint: test-setup 
-	# We're treating each file separately here, because of their scarsity
+	# We're treating each file separately here, because of their scarcity
 	# This will need to be re-evaluated if a full blown module gets in here
 	pylint --py3k $(shell find . -name "*.py")
 	caniusepython3 -r tests/requirements.txt

docs/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Navigation
 
+* [8.1.2](#812)
 * [8.1.1](#811)
 * [8.1.0.1](#8101)
 * [8.1.0](#810)
@@ -44,6 +45,13 @@
 
 ---
 
+## 8.1.2
+
+#### Changes
+* Bugfixes and documentation updates
+
+---
+
 ## 8.1.1
 
 #### Changes

docs/advanced/default.yml.spec.md

@@ -10,13 +10,13 @@
 The following is the full spec file for a `default.yml` that controls how Splunk gets provisioned.
 
 ```
-ansible_post_tasks: <str>
-* Comma-separated list of paths or URLs to custom Ansible playbooks to run AFTER Splunk has been setup using the provided site.yml
-* Default: null
+ansible_post_tasks: <list>
+* list of paths or URLs to custom Ansible playbooks to run AFTER Splunk has been setup using the provided site.yml
+* Default: []
 
-ansible_pre_tasks: <str>
-* Comma-separated list of paths or URLs to custom Ansible playbooks to run BEFORE Splunk sets up using the provided site.yml
-* Default: null
+ansible_pre_tasks: <list>
+* list of paths or URLs to custom Ansible playbooks to run BEFORE Splunk sets up using the provided site.yml
+* Default: []
 
 ansible_environment: <dict>
 * Map of environment variables used only during the execution context of all the Ansible tasks. For more information, see https://docs.ansible.com/ansible/latest/user_guide/playbooks_environment.html

inventory/environ.py

@@ -411,12 +411,32 @@ def getLaunchConf(vars_scope):
         launch.update({k:v for k,v in [x.split("=", 1) for x in settings.split(",")]})
     vars_scope["splunk"]["launch"] = launch
 
+def ensureListValue(value, separator):
+    if isinstance(value, list):
+        return value
+    elif (not value) or (not value.strip()):
+        return []
+    else:
+        return splitAndStrip(value, separator)
+
+def splitAndStrip(value, separator):
+    if not value:
+        return []
+    return [x.strip() for x in value.split(separator)]
+
+def transformEnvironmentVariable(environmentVariableName, transform, default):
+    if environmentVariableName in os.environ:
+        return transform(os.environ.get(environmentVariableName))
+    else:
+        return default
+
 def getAnsibleContext(vars_scope):
     """
     Parse parameters that influence Ansible execution
     """
-    vars_scope["ansible_pre_tasks"] = os.environ.get("SPLUNK_ANSIBLE_PRE_TASKS", vars_scope.get("ansible_pre_tasks"))
-    vars_scope["ansible_post_tasks"] = os.environ.get("SPLUNK_ANSIBLE_POST_TASKS", vars_scope.get("ansible_post_tasks"))
+    stringSeparator = ","
+    vars_scope["ansible_pre_tasks"] = transformEnvironmentVariable("SPLUNK_ANSIBLE_PRE_TASKS", lambda v: splitAndStrip(v, stringSeparator), ensureListValue(vars_scope.get("ansible_pre_tasks"), stringSeparator))
+    vars_scope["ansible_post_tasks"] = transformEnvironmentVariable("SPLUNK_ANSIBLE_POST_TASKS", lambda v: splitAndStrip(v, stringSeparator), ensureListValue(vars_scope.get("ansible_post_tasks"), stringSeparator))
     vars_scope["ansible_environment"] = vars_scope.get("ansible_environment") or {}
     env = os.environ.get("SPLUNK_ANSIBLE_ENV")
     if env:

roles/splunk_cluster_master/tasks/main.yml

@@ -1,14 +1,14 @@
 ---
 - include_tasks: ../../../roles/splunk_common/tasks/provision_apps.yml
   when:
-    - splunk.apps_location
+    - "'apps_location' in splunk and splunk.apps_location"
     - splunk.deployment_server is not defined or not splunk.deployment_server
 
 - include_tasks: ../../../roles/splunk_common/tasks/set_as_deployment_client.yml
   when: splunk.deployment_server is defined and splunk.deployment_server
 
 - include_tasks: prepare_apps_bundle.yml
-  when: splunk.apps_location
+  when: "'apps_location' in splunk and splunk.apps_location"
 
 - include_tasks: initialize_cluster_master.yml
 
@@ -17,10 +17,10 @@
 # This installed_apps list gets mutated by ESS bundle generation, hence we
 # need to refresh this array of apps before disabling them all
 - include_tasks: ../../../roles/splunk_common/tasks/find_installed_apps.yml
-  when: splunk.apps_location
+  when: "'apps_location' in splunk and splunk.apps_location"
 
 - include_tasks: ../../../roles/splunk_common/tasks/disable_installed_apps.yml
-  when: splunk.apps_location
+  when: "'apps_location' in splunk and splunk.apps_location"
 
 - include_tasks: setup_multisite.yml
   when:

roles/splunk_common/tasks/install_splunk_tgz.yml

@@ -9,6 +9,9 @@
     mode: 0666
   when: splunk.build_location is match("^(https?)://.*")
   register: download_result
+  until: download_result is succeeded
+  retries: 5
+  delay: "{{ retry_delay }}"
   become: yes
   become_user: "{{ privileged_user }}"
 

roles/splunk_deployer/tasks/main.yml

@@ -51,22 +51,20 @@
   delay: "{{ retry_delay }}"
   when:
     - splunk_search_head_cluster | bool
-    - splunk.apps_location
+    - "'apps_location' in splunk and splunk.apps_location"
 
 - include_tasks: ../../../roles/splunk_common/tasks/provision_apps.yml
   when:
-    - splunk.apps_location
+    - "'apps_location' in splunk and splunk.apps_location"
     - splunk.deployment_server is not defined or not splunk.deployment_server
 
 - include_tasks: ../../../roles/splunk_common/tasks/set_as_deployment_client.yml
   when: splunk.deployment_server is defined and splunk.deployment_server
 
 - include_tasks: push_apps_to_search_heads.yml
-  when:
-    - splunk.apps_location
+  when: "'apps_location' in splunk and splunk.apps_location"
 
 - include_tasks: ../../../roles/splunk_common/tasks/disable_installed_apps.yml
-  when:
-    - splunk.apps_location
+  when: "'apps_location' in splunk and splunk.apps_location"
 
 - include_tasks: ../../../roles/splunk_common/tasks/check_for_required_restarts.yml

roles/splunk_deployment_server/tasks/main.yml

@@ -2,8 +2,7 @@
 - include_tasks: ../../../roles/splunk_common/tasks/enable_forwarding.yml
 
 - include_tasks: create_deployment_apps.yml
-  when:
-    - splunk.apps_location
+  when: "'apps_location' in splunk and splunk.apps_location"
 
 - include_tasks: generate_server_classes.yml
 

roles/splunk_heavy_forwarder/tasks/main.yml

@@ -13,8 +13,7 @@
   when: splunk.add is defined
 
 - include_tasks: ../../../roles/splunk_common/tasks/provision_apps.yml
-  when:
-    - splunk.apps_location
+  when: "'apps_location' in splunk and splunk.apps_location"
 
 - name: Execute Splunk commands
   command: "{{ splunk.exec }} {{ item }} -auth {{ splunk.admin_user }}:{{ splunk.password }}"

roles/splunk_indexer/tasks/main.yml

@@ -12,7 +12,7 @@
 
 - include_tasks: ../../../roles/splunk_common/tasks/provision_apps.yml
   when:
-    - splunk.apps_location
+    - "'apps_location' in splunk and splunk.apps_location"
     - not splunk_indexer_cluster | bool
     - splunk.deployment_server is not defined or not splunk.deployment_server
 

roles/splunk_license_master/tasks/main.yml

@@ -2,8 +2,7 @@
 - include_tasks: ../../splunk_common/tasks/enable_forwarding.yml
 
 - include_tasks: ../../splunk_common/tasks/provision_apps.yml
-  when:
-    - splunk.apps_location
+  when: "'apps_location' in splunk and splunk.apps_location"
 
 - name: Enable license master local indexing
   ini_file:

roles/splunk_monitor/tasks/initialize_standalone_search_head.yml

@@ -9,7 +9,7 @@
 
 - include_tasks: ../../../roles/splunk_common/tasks/provision_apps.yml
   when:
-    - splunk.apps_location
+    - "'apps_location' in splunk and splunk.apps_location"
     - not splunk_search_head_cluster | bool
     - splunk.deployment_server is not defined or not splunk.deployment_server
 

roles/splunk_search_head/tasks/main.yml

@@ -44,7 +44,7 @@
 
 - include_tasks: ../../../roles/splunk_common/tasks/provision_apps.yml
   when:
-    - splunk.apps_location
+    - "'apps_location' in splunk and splunk.apps_location"
     - not splunk_search_head_cluster | bool
     - splunk.deployment_server is not defined or not splunk.deployment_server
 

roles/splunk_standalone/tasks/main.yml

@@ -36,7 +36,7 @@
 
 - include_tasks: ../../splunk_common/tasks/provision_apps.yml
   when:
-    - splunk.apps_location
+    - "'apps_location' in splunk and splunk.apps_location"
     - splunk.deployment_server is not defined or not splunk.deployment_server
 
 - include_tasks: ../../../roles/splunk_common/tasks/enable_dfs.yml

roles/splunk_universal_forwarder/tasks/main.yml

@@ -13,8 +13,7 @@
   when: splunk.add is defined
 
 - include_tasks: ../../../roles/splunk_common/tasks/provision_apps.yml
-  when:
-    - splunk.apps_location
+  when: "'apps_location' in splunk and splunk.apps_location"
 
 # Execute other Splunk commands
 - name: Execute Splunk commands

site.yml

@@ -8,13 +8,16 @@
 
     - block:
         - name: Execute pre-setup playbooks
+          loop: "{{ ansible_pre_tasks }}"
+          loop_control:
+            loop_var: ansible_pre_tasks_item
           include_tasks: execute_adhoc_plays.yml
           vars:
-            playbook: "{{ ansible_pre_tasks }}"
+            playbook: "{{ ansible_pre_tasks_item }}"
           when:
-            - ansible_pre_tasks is defined
-            - ansible_pre_tasks is not none
-            - ansible_pre_tasks is match("^(http|https|file)://.*")
+            - ansible_pre_tasks_item is defined
+            - ansible_pre_tasks_item is not none
+            - ansible_pre_tasks_item is match("^(http|https|file)://.*")
 
         - name: Provision role
           include_role:
@@ -23,13 +26,16 @@
             - splunk.role is defined
 
         - name: Execute post-setup playbooks
+          loop: "{{ ansible_post_tasks }}"
+          loop_control:
+            loop_var: ansible_post_tasks_item
           include_tasks: execute_adhoc_plays.yml
           vars:
-            playbook: "{{ ansible_post_tasks }}"
+            playbook: "{{ ansible_post_tasks_item }}"
           when:
-            - ansible_post_tasks is defined
-            - ansible_post_tasks is not none
-            - ansible_post_tasks is match("^(http|https|file)://.*")
+            - ansible_post_tasks_item is defined
+            - ansible_post_tasks_item is not none
+            - ansible_post_tasks_item is match("^(http|https|file)://.*")
 
         - name: Check all instances for required restarts
           include_tasks: ./roles/splunk_common/tasks/check_for_required_restarts.yml