feat: Support SHC (#120)

This feature adds support for SHC using an in-development feature of Splunk Enterprise/Cloud that is disabled by default. ```[global]/allowRestReplay=true``` must be enabled per Splunk Documentation as of Splunk 8.2 "Do not enable it without consulting Splunk support." Do not use with versions prior to 8.2

.reuse/dep5

@@ -73,6 +73,14 @@ Files: package/default/data/ui/views/*
 Copyright: $YEAR $NAME <$CONTACT>
 License: Apache-2.0
 
-Files: package/bin/geoipupdate/linux_amd64/*
-Copyright: 2018 - 2020 by MaxMind, Inc.
-License: MIT
+Files: *.lock
+Copyright: $YEAR $NAME <$CONTACT>
+License: Apache-2.0
+
+Files: README*
+Copyright: $YEAR $NAME <$CONTACT>
+License: Apache-2.0
+
+Files: semtag
+Copyright: $YEAR $NAME <$CONTACT>
+License: Apache-2.0

package/bin/SecKit_SA_geolocation_rh_updater.py

@@ -0,0 +1,162 @@
+# SPDX-FileCopyrightText: 2020 Splunk Inc (Ryan Faircloth) <[email protected]>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+This controller does the update
+"""
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+import tempfile
+from os.path import dirname
+
+from splunk import AuthorizationFailed, ResourceNotFound
+from splunk.clilib.bundle_paths import make_splunkhome_path
+from splunk.rest import simpleRequest
+
+ta_name = "SecKit_SA_geolocation"
+pattern = re.compile(r"[\\/]etc[\\/]apps[\\/][^\\/]+[\\/]bin[\\/]?$")
+new_paths = [path for path in sys.path if not pattern.search(path) or ta_name in path]
+new_paths.append(os.path.join(dirname(dirname(__file__)), "lib"))
+new_paths.insert(0, os.path.sep.join([os.path.dirname(__file__), ta_name]))
+sys.path = new_paths
+
+from seckit_helpers import rest_handler
+
+
+def setup_logger(level):
+    """
+    Setup a logger for the REST handler
+    """
+
+    logger = logging.getLogger(
+        "splunk.appserver.SecKit_SA_geolocation_rh_updater.handler"
+    )
+    logger.propagate = (
+        False  # Prevent the log messages from being duplicated in the python.log file
+    )
+    logger.setLevel(level)
+
+    log_file_path = make_splunkhome_path(
+        ["var", "log", "splunk", "SecKit_SA_geolocation_rh_updater.log"]
+    )
+    file_handler = logging.handlers.RotatingFileHandler(
+        log_file_path, maxBytes=25000000, backupCount=5
+    )
+
+    formatter = logging.Formatter(
+        "%(asctime)s %(levelname)s pid=%(process)d tid=%(threadName)s "
+        "file=%(filename)s:%(funcName)s:%(lineno)d | %(message)s"
+    )
+    file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+    return logger
+
+
+logger = setup_logger(logging.INFO)
+
+
+class GeoipUpdateHandler(rest_handler.RESTHandler):
+    """
+    This is a REST handler that supports backing up lookup files.
+    This is broken out as a separate handler so that this handler can be replayed on other search
+    heads via the allowRestReplay setting in restmap.conf.
+    """
+
+    def __init__(self, command_line, command_arg):
+        super().__init__(command_line, command_arg, logger)
+
+    def post_update(self, request_info, id, token, db, proxy_settings=None, **kwargs):
+        logger.info("Asked to update")
+        # in_string = "sdfsd"
+        logger.info(f"Asked to update {request_info}")
+        logger.info(f"Asked to update id={id} token={token} db={db}")
+        # payload = json.loads(request["payload"])
+
+        try:
+            proxy_settings = {}
+            logger.info("Trying to update")
+            with tempfile.NamedTemporaryFile(
+                mode="w", suffix=".conf", prefix="GeoIP"
+            ) as file:
+                file.write("\nAccountID " + id)
+                file.write("\nLicenseKey " + token)
+                file.write("\nEditionIDs " + db)
+
+                if proxy_settings == {}:
+                    logger.debug("no proxy")
+                else:
+                    file.write(
+                        "\nProxy "
+                        + proxy_settings["proxy_url"]
+                        + ":"
+                        + proxy_settings["proxy_port"]
+                    )
+                    if not proxy_settings["proxy_username"] is None:
+                        file.write(
+                            "\nProxyUserPassword "
+                            + proxy_settings["proxy_username"]
+                            + ":"
+                            + proxy_settings["proxy_password"]
+                        )
+
+                file.flush()
+                guargs = str(
+                    os.path.expandvars(
+                        "-v -d $SPLUNK_HOME/etc/apps/SecKit_SA_geolocation/data/ -f "
+                        + file.name
+                    )
+                )
+
+                try:
+                    subprocess.check_output(
+                        [
+                            "$SPLUNK_HOME/etc/apps/SecKit_SA_geolocation/bin/geoipupdate/linux_amd64/geoipupdate "
+                            + guargs
+                        ],
+                        shell=True,
+                        stderr=subprocess.STDOUT,
+                    )  # nosemgrep: python.lang.security.audit.dangerous-subprocess-use
+                except CalledProcessError as e:
+                    logger.exception(e)
+                    logger.error("command args:\n")
+                    logger.error(e.cmd)
+                    logger.error("output:\n")
+                    logger.error(e.output.decode("utf-8"))
+                    sys.exit(1)
+
+                mmdb_dir = os.path.expandvars(
+                    "$SPLUNK_HOME/etc/apps/SecKit_SA_geolocation/data/"
+                )
+                files = os.listdir(mmdb_dir)
+                for name in files:
+                    if name.endswith(".mmdb"):
+                        inode = os.stat(os.path.join(mmdb_dir, name))
+                        logger.info(
+                            "mmdb="
+                            + name
+                            + " size="
+                            + str(inode.st_size)
+                            + " mtime="
+                            + str(inode.st_mtime)
+                        )
+
+            # Everything worked, return accordingly
+            return {
+                "payload": "done",  # Payload of the request.
+                "status": 200,  # HTTP status code
+            }
+
+        except:
+
+            logger.exception(
+                "Exception generated when attempting to backup a lookup file"
+            )
+            return {
+                "payload": "Failed",  # Payload of the request.
+                "status": 500,  # HTTP status code
+            }

package/bin/SecKit_geo_lookup.py

@@ -1,14 +1,14 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # SPDX-FileCopyrightText: 2020 Splunk Inc (Ryan Faircloth) <[email protected]>
 #
 # SPDX-License-Identifier: Apache-2.0
 
-import import_declare_test
-import os.path
 import csv
+import os.path
 import sys
-import geoip2.database
 
+import import_declare_test
+import geoip2.database
 
 """ An adapter that takes an ip as input and produces gelocatin data
     based on the max mind data sets
@@ -17,7 +17,7 @@
 
 module_dir = os.path.dirname(os.path.realpath(__file__))
 app_dir = os.path.abspath(os.path.join(module_dir, os.pardir))
-data_path = os.path.join(app_dir, 'data')
+data_path = os.path.join(app_dir, "data")
 
 
 def main():
@@ -28,80 +28,86 @@ def main():
     outfile = sys.stdout
 
     r = csv.DictReader(infile)
-    header = r.fieldnames
+    r.fieldnames
 
     w = csv.DictWriter(outfile, fieldnames=r.fieldnames)
     w.writeheader()
 
     for result in r:
         try:
             try:
-                city2_file = os.path.join(data_path, 'GeoIP2-City.mmdb')
-                city2lite_file = os.path.join(data_path, 'GeoLite2-City.mmdb')
-                if (os.path.isfile(city2_file)):
+                city2_file = os.path.join(data_path, "GeoIP2-City.mmdb")
+                city2lite_file = os.path.join(data_path, "GeoLite2-City.mmdb")
+                if os.path.isfile(city2_file):
                     city2reader = geoip2.database.Reader(city2_file)
-                elif (os.path.isfile(city2lite_file)):
+                elif os.path.isfile(city2lite_file):
                     city2reader = geoip2.database.Reader(city2lite_file)
                 else:
                     city2reader = None
 
                 if not city2reader is None:
                     city2response = city2reader.city(result[ipfield])
-                    result['country'] = city2response.country.iso_code
-                    result['city'] = city2response.city.name
-                    result['lat'] = city2response.location.latitude
-                    result['long'] = city2response.location.longitude
+                    result["country"] = city2response.country.iso_code
+                    result["city"] = city2response.city.name
+                    result["lat"] = city2response.location.latitude
+                    result["long"] = city2response.location.longitude
             except geoip2.errors.AddressNotFoundError:
-                donothing = ""
+                pass
 
             try:
-                ct_file = os.path.join(data_path, 'GeoIP2-Connection-Type.mmdb')
-                if (os.path.isfile(ct_file)):
+                ct_file = os.path.join(data_path, "GeoIP2-Connection-Type.mmdb")
+                if os.path.isfile(ct_file):
                     ctreader = geoip2.database.Reader(ct_file)
                     ct_response = ctreader.connection_type(result[ipfield])
-                    result['connection_type'] = ct_response.connection_type
-                    result['network'] = ct_response.network
+                    result["connection_type"] = ct_response.connection_type
+                    result["network"] = ct_response.network
             except geoip2.errors.AddressNotFoundError:
-                donothing = ""
-                
+                pass
+
             try:
-                asn_file = os.path.join(data_path, 'GeoLite2-ASN.mmdb')
-                if (os.path.isfile(asn_file)):
+                asn_file = os.path.join(data_path, "GeoLite2-ASN.mmdb")
+                if os.path.isfile(asn_file):
                     asnreader = geoip2.database.Reader(asn_file)
                     asn_response = asnreader.asn(result[ipfield])
-                    result['isp_ip'] = asn_response.ip_address
-                    result['isp_asn'] = asn_response.autonomous_system_number
-                    result['isp_asn_organization'] = asn_response.autonomous_system_organization
+                    result["isp_ip"] = asn_response.ip_address
+                    result["isp_asn"] = asn_response.autonomous_system_number
+                    result[
+                        "isp_asn_organization"
+                    ] = asn_response.autonomous_system_organization
             except geoip2.errors.AddressNotFoundError:
-                donothing = ""
+                pass
 
             try:
-                isp_file = os.path.join(data_path, 'GeoIP2-ISP.mmdb')
-                if (os.path.isfile(isp_file)):
+                isp_file = os.path.join(data_path, "GeoIP2-ISP.mmdb")
+                if os.path.isfile(isp_file):
                     ispreader = geoip2.database.Reader(isp_file)
                     isp_response = ispreader.isp(result[ipfield])
-                    result['isp'] = isp_response.isp
-                    result['isp_ip'] = isp_response.ip_address
-                    result['isp_asn'] = isp_response.autonomous_system_number
-                    result['isp_asn_organization'] = isp_response.autonomous_system_organization
+                    result["isp"] = isp_response.isp
+                    result["isp_ip"] = isp_response.ip_address
+                    result["isp_asn"] = isp_response.autonomous_system_number
+                    result[
+                        "isp_asn_organization"
+                    ] = isp_response.autonomous_system_organization
             except geoip2.errors.AddressNotFoundError:
-                donothing = ""
+                pass
 
             try:
-                anon_file = os.path.join(data_path, 'GeoIP2-Anonymous-IP.mmdb')
-                if (os.path.isfile(anon_file)):
+                anon_file = os.path.join(data_path, "GeoIP2-Anonymous-IP.mmdb")
+                if os.path.isfile(anon_file):
                     anonreader = geoip2.database.Reader(anon_file)
                     anon_response = anonreader.anonymous_ip(result[ipfield])
-                    result['is_anonymous'] = anon_response.is_anonymous
-                    result['is_anonymous_proxy'] = anon_response.is_anonymous_proxy                 
-                    result['is_anonymous_vpn'] = anon_response.is_anonymous_vpn
-                    result['is_hosting_provider'] = anon_response.is_hosting_provider
-                    result['is_legitimate_proxy'] = anon_response.is_legitimate_proxy
-                    result['is_public_proxy'] = anon_response.is_public_proxy
-                    result['is_satellite_provider'] = anon_response.is_satellite_provider
-                    result['is_tor_exit_node'] = anon_response.is_tor_exit_node
+                    result["is_anonymous"] = anon_response.is_anonymous
+                    result["is_anonymous_proxy"] = anon_response.is_anonymous_proxy
+                    result["is_anonymous_vpn"] = anon_response.is_anonymous_vpn
+                    result["is_hosting_provider"] = anon_response.is_hosting_provider
+                    result["is_legitimate_proxy"] = anon_response.is_legitimate_proxy
+                    result["is_public_proxy"] = anon_response.is_public_proxy
+                    result[
+                        "is_satellite_provider"
+                    ] = anon_response.is_satellite_provider
+                    result["is_tor_exit_node"] = anon_response.is_tor_exit_node
             except geoip2.errors.AddressNotFoundError:
-                donothing = ""
+                pass
 
         except ValueError:
             pass