Replace "whitelist/blacklist" with "allowed/blocked"

conf/whitelist.conf.example → conf/allowed_metrics.conf.example

@@ -1,5 +1,5 @@
 # This file takes a single regular expression per line
-# If USE_WHITELIST is set to True in carbon.conf, only metrics received which
+# If USE_METRIC_FILTERS is set to True in carbon.conf, only metrics received which
 # match one of these expressions will be persisted. If this file is empty or
 # missing, all metrics will pass through.
 # This file is reloaded automatically when changes are made

conf/blacklist.conf.example → conf/blocked_metrics.conf.example

@@ -1,5 +1,5 @@
 # This file takes a single regular expression per line
-# If USE_WHITELIST is set to True in carbon.conf, any metrics received which
+# If USE_METRIC_FILTERS is set to True in carbon.conf, any metrics received which
 # match one of these expressions will be dropped
 # This file is reloaded automatically when changes are made
 ^some\.noisy\.metric\.prefix\..*

conf/carbon.conf.example

@@ -14,12 +14,12 @@
 # To change other directory paths, add settings to this file. The following
 # configuration variables are available with these default values:
 #
-#   STORAGE_DIR    = $GRAPHITE_STORAGE_DIR
-#   LOCAL_DATA_DIR = %(STORAGE_DIR)s/whisper/
-#   WHITELISTS_DIR = %(STORAGE_DIR)s/lists/
-#   CONF_DIR       = %(STORAGE_DIR)s/conf/
-#   LOG_DIR        = %(STORAGE_DIR)s/log/
-#   PID_DIR        = %(STORAGE_DIR)s/
+#   STORAGE_DIR        = $GRAPHITE_STORAGE_DIR
+#   LOCAL_DATA_DIR     = %(STORAGE_DIR)s/whisper/
+#   METRIC_FILTERS_DIR = %(STORAGE_DIR)s/lists/
+#   CONF_DIR           = %(STORAGE_DIR)s/conf/
+#   LOG_DIR            = %(STORAGE_DIR)s/log/
+#   PID_DIR            = %(STORAGE_DIR)s/
 #
 # For FHS style directory structures, use:
 #
@@ -224,6 +224,11 @@ WHISPER_FALLOCATE_CREATE = True
 # page cache memory by posix_fadvise(2) with POSIX_FADVISE_RANDOM option.
 # WHISPER_FADVISE_RANDOM = False
 
+# Set this to True to enable metric filters with CONF_DIR/allowed_metrics.conf
+# and CONF_DIR/blocked_metrics.conf. If the allowed list is missing or empty,
+# all metrics will pass through.
+# USE_METRIC_FILTERS = False
+
 # By default all nodes stored in Ceres are cached in memory to improve the
 # throughput of reads and writes to underlying slices. Turning this off will
 # greatly reduce memory consumption for databases with millions of metrics, at
@@ -267,11 +272,6 @@ WHISPER_FALLOCATE_CREATE = True
 # multiple carbon-cache daemons are writing to the same files.
 # CERES_LOCK_WRITES = False
 
-# Set this to True to enable whitelisting and blacklisting of metrics in
-# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is
-# missing or empty, all metrics will pass through
-# USE_WHITELIST = False
-
 # By default, carbon itself will log statistics (such as a count,
 # metricsReceived) with the top level prefix of 'carbon' at an interval of 60
 # seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation
@@ -516,10 +516,10 @@ USE_FLOW_CONTROL = True
 # metrics have been sent in specified time in seconds
 #METRIC_CLIENT_IDLE_TIMEOUT = None
 
-# Set this to True to enable whitelisting and blacklisting of metrics in
-# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is
-# missing or empty, all metrics will pass through
-# USE_WHITELIST = False
+# Set this to True to enable metric filters with CONF_DIR/allowed_metrics.conf
+# and CONF_DIR/blocked_metrics.conf. If the allowed list is missing or empty,
+# all metrics will pass through.
+# USE_METRIC_FILTERS = False
 
 # By default, carbon itself will log statistics (such as a count,
 # metricsReceived) with the top level prefix of 'carbon' at an interval of 60
@@ -656,10 +656,10 @@ MAX_AGGREGATION_INTERVALS = 5
 # having to first wait rule.frequency seconds.
 # WRITE_BACK_FREQUENCY = 0
 
-# Set this to True to enable whitelisting and blacklisting of metrics in
-# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is
-# missing or empty, all metrics will pass through
-# USE_WHITELIST = False
+# Set this to True to enable metric filters with CONF_DIR/allowed_metrics.conf
+# and CONF_DIR/blocked_metrics.conf. If the allowed list is missing or empty,
+# all metrics will pass through.
+# USE_METRIC_FILTERS = False
 
 # By default, carbon itself will log statistics (such as a count,
 # metricsReceived) with the top level prefix of 'carbon' at an interval of 60

lib/carbon/conf.py

@@ -103,7 +103,7 @@
   DESTINATION_POOL_REPLICAS=False,
   USE_FLOW_CONTROL=True,
   USE_INSECURE_UNPICKLER=False,
-  USE_WHITELIST=False,
+  USE_METRIC_FILTERS=False,
   CARBON_METRIC_PREFIX='carbon',
   CARBON_METRIC_INTERVAL=60,
   CACHE_WRITE_STRATEGY='sorted',
@@ -228,8 +228,8 @@ class CarbonCacheOptions(usage.Options):
         ["config", "c", None, "Use the given config file."],
         ["instance", "", "a", "Manage a specific carbon instance."],
         ["logdir", "", None, "Write logs to the given directory."],
-        ["whitelist", "", None, "List of metric patterns to allow."],
-        ["blacklist", "", None, "List of metric patterns to disallow."],
+        ["allowed_metrics", "", None, "List of metric patterns to allow."],
+        ["blocked_metrics", "", None, "List of metric patterns to block."],
     ]
 
     def postOptions(self):
@@ -259,7 +259,7 @@ def cleanpath(path):
           return os.path.normpath(os.path.expanduser(path))
         settings["STORAGE_DIR"] = cleanpath(settings["STORAGE_DIR"])
         settings["LOCAL_DATA_DIR"] = cleanpath(settings["LOCAL_DATA_DIR"])
-        settings["WHITELISTS_DIR"] = cleanpath(settings["WHITELISTS_DIR"])
+        settings["METRIC_FILTERS_DIR"] = os.path.normpath(os.path.expanduser(settings["METRIC_FILTERS_DIR"]))
         settings["PID_DIR"] = cleanpath(settings["PID_DIR"])
         settings["LOG_DIR"] = cleanpath(settings["LOG_DIR"])
         settings["pidfile"] = cleanpath(settings["pidfile"])
@@ -319,13 +319,13 @@ def cleanpath(path):
                         os.chown(logdir, self.parent["uid"], self.parent["gid"])
                 log.logToDir(logdir)
 
-        if self["whitelist"] is None:
-            self["whitelist"] = join(settings["CONF_DIR"], "whitelist.conf")
-        settings["whitelist"] = self["whitelist"]
+        if self["allowed_metrics"] is None:
+            self["allowed_metrics"] = join(settings["METRIC_FILTERS_DIR"], "allowed_metrics.conf")
+        settings["allowed_metrics"] = self["allowed_metrics"]
 
-        if self["blacklist"] is None:
-            self["blacklist"] = join(settings["CONF_DIR"], "blacklist.conf")
-        settings["blacklist"] = self["blacklist"]
+        if self["blocked_metrics"] is None:
+            self["blocked_metrics"] = join(settings["METRIC_FILTERS_DIR"], "blocked_metrics.conf")
+        settings["blocked_metrics"] = self["blocked_metrics"]
 
     def parseArgs(self, *action):
         """If an action was provided, store it for further processing."""
@@ -525,13 +525,21 @@ def get_default_parser(usage="%prog [options] <start|stop|status>"):
         default=None,
         help="Use the given config file")
     parser.add_option(
-      "--whitelist",
+      "--allowed_metrics",
       default=None,
-      help="Use the given whitelist file")
+      help="Use the given allowed_metrics file")
     parser.add_option(
-      "--blacklist",
+      "--blocked_metrics",
       default=None,
-      help="Use the given blacklist file")
+      help="Use the given blocked_metrics file")
+    parser.add_option(
+      "--whitelist",    ##DEPRECATED
+      default=None,
+      help="DEPRECATED: Use the '--allowed_metrics' option")
+    parser.add_option(
+      "--blacklist",    ##DEPRECATED
+      default=None,
+      help="DEPRECATED: Use the ''--blocked_metrics' option")
     parser.add_option(
         "--logdir",
         default=None,
@@ -636,7 +644,7 @@ def update_STORAGE_DIR_deps():
         settings.setdefault(
             "LOCAL_DATA_DIR", join(settings["STORAGE_DIR"], "whisper"))
         settings.setdefault(
-            "WHITELISTS_DIR", join(settings["STORAGE_DIR"], "lists"))
+            "METRIC_FILTERS_DIR", join(settings["STORAGE_DIR"], "lists"))
 
     # Read configuration options from program-specific section.
     section = program[len("carbon-"):]

lib/carbon/instrumentation.py

@@ -148,8 +148,8 @@ def recordMetrics():
   # common metrics
   record('activeConnections', len(state.connectedMetricReceiverProtocols))
   record('metricsReceived', myStats.get('metricsReceived', 0))
-  record('blacklistMatches', myStats.get('blacklistMatches', 0))
-  record('whitelistRejects', myStats.get('whitelistRejects', 0))
+  record('blocked_metricsMatches', myStats.get('blocked_metricsMatches', 0))
+  record('allowed_metricsRejects', myStats.get('allowed_metricsRejects', 0))
   record('cpuUsage', getCpuUsage())
 
   # And here preserve count of messages received in the prior period

lib/carbon/protocols.py

@@ -11,7 +11,7 @@
 from twisted.protocols.policies import TimeoutMixin
 from carbon import log, events, state, management
 from carbon.conf import settings
-from carbon.regexlist import WhiteList, BlackList
+from carbon.regexlist import allowed_metrics, blocked_metrics
 from carbon.util import pickle, get_unpickler
 from carbon.util import PluginRegistrar
 from six import with_metaclass
@@ -166,17 +166,18 @@ def connectionLost(self, reason):
       events.resumeReceivingMetrics.removeHandler(self.resumeReceiving)
 
   def metricReceived(self, metric, datapoint):
-    if BlackList and metric in BlackList:
-      instrumentation.increment('blacklistMatches')
+    if blocked_metrics and metric in blocked_metrics:
+      instrumentation.increment('blocked_metricsMatches')
       return
-    if WhiteList and metric not in WhiteList:
-      instrumentation.increment('whitelistRejects')
+    if allowed_metrics and metric not in allowed_metrics:
+      instrumentation.increment('allowed_metricsRejects')
       return
     if datapoint[1] != datapoint[1]:  # filter out NaN values
       return
     # use current time if none given: https://github.com/graphite-project/carbon/issues/54
     if int(datapoint[0]) == -1:
       datapoint = (time.time(), datapoint[1])
+
     res = settings.MIN_TIMESTAMP_RESOLUTION
     if res:
       datapoint = (int(datapoint[0]) // res * res, datapoint[1])

lib/carbon/regexlist.py

@@ -5,7 +5,7 @@
 
 
 class RegexList:
-  """ Maintain a list of regex for matching whitelist and blacklist """
+  """ Maintain a list of regex for matching allowed and blocked metrics. """
 
   def __init__(self):
     self.regex_list = []
@@ -59,5 +59,11 @@ def __nonzero__(self):
   __bool__ = __nonzero__  # py2/3 compatibility
 
 
-WhiteList = RegexList()
-BlackList = RegexList()
+"""
+##DEPRECATION: These were originally called `WhiteList` and `BlackList`.  The
+new names might fail some styleguide opinions.  We can change them later.
+"""
+allowed_metrics = RegexList()
+blocked_metrics = RegexList()
+WhiteList = RegexList()     ##DEPRECATING
+BlackList = RegexList()     ##DEPRECATING

lib/carbon/service.py

@@ -55,10 +55,21 @@ def createBaseService(config, settings):
     root_service = CarbonRootService()
     root_service.setName(settings.program)
 
-    if settings.USE_WHITELIST:
-      from carbon.regexlist import WhiteList, BlackList
-      WhiteList.read_from(settings.whitelist)
-      BlackList.read_from(settings.blacklist)
+    if settings.USE_WHITELIST:  # DEPRECATING per https://github.com/graphite-project/carbon/issues/567
+        settings.USE_METRIC_FILTERS = True
+        log.msg("NOTE: 'USE_WHITELIST' is being deprecated in favor of 'USE_METRIC_FILTERS'.")
+        WhiteList.read_from(settings.whitelist)     #DEPRECATING
+        BlackList.read_from(settings.blacklist)     #DEPRECATING
+
+    if settings.USE_METRIC_FILTERS:
+        from carbon.regexlist import allowed_metrics, blocked_metrics
+        allowed_metrics.read_from(settings.allowed_metrics)
+        blocked_metrics.read_from(settings.blocked_metrics)
+        # I hope that this does what I think.
+        if WhiteList:                               #DEPRECATING
+            allowed_metrics.update(WhiteList)       #DEPRECATING
+        if BlackList:                               #DEPRECATING
+            blocked_metrics.update(BlackList)       #DEPRECATING
 
     # Instantiate an instrumentation service that will record metrics about
     # this service.

lib/carbon/tests/test_conf.py

@@ -417,9 +417,9 @@ def test_local_data_dir_depends_on_storage_dir(self):
             ROOT_DIR="foo")
         self.assertEqual(join("bar", "whisper"), settings.LOCAL_DATA_DIR)
 
-    def test_whitelists_dir_depends_on_storage_dir(self):
+    def test_metric_filters_dir_depends_on_storage_dir(self):
         """
-        Tests 'STORAGE_DIR' dependency 'WHITELISTS_DIR'
+        Tests 'STORAGE_DIR' dependency 'METRIC_FILTERS_DIR'
         """
         config = self.makeFile(
             content=("[foo]\n"
@@ -429,4 +429,4 @@ def test_whitelists_dir_depends_on_storage_dir(self):
             FakeOptions(config=config, instance=None,
                         pidfile=None, logdir=None),
             ROOT_DIR="foo")
-        self.assertEqual(join("bar", "lists"), settings.WHITELISTS_DIR)
+        self.assertEqual(join("bar", "lists"), settings.METRIC_FILTERS_DIR)