diff --git a/conf/whitelist.conf.example b/conf/allowed_metrics.conf.example similarity index 74% rename from conf/whitelist.conf.example rename to conf/allowed_metrics.conf.example index f84895d0a..6a0553620 100644 --- a/conf/whitelist.conf.example +++ b/conf/allowed_metrics.conf.example @@ -1,5 +1,5 @@ # This file takes a single regular expression per line -# If USE_WHITELIST is set to True in carbon.conf, only metrics received which +# If USE_METRIC_FILTERS is set to True in carbon.conf, only metrics received which # match one of these expressions will be persisted. If this file is empty or # missing, all metrics will pass through. # This file is reloaded automatically when changes are made diff --git a/conf/blacklist.conf.example b/conf/blocked_metrics.conf.example similarity index 83% rename from conf/blacklist.conf.example rename to conf/blocked_metrics.conf.example index 8038a7ef1..36cd8d93c 100644 --- a/conf/blacklist.conf.example +++ b/conf/blocked_metrics.conf.example @@ -1,5 +1,5 @@ # This file takes a single regular expression per line -# If USE_WHITELIST is set to True in carbon.conf, any metrics received which +# If USE_METRIC_FILTERS is set to True in carbon.conf, any metrics received which # match one of these expressions will be dropped # This file is reloaded automatically when changes are made ^some\.noisy\.metric\.prefix\..* diff --git a/conf/carbon.conf.example b/conf/carbon.conf.example index 4465fb4dc..c517695dc 100644 --- a/conf/carbon.conf.example +++ b/conf/carbon.conf.example @@ -14,12 +14,12 @@ # To change other directory paths, add settings to this file. The following # configuration variables are available with these default values: # -# STORAGE_DIR = $GRAPHITE_STORAGE_DIR -# LOCAL_DATA_DIR = %(STORAGE_DIR)s/whisper/ -# WHITELISTS_DIR = %(STORAGE_DIR)s/lists/ -# CONF_DIR = %(STORAGE_DIR)s/conf/ -# LOG_DIR = %(STORAGE_DIR)s/log/ -# PID_DIR = %(STORAGE_DIR)s/ +# STORAGE_DIR = $GRAPHITE_STORAGE_DIR +# LOCAL_DATA_DIR = %(STORAGE_DIR)s/whisper/ +# USE_METRIC_FILTERS = %(STORAGE_DIR)s/lists/ +# CONF_DIR = %(STORAGE_DIR)s/conf/ +# LOG_DIR = %(STORAGE_DIR)s/log/ +# PID_DIR = %(STORAGE_DIR)s/ # # For FHS style directory structures, use: # @@ -112,7 +112,7 @@ CACHE_QUERY_PORT = 7002 USE_FLOW_CONTROL = True # If enabled this setting is used to timeout metric client connection if no -# metrics have been sent in specified time in seconds +# metrics have been sent in specified time in seconds #METRIC_CLIENT_IDLE_TIMEOUT = None # By default, carbon-cache will log every whisper update and cache hit. @@ -176,14 +176,14 @@ WHISPER_AUTOFLUSH = False # On systems which has a large number of metrics, an amount of Whisper write(2)'s # pageback sometimes cause disk thrashing due to memory shortage, so that abnormal -# disk reads occur. Enabling this option makes it possible to decrease useless +# disk reads occur. Enabling this option makes it possible to decrease useless # page cache memory by posix_fadvise(2) with POSIX_FADVISE_RANDOM option. # WHISPER_FADVISE_RANDOM = False -# Set this to True to enable whitelisting and blacklisting of metrics in -# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is -# missing or empty, all metrics will pass through -# USE_WHITELIST = False +# Set this to True to enable control over what metrics are allowed to be handled +# CONF_DIR/allowed_metrics.conf and CONF_DIR/blocked_metrics.conf. If the +# allowed list is missing or empty, all metrics will pass through. +# USE_METRIC_FILTERS = False # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 @@ -269,7 +269,7 @@ REPLICATION_FACTOR = 1 # For REPLICATION_FACTOR >=2, set DIVERSE_REPLICAS to True to guarantee replicas # across distributed hosts. With this setting disabled, it's possible that replicas -# may be sent to different caches on the same host. This has been the default +# may be sent to different caches on the same host. This has been the default # behavior since introduction of 'consistent-hashing' relay method. # Note that enabling this on an existing pre-0.9.14 cluster will require rebalancing # your metrics across the cluster nodes using a tool like Carbonate. @@ -336,13 +336,13 @@ TIME_TO_DEFER_SENDING = 0.0001 USE_FLOW_CONTROL = True # If enabled this setting is used to timeout metric client connection if no -# metrics have been sent in specified time in seconds +# metrics have been sent in specified time in seconds #METRIC_CLIENT_IDLE_TIMEOUT = None -# Set this to True to enable whitelisting and blacklisting of metrics in -# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is -# missing or empty, all metrics will pass through -# USE_WHITELIST = False +# Set this to True to enable control over what metrics are allowed to be handled +# CONF_DIR/allowed_metrics.conf and CONF_DIR/blocked_metrics.conf. If the +# allowed list is missing or empty, all metrics will pass through. +# USE_METRIC_FILTERS = False # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 @@ -434,7 +434,7 @@ MAX_QUEUE_SIZE = 10000 USE_FLOW_CONTROL = True # If enabled this setting is used to timeout metric client connection if no -# metrics have been sent in specified time in seconds +# metrics have been sent in specified time in seconds #METRIC_CLIENT_IDLE_TIMEOUT = None # This defines the maximum "message size" between carbon daemons. @@ -460,10 +460,10 @@ MAX_AGGREGATION_INTERVALS = 5 # having to first wait rule.frequency seconds. # WRITE_BACK_FREQUENCY = 0 -# Set this to True to enable whitelisting and blacklisting of metrics in -# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is -# missing or empty, all metrics will pass through -# USE_WHITELIST = False +# Set this to True to enable control over what metrics are allowed to be handled +# CONF_DIR/allowed_metrics.conf and CONF_DIR/blocked_metrics.conf. If the +# allowed list is missing or empty, all metrics will pass through. +# USE_METRIC_FILTERS = False # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 diff --git a/lib/carbon/conf.py b/lib/carbon/conf.py index bd4d41261..1ba253567 100644 --- a/lib/carbon/conf.py +++ b/lib/carbon/conf.py @@ -73,7 +73,7 @@ DESTINATIONS=[], USE_FLOW_CONTROL=True, USE_INSECURE_UNPICKLER=False, - USE_WHITELIST=False, + USE_METRIC_FILTERS=False, CARBON_METRIC_PREFIX='carbon', CARBON_METRIC_INTERVAL=60, CACHE_WRITE_STRATEGY='sorted', @@ -191,8 +191,8 @@ class CarbonCacheOptions(usage.Options): ["config", "c", None, "Use the given config file."], ["instance", "", "a", "Manage a specific carbon instance."], ["logdir", "", None, "Write logs to the given directory."], - ["whitelist", "", None, "List of metric patterns to allow."], - ["blacklist", "", None, "List of metric patterns to disallow."], + ["allowed_metrics", "", None, "List of metric patterns to allow."], + ["blocked_metrics", "", None, "List of metric patterns to disallow."], ] def postOptions(self): @@ -220,7 +220,7 @@ def postOptions(self): # Normalize and expand paths settings["STORAGE_DIR"] = os.path.normpath(os.path.expanduser(settings["STORAGE_DIR"])) settings["LOCAL_DATA_DIR"] = os.path.normpath(os.path.expanduser(settings["LOCAL_DATA_DIR"])) - settings["WHITELISTS_DIR"] = os.path.normpath(os.path.expanduser(settings["WHITELISTS_DIR"])) + settings["METRIC_FILTERS_DIR"] = os.path.normpath(os.path.expanduser(settings["METRIC_FILTERS_DIR"])) settings["PID_DIR"] = os.path.normpath(os.path.expanduser(settings["PID_DIR"])) settings["LOG_DIR"] = os.path.normpath(os.path.expanduser(settings["LOG_DIR"])) settings["pidfile"] = os.path.normpath(os.path.expanduser(settings["pidfile"])) @@ -272,13 +272,13 @@ def postOptions(self): os.chown(logdir, self.parent["uid"], self.parent["gid"]) log.logToDir(logdir) - if self["whitelist"] is None: - self["whitelist"] = join(settings["CONF_DIR"], "whitelist.conf") - settings["whitelist"] = self["whitelist"] + if self["allowed_metrics"] is None: + self["allowed_metrics"] = join(settings["CONF_DIR"], "allowed_metrics.conf") + settings["allowed_metrics"] = self["allowed_metrics"] - if self["blacklist"] is None: - self["blacklist"] = join(settings["CONF_DIR"], "blacklist.conf") - settings["blacklist"] = self["blacklist"] + if self["blocked_metrics"] is None: + self["blocked_metrics"] = join(settings["CONF_DIR"], "blocked_metrics.conf") + settings["blocked_metrics"] = self["blocked_metrics"] def parseArgs(self, *action): """If an action was provided, store it for further processing.""" @@ -453,13 +453,21 @@ def get_default_parser(usage="%prog [options] "): default=None, help="Use the given config file") parser.add_option( - "--whitelist", + "--allowed_metrics", default=None, - help="Use the given whitelist file") + help="Use the given allowed_metrics file") parser.add_option( - "--blacklist", + "--blocked_metrics", default=None, - help="Use the given blacklist file") + help="Use the given blocked_metrics file") + parser.add_option( + "--whitelist", ##DEPRECATED + default=None, + help="DEPRECATED: Use the '--allowed_metrics' option") + parser.add_option( + "--blacklist", ##DEPRECATED + default=None, + help="DEPRECATED: Use the ''--blocked_metrics' option") parser.add_option( "--logdir", default=None, @@ -555,7 +563,7 @@ def read_config(program, options, **kwargs): settings.setdefault( "LOCAL_DATA_DIR", join(settings["STORAGE_DIR"], "whisper")) settings.setdefault( - "WHITELISTS_DIR", join(settings["STORAGE_DIR"], "lists")) + "METRIC_FILTERS_DIR", join(settings["STORAGE_DIR"], "lists")) # Read configuration options from program-specific section. section = program[len("carbon-"):] diff --git a/lib/carbon/instrumentation.py b/lib/carbon/instrumentation.py index d953ed588..04a15c8c1 100644 --- a/lib/carbon/instrumentation.py +++ b/lib/carbon/instrumentation.py @@ -137,8 +137,8 @@ def recordMetrics(): # common metrics record('activeConnections', len(state.connectedMetricReceiverProtocols)) record('metricsReceived', myStats.get('metricsReceived', 0)) - record('blacklistMatches', myStats.get('blacklistMatches', 0)) - record('whitelistRejects', myStats.get('whitelistRejects', 0)) + record('blocked_metricsMatches', myStats.get('blocked_metricsMatches', 0)) + record('allowed_metricsRejects', myStats.get('allowed_metricsRejects', 0)) record('cpuUsage', getCpuUsage()) # And here preserve count of messages received in the prior periiod diff --git a/lib/carbon/protocols.py b/lib/carbon/protocols.py index d37c37e5f..84ab33a99 100644 --- a/lib/carbon/protocols.py +++ b/lib/carbon/protocols.py @@ -7,7 +7,7 @@ from twisted.protocols.policies import TimeoutMixin from carbon import log, events, state, management from carbon.conf import settings -from carbon.regexlist import WhiteList, BlackList +from carbon.regexlist import allowed_metrics, blocked_metrics from carbon.util import pickle, get_unpickler @@ -56,17 +56,17 @@ def connectionLost(self, reason): events.resumeReceivingMetrics.removeHandler(self.resumeReceiving) def metricReceived(self, metric, datapoint): - if BlackList and metric in BlackList: - instrumentation.increment('blacklistMatches') + if blocked_metrics and metric in blocked_metrics: + instrumentation.increment('blocked_metricsMatches') return - if WhiteList and metric not in WhiteList: - instrumentation.increment('whitelistRejects') + if allowed_metrics and metric not in allowed_metrics: + instrumentation.increment('allowed_metricsRejects') return if datapoint[1] != datapoint[1]: # filter out NaN values return if int(datapoint[0]) == -1: # use current time if none given: https://github.com/graphite-project/carbon/issues/54 datapoint = (time.time(), datapoint[1]) - + events.metricReceived(metric, datapoint) self.resetTimeout() diff --git a/lib/carbon/regexlist.py b/lib/carbon/regexlist.py index 446d9db27..8017a66c0 100644 --- a/lib/carbon/regexlist.py +++ b/lib/carbon/regexlist.py @@ -6,7 +6,7 @@ class RegexList: - """ Maintain a list of regex for matching whitelist and blacklist """ + """ Maintain a list of regex for matching allowed and blocked metrics. """ def __init__(self): self.regex_list = [] @@ -58,5 +58,11 @@ def __nonzero__(self): return bool(self.regex_list) -WhiteList = RegexList() -BlackList = RegexList() +""" +##DEPRECATION: These were originally called `WhiteList` and `BlackList`. The +new names might fail some styleguide opinions. We can change them later. +""" +allowed_metrics = RegexList() +blocked_metrics = RegexList() +WhiteList = RegexList() ##DEPRECATING +BlackList = RegexList() ##DEPRECATING diff --git a/lib/carbon/service.py b/lib/carbon/service.py index 34d52580c..f61b41b3b 100644 --- a/lib/carbon/service.py +++ b/lib/carbon/service.py @@ -52,10 +52,21 @@ def createBaseService(config, settings): root_service = CarbonRootService() root_service.setName(settings.program) - if settings.USE_WHITELIST: - from carbon.regexlist import WhiteList, BlackList - WhiteList.read_from(settings.whitelist) - BlackList.read_from(settings.blacklist) + if settings.USE_WHITELIST: # DEPRECATING per https://github.com/graphite-project/carbon/issues/567 + settings.USE_METRIC_FILTERS = True + log.msg("NOTE: 'USE_WHITELIST' is being deprecated in favor of 'USE_METRIC_FILTERS'.") + WhiteList.read_from(settings.whitelist) #DEPRECATING + BlackList.read_from(settings.blacklist) #DEPRECATING + + if settings.USE_METRIC_FILTERS: + from carbon.regexlist import allowed_metrics, blocked_metrics + allowed_metrics.read_from(settings.allowed_metrics) + blocked_metrics.read_from(settings.blocked_metrics) + # I hope that this does what I think. + if WhiteList: #DEPRECATING + allowed_metrics.update(WhiteList) #DEPRECATING + if BlackList: #DEPRECATING + blocked_metrics.update(BlackList) #DEPRECATING # Instantiate an instrumentation service that will record metrics about # this service.