From 0191348a12a8eb26c8364282daf1f5ae079a0fd2 Mon Sep 17 00:00:00 2001 From: H Waldo G Date: Sat, 2 Jul 2016 16:52:36 -0400 Subject: [PATCH 01/17] renaming example allow|block lists example files --- conf/{whitelist.conf.example => allowed_metrics.conf.example} | 0 conf/{blacklist.conf.example => blocked_metrics.conf.example} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename conf/{whitelist.conf.example => allowed_metrics.conf.example} (100%) rename conf/{blacklist.conf.example => blocked_metrics.conf.example} (100%) diff --git a/conf/whitelist.conf.example b/conf/allowed_metrics.conf.example similarity index 100% rename from conf/whitelist.conf.example rename to conf/allowed_metrics.conf.example diff --git a/conf/blacklist.conf.example b/conf/blocked_metrics.conf.example similarity index 100% rename from conf/blacklist.conf.example rename to conf/blocked_metrics.conf.example From a66dc2053b6529d037f6fc02c83869cb0d945f56 Mon Sep 17 00:00:00 2001 From: H Waldo G Date: Sat, 2 Jul 2016 17:00:15 -0400 Subject: [PATCH 02/17] changed name of config item from 'USE_WHITELIST' to 'USE_METRICSLIST' --- conf/allowed_metrics.conf.example | 2 +- conf/blocked_metrics.conf.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/allowed_metrics.conf.example b/conf/allowed_metrics.conf.example index f84895d0a..322cdf823 100644 --- a/conf/allowed_metrics.conf.example +++ b/conf/allowed_metrics.conf.example @@ -1,5 +1,5 @@ # This file takes a single regular expression per line -# If USE_WHITELIST is set to True in carbon.conf, only metrics received which +# If USE_METRICSLIST is set to True in carbon.conf, only metrics received which # match one of these expressions will be persisted. If this file is empty or # missing, all metrics will pass through. # This file is reloaded automatically when changes are made diff --git a/conf/blocked_metrics.conf.example b/conf/blocked_metrics.conf.example index 8038a7ef1..4b738da2e 100644 --- a/conf/blocked_metrics.conf.example +++ b/conf/blocked_metrics.conf.example @@ -1,5 +1,5 @@ # This file takes a single regular expression per line -# If USE_WHITELIST is set to True in carbon.conf, any metrics received which +# If USE_METRICSLIST is set to True in carbon.conf, any metrics received which # match one of these expressions will be dropped # This file is reloaded automatically when changes are made ^some\.noisy\.metric\.prefix\..* From 044c3591328a6758d54dd029f12525265a6b7d15 Mon Sep 17 00:00:00 2001 From: H Waldo G Date: Sat, 2 Jul 2016 17:07:49 -0400 Subject: [PATCH 03/17] updating terms in carbon.conf.example --- conf/carbon.conf.example | 46 ++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/conf/carbon.conf.example b/conf/carbon.conf.example index 4465fb4dc..c24dfd915 100644 --- a/conf/carbon.conf.example +++ b/conf/carbon.conf.example @@ -14,12 +14,12 @@ # To change other directory paths, add settings to this file. The following # configuration variables are available with these default values: # -# STORAGE_DIR = $GRAPHITE_STORAGE_DIR -# LOCAL_DATA_DIR = %(STORAGE_DIR)s/whisper/ -# WHITELISTS_DIR = %(STORAGE_DIR)s/lists/ -# CONF_DIR = %(STORAGE_DIR)s/conf/ -# LOG_DIR = %(STORAGE_DIR)s/log/ -# PID_DIR = %(STORAGE_DIR)s/ +# STORAGE_DIR = $GRAPHITE_STORAGE_DIR +# LOCAL_DATA_DIR = %(STORAGE_DIR)s/whisper/ +# USE_METRICSLIST = %(STORAGE_DIR)s/lists/ +# CONF_DIR = %(STORAGE_DIR)s/conf/ +# LOG_DIR = %(STORAGE_DIR)s/log/ +# PID_DIR = %(STORAGE_DIR)s/ # # For FHS style directory structures, use: # @@ -112,7 +112,7 @@ CACHE_QUERY_PORT = 7002 USE_FLOW_CONTROL = True # If enabled this setting is used to timeout metric client connection if no -# metrics have been sent in specified time in seconds +# metrics have been sent in specified time in seconds #METRIC_CLIENT_IDLE_TIMEOUT = None # By default, carbon-cache will log every whisper update and cache hit. @@ -176,14 +176,14 @@ WHISPER_AUTOFLUSH = False # On systems which has a large number of metrics, an amount of Whisper write(2)'s # pageback sometimes cause disk thrashing due to memory shortage, so that abnormal -# disk reads occur. Enabling this option makes it possible to decrease useless +# disk reads occur. Enabling this option makes it possible to decrease useless # page cache memory by posix_fadvise(2) with POSIX_FADVISE_RANDOM option. # WHISPER_FADVISE_RANDOM = False -# Set this to True to enable whitelisting and blacklisting of metrics in -# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is -# missing or empty, all metrics will pass through -# USE_WHITELIST = False +# Set this to True to enable control over what metrics are allowed to be handled +# CONF_DIR/allowed_metrics.conf and CONF_DIR/blacklist.conf. If the allowed list +# is missing or empty, all metrics will pass through +# USE_METRICSLIST = False # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 @@ -269,7 +269,7 @@ REPLICATION_FACTOR = 1 # For REPLICATION_FACTOR >=2, set DIVERSE_REPLICAS to True to guarantee replicas # across distributed hosts. With this setting disabled, it's possible that replicas -# may be sent to different caches on the same host. This has been the default +# may be sent to different caches on the same host. This has been the default # behavior since introduction of 'consistent-hashing' relay method. # Note that enabling this on an existing pre-0.9.14 cluster will require rebalancing # your metrics across the cluster nodes using a tool like Carbonate. @@ -336,13 +336,13 @@ TIME_TO_DEFER_SENDING = 0.0001 USE_FLOW_CONTROL = True # If enabled this setting is used to timeout metric client connection if no -# metrics have been sent in specified time in seconds +# metrics have been sent in specified time in seconds #METRIC_CLIENT_IDLE_TIMEOUT = None -# Set this to True to enable whitelisting and blacklisting of metrics in -# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is -# missing or empty, all metrics will pass through -# USE_WHITELIST = False +# Set this to True to enable control over what metrics are allowed to be handled +# CONF_DIR/allowed_metrics.conf and CONF_DIR/blacklist.conf. If the allowed list +# is missing or empty, all metrics will pass through +# USE_METRICSLIST = False # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 @@ -434,7 +434,7 @@ MAX_QUEUE_SIZE = 10000 USE_FLOW_CONTROL = True # If enabled this setting is used to timeout metric client connection if no -# metrics have been sent in specified time in seconds +# metrics have been sent in specified time in seconds #METRIC_CLIENT_IDLE_TIMEOUT = None # This defines the maximum "message size" between carbon daemons. @@ -460,10 +460,10 @@ MAX_AGGREGATION_INTERVALS = 5 # having to first wait rule.frequency seconds. # WRITE_BACK_FREQUENCY = 0 -# Set this to True to enable whitelisting and blacklisting of metrics in -# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is -# missing or empty, all metrics will pass through -# USE_WHITELIST = False +# Set this to True to enable control over what metrics are allowed to be handled +# CONF_DIR/allowed_metrics.conf and CONF_DIR/blacklist.conf. If the allowed list +# is missing or empty, all metrics will pass through +# USE_METRICSLIST = False # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 From ebcd06b67df6d4efa658063fd3d70c3a4dacfc6a Mon Sep 17 00:00:00 2001 From: H Waldo G Date: Sat, 2 Jul 2016 17:11:07 -0400 Subject: [PATCH 04/17] changing 'blacklist' to 'blocked_metrics' in carbon.conf.example --- conf/carbon.conf.example | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/conf/carbon.conf.example b/conf/carbon.conf.example index c24dfd915..acbc070cd 100644 --- a/conf/carbon.conf.example +++ b/conf/carbon.conf.example @@ -181,8 +181,8 @@ WHISPER_AUTOFLUSH = False # WHISPER_FADVISE_RANDOM = False # Set this to True to enable control over what metrics are allowed to be handled -# CONF_DIR/allowed_metrics.conf and CONF_DIR/blacklist.conf. If the allowed list -# is missing or empty, all metrics will pass through +# CONF_DIR/allowed_metrics.conf and CONF_DIR/blocked_metrics.conf. If the +# allowed list is missing or empty, all metrics will pass through. # USE_METRICSLIST = False # By default, carbon itself will log statistics (such as a count, @@ -340,8 +340,8 @@ USE_FLOW_CONTROL = True #METRIC_CLIENT_IDLE_TIMEOUT = None # Set this to True to enable control over what metrics are allowed to be handled -# CONF_DIR/allowed_metrics.conf and CONF_DIR/blacklist.conf. If the allowed list -# is missing or empty, all metrics will pass through +# CONF_DIR/allowed_metrics.conf and CONF_DIR/blocked_metrics.conf. If the +# allowed list is missing or empty, all metrics will pass through. # USE_METRICSLIST = False # By default, carbon itself will log statistics (such as a count, @@ -461,8 +461,8 @@ MAX_AGGREGATION_INTERVALS = 5 # WRITE_BACK_FREQUENCY = 0 # Set this to True to enable control over what metrics are allowed to be handled -# CONF_DIR/allowed_metrics.conf and CONF_DIR/blacklist.conf. If the allowed list -# is missing or empty, all metrics will pass through +# CONF_DIR/allowed_metrics.conf and CONF_DIR/blocked_metrics.conf. If the +# allowed list is missing or empty, all metrics will pass through. # USE_METRICSLIST = False # By default, carbon itself will log statistics (such as a count, From ffc5d251cfdb790341fd350e74b03a65ead23acf Mon Sep 17 00:00:00 2001 From: H Waldo G Date: Sat, 2 Jul 2016 17:27:49 -0400 Subject: [PATCH 05/17] updating lib/carbon/conf.py to replace white/black terminology in favor of allowed/blocked --- lib/carbon/conf.py | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/lib/carbon/conf.py b/lib/carbon/conf.py index bd4d41261..cd0847bb2 100644 --- a/lib/carbon/conf.py +++ b/lib/carbon/conf.py @@ -73,7 +73,7 @@ DESTINATIONS=[], USE_FLOW_CONTROL=True, USE_INSECURE_UNPICKLER=False, - USE_WHITELIST=False, + USE_METRICSLIST=False, CARBON_METRIC_PREFIX='carbon', CARBON_METRIC_INTERVAL=60, CACHE_WRITE_STRATEGY='sorted', @@ -191,8 +191,8 @@ class CarbonCacheOptions(usage.Options): ["config", "c", None, "Use the given config file."], ["instance", "", "a", "Manage a specific carbon instance."], ["logdir", "", None, "Write logs to the given directory."], - ["whitelist", "", None, "List of metric patterns to allow."], - ["blacklist", "", None, "List of metric patterns to disallow."], + ["allowed_metrics", "", None, "List of metric patterns to allow."], + ["blocked_metrics", "", None, "List of metric patterns to disallow."], ] def postOptions(self): @@ -220,7 +220,7 @@ def postOptions(self): # Normalize and expand paths settings["STORAGE_DIR"] = os.path.normpath(os.path.expanduser(settings["STORAGE_DIR"])) settings["LOCAL_DATA_DIR"] = os.path.normpath(os.path.expanduser(settings["LOCAL_DATA_DIR"])) - settings["WHITELISTS_DIR"] = os.path.normpath(os.path.expanduser(settings["WHITELISTS_DIR"])) + settings["METRICSLIST_DIR"] = os.path.normpath(os.path.expanduser(settings["METRICSLIST_DIR"])) settings["PID_DIR"] = os.path.normpath(os.path.expanduser(settings["PID_DIR"])) settings["LOG_DIR"] = os.path.normpath(os.path.expanduser(settings["LOG_DIR"])) settings["pidfile"] = os.path.normpath(os.path.expanduser(settings["pidfile"])) @@ -272,13 +272,13 @@ def postOptions(self): os.chown(logdir, self.parent["uid"], self.parent["gid"]) log.logToDir(logdir) - if self["whitelist"] is None: - self["whitelist"] = join(settings["CONF_DIR"], "whitelist.conf") - settings["whitelist"] = self["whitelist"] + if self["allowed_metrics"] is None: + self["allowed_metrics"] = join(settings["CONF_DIR"], "allowed_metrics.conf") + settings["allowed_metrics"] = self["allowed_metrics"] - if self["blacklist"] is None: - self["blacklist"] = join(settings["CONF_DIR"], "blacklist.conf") - settings["blacklist"] = self["blacklist"] + if self["blocked_metrics"] is None: + self["blocked_metrics"] = join(settings["CONF_DIR"], "blocked_metrics.conf") + settings["blocked_metrics"] = self["blocked_metrics"] def parseArgs(self, *action): """If an action was provided, store it for further processing.""" @@ -452,14 +452,22 @@ def get_default_parser(usage="%prog [options] "): "--config", default=None, help="Use the given config file") + parser.add_option( + "--allowed_metrics", + default=None, + help="Use the given allowed_metrics file") + parser.add_option( + "--blocked_metrics", + default=None, + help="Use the given blocked_metrics file") parser.add_option( "--whitelist", default=None, - help="Use the given whitelist file") + help="DEPRECATED: Use the '--allowed_metrics' option") parser.add_option( "--blacklist", default=None, - help="Use the given blacklist file") + help="DEPRECATED: Use the ''--blocked_metrics' option") parser.add_option( "--logdir", default=None, @@ -555,7 +563,7 @@ def read_config(program, options, **kwargs): settings.setdefault( "LOCAL_DATA_DIR", join(settings["STORAGE_DIR"], "whisper")) settings.setdefault( - "WHITELISTS_DIR", join(settings["STORAGE_DIR"], "lists")) + "METRICSLIST_DIR", join(settings["STORAGE_DIR"], "lists")) # Read configuration options from program-specific section. section = program[len("carbon-"):] From b36ef29e725e8543b984533a79badb213c294ad6 Mon Sep 17 00:00:00 2001 From: H Waldo G Date: Sat, 2 Jul 2016 17:33:00 -0400 Subject: [PATCH 06/17] renaming blacklistMatches to blocked_metricsMatches and whitelistRejects to allowed_metricsRejects --- lib/carbon/instrumentation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/carbon/instrumentation.py b/lib/carbon/instrumentation.py index d953ed588..04a15c8c1 100644 --- a/lib/carbon/instrumentation.py +++ b/lib/carbon/instrumentation.py @@ -137,8 +137,8 @@ def recordMetrics(): # common metrics record('activeConnections', len(state.connectedMetricReceiverProtocols)) record('metricsReceived', myStats.get('metricsReceived', 0)) - record('blacklistMatches', myStats.get('blacklistMatches', 0)) - record('whitelistRejects', myStats.get('whitelistRejects', 0)) + record('blocked_metricsMatches', myStats.get('blocked_metricsMatches', 0)) + record('allowed_metricsRejects', myStats.get('allowed_metricsRejects', 0)) record('cpuUsage', getCpuUsage()) # And here preserve count of messages received in the prior periiod From f5d2b162a4df3f4149b8180226c665ce65105082 Mon Sep 17 00:00:00 2001 From: H Waldo G Date: Sat, 2 Jul 2016 17:36:03 -0400 Subject: [PATCH 07/17] adding code comment of DEPRECATED to the option name to make it easier to see progress --- lib/carbon/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/carbon/conf.py b/lib/carbon/conf.py index cd0847bb2..1f31e5c8f 100644 --- a/lib/carbon/conf.py +++ b/lib/carbon/conf.py @@ -461,11 +461,11 @@ def get_default_parser(usage="%prog [options] "): default=None, help="Use the given blocked_metrics file") parser.add_option( - "--whitelist", + "--whitelist", ##DEPRECATED default=None, help="DEPRECATED: Use the '--allowed_metrics' option") parser.add_option( - "--blacklist", + "--blacklist", ##DEPRECATED default=None, help="DEPRECATED: Use the ''--blocked_metrics' option") parser.add_option( From c982c586348bb5a9bdc598c2f89e1c3b91bf7015 Mon Sep 17 00:00:00 2001 From: H Waldo G Date: Sat, 2 Jul 2016 17:43:33 -0400 Subject: [PATCH 08/17] taking the pen to regexlist --- lib/carbon/regexlist.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/carbon/regexlist.py b/lib/carbon/regexlist.py index 446d9db27..a089e6963 100644 --- a/lib/carbon/regexlist.py +++ b/lib/carbon/regexlist.py @@ -6,7 +6,7 @@ class RegexList: - """ Maintain a list of regex for matching whitelist and blacklist """ + """ Maintain a list of regex for matching allowed and blocked metrics. """ def __init__(self): self.regex_list = [] @@ -58,5 +58,9 @@ def __nonzero__(self): return bool(self.regex_list) -WhiteList = RegexList() -BlackList = RegexList() +""" +##DEPRECATION: These were originally called `WhiteList` and `BlackList`. The +new names might fail some styleguide opinions. We can change them later. +""" +allowed_metrics = RegexList() +blocked_metrics = RegexList() From 43b5784dad45aaa7eda2f3a0cd85e49b83517ac3 Mon Sep 17 00:00:00 2001 From: H Waldo G Date: Sat, 2 Jul 2016 18:23:52 -0400 Subject: [PATCH 09/17] renaming USE_METRICSLIST option to USE_METRIC_FILTERS --- conf/allowed_metrics.conf.example | 2 +- conf/blocked_metrics.conf.example | 2 +- conf/carbon.conf.example | 10 +++++----- lib/carbon/conf.py | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/conf/allowed_metrics.conf.example b/conf/allowed_metrics.conf.example index 322cdf823..6a0553620 100644 --- a/conf/allowed_metrics.conf.example +++ b/conf/allowed_metrics.conf.example @@ -1,5 +1,5 @@ # This file takes a single regular expression per line -# If USE_METRICSLIST is set to True in carbon.conf, only metrics received which +# If USE_METRIC_FILTERS is set to True in carbon.conf, only metrics received which # match one of these expressions will be persisted. If this file is empty or # missing, all metrics will pass through. # This file is reloaded automatically when changes are made diff --git a/conf/blocked_metrics.conf.example b/conf/blocked_metrics.conf.example index 4b738da2e..36cd8d93c 100644 --- a/conf/blocked_metrics.conf.example +++ b/conf/blocked_metrics.conf.example @@ -1,5 +1,5 @@ # This file takes a single regular expression per line -# If USE_METRICSLIST is set to True in carbon.conf, any metrics received which +# If USE_METRIC_FILTERS is set to True in carbon.conf, any metrics received which # match one of these expressions will be dropped # This file is reloaded automatically when changes are made ^some\.noisy\.metric\.prefix\..* diff --git a/conf/carbon.conf.example b/conf/carbon.conf.example index acbc070cd..451471034 100644 --- a/conf/carbon.conf.example +++ b/conf/carbon.conf.example @@ -16,7 +16,7 @@ # # STORAGE_DIR = $GRAPHITE_STORAGE_DIR # LOCAL_DATA_DIR = %(STORAGE_DIR)s/whisper/ -# USE_METRICSLIST = %(STORAGE_DIR)s/lists/ +# USE_METRIC_FILTERS = %(STORAGE_DIR)s/lists/ # CONF_DIR = %(STORAGE_DIR)s/conf/ # LOG_DIR = %(STORAGE_DIR)s/log/ # PID_DIR = %(STORAGE_DIR)s/ @@ -183,7 +183,7 @@ WHISPER_AUTOFLUSH = False # Set this to True to enable control over what metrics are allowed to be handled # CONF_DIR/allowed_metrics.conf and CONF_DIR/blocked_metrics.conf. If the # allowed list is missing or empty, all metrics will pass through. -# USE_METRICSLIST = False +# USE_METRIC_FILTERS = False # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 @@ -342,7 +342,7 @@ USE_FLOW_CONTROL = True # Set this to True to enable control over what metrics are allowed to be handled # CONF_DIR/allowed_metrics.conf and CONF_DIR/blocked_metrics.conf. If the # allowed list is missing or empty, all metrics will pass through. -# USE_METRICSLIST = False +# USE_METRIC_FILTERS = False # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 @@ -461,9 +461,9 @@ MAX_AGGREGATION_INTERVALS = 5 # WRITE_BACK_FREQUENCY = 0 # Set this to True to enable control over what metrics are allowed to be handled -# CONF_DIR/allowed_metrics.conf and CONF_DIR/blocked_metrics.conf. If the +# CONF_DIR/allowed_metrics.conf and CONF_DIR/blocked_metrics.conf. If the # allowed list is missing or empty, all metrics will pass through. -# USE_METRICSLIST = False +# USE_METRIC_FILTERS = False # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 diff --git a/lib/carbon/conf.py b/lib/carbon/conf.py index 1f31e5c8f..1ba253567 100644 --- a/lib/carbon/conf.py +++ b/lib/carbon/conf.py @@ -73,7 +73,7 @@ DESTINATIONS=[], USE_FLOW_CONTROL=True, USE_INSECURE_UNPICKLER=False, - USE_METRICSLIST=False, + USE_METRIC_FILTERS=False, CARBON_METRIC_PREFIX='carbon', CARBON_METRIC_INTERVAL=60, CACHE_WRITE_STRATEGY='sorted', @@ -220,7 +220,7 @@ def postOptions(self): # Normalize and expand paths settings["STORAGE_DIR"] = os.path.normpath(os.path.expanduser(settings["STORAGE_DIR"])) settings["LOCAL_DATA_DIR"] = os.path.normpath(os.path.expanduser(settings["LOCAL_DATA_DIR"])) - settings["METRICSLIST_DIR"] = os.path.normpath(os.path.expanduser(settings["METRICSLIST_DIR"])) + settings["METRIC_FILTERS_DIR"] = os.path.normpath(os.path.expanduser(settings["METRIC_FILTERS_DIR"])) settings["PID_DIR"] = os.path.normpath(os.path.expanduser(settings["PID_DIR"])) settings["LOG_DIR"] = os.path.normpath(os.path.expanduser(settings["LOG_DIR"])) settings["pidfile"] = os.path.normpath(os.path.expanduser(settings["pidfile"])) @@ -563,7 +563,7 @@ def read_config(program, options, **kwargs): settings.setdefault( "LOCAL_DATA_DIR", join(settings["STORAGE_DIR"], "whisper")) settings.setdefault( - "METRICSLIST_DIR", join(settings["STORAGE_DIR"], "lists")) + "METRIC_FILTERS_DIR", join(settings["STORAGE_DIR"], "lists")) # Read configuration options from program-specific section. section = program[len("carbon-"):] From 62265f6e03b7045c68e0a852c4e2c2fdd0c4b546 Mon Sep 17 00:00:00 2001 From: H Waldo G Date: Sat, 2 Jul 2016 18:24:24 -0400 Subject: [PATCH 10/17] fixing alignment --- conf/carbon.conf.example | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conf/carbon.conf.example b/conf/carbon.conf.example index 451471034..c517695dc 100644 --- a/conf/carbon.conf.example +++ b/conf/carbon.conf.example @@ -14,12 +14,12 @@ # To change other directory paths, add settings to this file. The following # configuration variables are available with these default values: # -# STORAGE_DIR = $GRAPHITE_STORAGE_DIR -# LOCAL_DATA_DIR = %(STORAGE_DIR)s/whisper/ +# STORAGE_DIR = $GRAPHITE_STORAGE_DIR +# LOCAL_DATA_DIR = %(STORAGE_DIR)s/whisper/ # USE_METRIC_FILTERS = %(STORAGE_DIR)s/lists/ -# CONF_DIR = %(STORAGE_DIR)s/conf/ -# LOG_DIR = %(STORAGE_DIR)s/log/ -# PID_DIR = %(STORAGE_DIR)s/ +# CONF_DIR = %(STORAGE_DIR)s/conf/ +# LOG_DIR = %(STORAGE_DIR)s/log/ +# PID_DIR = %(STORAGE_DIR)s/ # # For FHS style directory structures, use: # From 09d3e7376af74f62b68a65672696f10398d17a4c Mon Sep 17 00:00:00 2001 From: H Waldo G Date: Sat, 2 Jul 2016 18:34:46 -0400 Subject: [PATCH 11/17] changing metric_filters names in lib/carbon/protocols.py --- lib/carbon/protocols.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/carbon/protocols.py b/lib/carbon/protocols.py index d37c37e5f..84ab33a99 100644 --- a/lib/carbon/protocols.py +++ b/lib/carbon/protocols.py @@ -7,7 +7,7 @@ from twisted.protocols.policies import TimeoutMixin from carbon import log, events, state, management from carbon.conf import settings -from carbon.regexlist import WhiteList, BlackList +from carbon.regexlist import allowed_metrics, blocked_metrics from carbon.util import pickle, get_unpickler @@ -56,17 +56,17 @@ def connectionLost(self, reason): events.resumeReceivingMetrics.removeHandler(self.resumeReceiving) def metricReceived(self, metric, datapoint): - if BlackList and metric in BlackList: - instrumentation.increment('blacklistMatches') + if blocked_metrics and metric in blocked_metrics: + instrumentation.increment('blocked_metricsMatches') return - if WhiteList and metric not in WhiteList: - instrumentation.increment('whitelistRejects') + if allowed_metrics and metric not in allowed_metrics: + instrumentation.increment('allowed_metricsRejects') return if datapoint[1] != datapoint[1]: # filter out NaN values return if int(datapoint[0]) == -1: # use current time if none given: https://github.com/graphite-project/carbon/issues/54 datapoint = (time.time(), datapoint[1]) - + events.metricReceived(metric, datapoint) self.resetTimeout() From 8eed6e569c453f1d5fdb239551f4c29683947d6e Mon Sep 17 00:00:00 2001 From: H Waldo G Date: Sat, 2 Jul 2016 18:55:13 -0400 Subject: [PATCH 12/17] I think I need to keep these two calls in for now to allow graceful deprecation --- lib/carbon/regexlist.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/carbon/regexlist.py b/lib/carbon/regexlist.py index a089e6963..8017a66c0 100644 --- a/lib/carbon/regexlist.py +++ b/lib/carbon/regexlist.py @@ -64,3 +64,5 @@ def __nonzero__(self): """ allowed_metrics = RegexList() blocked_metrics = RegexList() +WhiteList = RegexList() ##DEPRECATING +BlackList = RegexList() ##DEPRECATING From c1b57aec2b65e7913de5b223e7a2003b25853d77 Mon Sep 17 00:00:00 2001 From: H Waldo G Date: Sat, 2 Jul 2016 19:06:50 -0400 Subject: [PATCH 13/17] attmempting to deprecate white/black-list while not ripping out their support --- lib/carbon/service.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/carbon/service.py b/lib/carbon/service.py index 34d52580c..f61b41b3b 100644 --- a/lib/carbon/service.py +++ b/lib/carbon/service.py @@ -52,10 +52,21 @@ def createBaseService(config, settings): root_service = CarbonRootService() root_service.setName(settings.program) - if settings.USE_WHITELIST: - from carbon.regexlist import WhiteList, BlackList - WhiteList.read_from(settings.whitelist) - BlackList.read_from(settings.blacklist) + if settings.USE_WHITELIST: # DEPRECATING per https://github.com/graphite-project/carbon/issues/567 + settings.USE_METRIC_FILTERS = True + log.msg("NOTE: 'USE_WHITELIST' is being deprecated in favor of 'USE_METRIC_FILTERS'.") + WhiteList.read_from(settings.whitelist) #DEPRECATING + BlackList.read_from(settings.blacklist) #DEPRECATING + + if settings.USE_METRIC_FILTERS: + from carbon.regexlist import allowed_metrics, blocked_metrics + allowed_metrics.read_from(settings.allowed_metrics) + blocked_metrics.read_from(settings.blocked_metrics) + # I hope that this does what I think. + if WhiteList: #DEPRECATING + allowed_metrics.update(WhiteList) #DEPRECATING + if BlackList: #DEPRECATING + blocked_metrics.update(BlackList) #DEPRECATING # Instantiate an instrumentation service that will record metrics about # this service. From c12185cda751e6faccb6ac835641d3cea3c2d489 Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Thu, 15 Sep 2016 13:25:27 -0400 Subject: [PATCH 14/17] fix test --- lib/carbon/tests/test_conf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/carbon/tests/test_conf.py b/lib/carbon/tests/test_conf.py index 718d2e657..2e14445c7 100644 --- a/lib/carbon/tests/test_conf.py +++ b/lib/carbon/tests/test_conf.py @@ -396,9 +396,9 @@ def test_local_data_dir_depends_on_storage_dir(self): ROOT_DIR="foo") self.assertEqual(join("bar", "whisper"), settings.LOCAL_DATA_DIR) - def test_whitelists_dir_depends_on_storage_dir(self): + def test_metric_filters_dir_depends_on_storage_dir(self): """ - Tests 'STORAGE_DIR' dependency 'WHITELISTS_DIR' + Tests 'STORAGE_DIR' dependency 'METRIC_FILTERS_DIR' """ config = self.makeFile( content=("[foo]\n" @@ -408,4 +408,4 @@ def test_whitelists_dir_depends_on_storage_dir(self): FakeOptions(config=config, instance=None, pidfile=None, logdir=None), ROOT_DIR="foo") - self.assertEqual(join("bar", "lists"), settings.WHITELISTS_DIR) + self.assertEqual(join("bar", "lists"), settings.METRIC_FILTERS_DIR) From 880ecde1a704e5d427317a5e68c1cb939e5e798a Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Thu, 15 Sep 2016 13:26:05 -0400 Subject: [PATCH 15/17] fix this example --- conf/carbon.conf.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/carbon.conf.example b/conf/carbon.conf.example index 532eef769..1f36f66db 100644 --- a/conf/carbon.conf.example +++ b/conf/carbon.conf.example @@ -16,7 +16,7 @@ # # STORAGE_DIR = $GRAPHITE_STORAGE_DIR # LOCAL_DATA_DIR = %(STORAGE_DIR)s/whisper/ -# USE_METRIC_FILTERS = %(STORAGE_DIR)s/lists/ +# METRIC_FILTERS_DIR = %(STORAGE_DIR)s/lists/ # CONF_DIR = %(STORAGE_DIR)s/conf/ # LOG_DIR = %(STORAGE_DIR)s/log/ # PID_DIR = %(STORAGE_DIR)s/ From a429e0aef1dc7b682eaaf469519d6245fd4497e9 Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Thu, 15 Sep 2016 13:26:39 -0400 Subject: [PATCH 16/17] minor improvements for clarity --- conf/carbon.conf.example | 18 +++++++++--------- lib/carbon/conf.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/conf/carbon.conf.example b/conf/carbon.conf.example index 1f36f66db..640713f26 100644 --- a/conf/carbon.conf.example +++ b/conf/carbon.conf.example @@ -179,9 +179,9 @@ WHISPER_FALLOCATE_CREATE = True # page cache memory by posix_fadvise(2) with POSIX_FADVISE_RANDOM option. # WHISPER_FADVISE_RANDOM = False -# Set this to True to enable control over what metrics are allowed to be handled -# CONF_DIR/allowed_metrics.conf and CONF_DIR/blocked_metrics.conf. If the -# allowed list is missing or empty, all metrics will pass through. +# Set this to True to enable metric filters with CONF_DIR/allowed_metrics.conf +# and CONF_DIR/blocked_metrics.conf. If the allowed list is missing or empty, +# all metrics will pass through. # USE_METRIC_FILTERS = False # By default, carbon itself will log statistics (such as a count, @@ -338,9 +338,9 @@ USE_FLOW_CONTROL = True # metrics have been sent in specified time in seconds #METRIC_CLIENT_IDLE_TIMEOUT = None -# Set this to True to enable control over what metrics are allowed to be handled -# CONF_DIR/allowed_metrics.conf and CONF_DIR/blocked_metrics.conf. If the -# allowed list is missing or empty, all metrics will pass through. +# Set this to True to enable metric filters with CONF_DIR/allowed_metrics.conf +# and CONF_DIR/blocked_metrics.conf. If the allowed list is missing or empty, +# all metrics will pass through. # USE_METRIC_FILTERS = False # By default, carbon itself will log statistics (such as a count, @@ -459,9 +459,9 @@ MAX_AGGREGATION_INTERVALS = 5 # having to first wait rule.frequency seconds. # WRITE_BACK_FREQUENCY = 0 -# Set this to True to enable control over what metrics are allowed to be handled -# CONF_DIR/allowed_metrics.conf and CONF_DIR/blocked_metrics.conf. If the -# allowed list is missing or empty, all metrics will pass through. +# Set this to True to enable metric filters with CONF_DIR/allowed_metrics.conf +# and CONF_DIR/blocked_metrics.conf. If the allowed list is missing or empty, +# all metrics will pass through. # USE_METRIC_FILTERS = False # By default, carbon itself will log statistics (such as a count, diff --git a/lib/carbon/conf.py b/lib/carbon/conf.py index 26cba2687..50b94f263 100644 --- a/lib/carbon/conf.py +++ b/lib/carbon/conf.py @@ -191,7 +191,7 @@ class CarbonCacheOptions(usage.Options): ["instance", "", "a", "Manage a specific carbon instance."], ["logdir", "", None, "Write logs to the given directory."], ["allowed_metrics", "", None, "List of metric patterns to allow."], - ["blocked_metrics", "", None, "List of metric patterns to disallow."], + ["blocked_metrics", "", None, "List of metric patterns to block."], ] def postOptions(self): From 472be28d1fc3aacb4b941afd679e232beb3ba13f Mon Sep 17 00:00:00 2001 From: obfuscurity Date: Thu, 15 Sep 2016 14:52:18 -0400 Subject: [PATCH 17/17] set correct dir --- lib/carbon/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/carbon/conf.py b/lib/carbon/conf.py index 50b94f263..0d5d4072b 100644 --- a/lib/carbon/conf.py +++ b/lib/carbon/conf.py @@ -295,11 +295,11 @@ def postOptions(self): log.logToDir(logdir) if self["allowed_metrics"] is None: - self["allowed_metrics"] = join(settings["CONF_DIR"], "allowed_metrics.conf") + self["allowed_metrics"] = join(settings["METRIC_FILTERS_DIR"], "allowed_metrics.conf") settings["allowed_metrics"] = self["allowed_metrics"] if self["blocked_metrics"] is None: - self["blocked_metrics"] = join(settings["CONF_DIR"], "blocked_metrics.conf") + self["blocked_metrics"] = join(settings["METRIC_FILTERS_DIR"], "blocked_metrics.conf") settings["blocked_metrics"] = self["blocked_metrics"] def parseArgs(self, *action):