diff --git a/conf/whitelist.conf.example b/conf/allowed_metrics.conf.example similarity index 74% rename from conf/whitelist.conf.example rename to conf/allowed_metrics.conf.example index f84895d0a..6a0553620 100644 --- a/conf/whitelist.conf.example +++ b/conf/allowed_metrics.conf.example @@ -1,5 +1,5 @@ # This file takes a single regular expression per line -# If USE_WHITELIST is set to True in carbon.conf, only metrics received which +# If USE_METRIC_FILTERS is set to True in carbon.conf, only metrics received which # match one of these expressions will be persisted. If this file is empty or # missing, all metrics will pass through. # This file is reloaded automatically when changes are made diff --git a/conf/blacklist.conf.example b/conf/blocked_metrics.conf.example similarity index 83% rename from conf/blacklist.conf.example rename to conf/blocked_metrics.conf.example index 8038a7ef1..36cd8d93c 100644 --- a/conf/blacklist.conf.example +++ b/conf/blocked_metrics.conf.example @@ -1,5 +1,5 @@ # This file takes a single regular expression per line -# If USE_WHITELIST is set to True in carbon.conf, any metrics received which +# If USE_METRIC_FILTERS is set to True in carbon.conf, any metrics received which # match one of these expressions will be dropped # This file is reloaded automatically when changes are made ^some\.noisy\.metric\.prefix\..* diff --git a/conf/carbon.conf.example b/conf/carbon.conf.example index e919de1d9..890108544 100644 --- a/conf/carbon.conf.example +++ b/conf/carbon.conf.example @@ -14,12 +14,12 @@ # To change other directory paths, add settings to this file. The following # configuration variables are available with these default values: # -# STORAGE_DIR = $GRAPHITE_STORAGE_DIR -# LOCAL_DATA_DIR = %(STORAGE_DIR)s/whisper/ -# WHITELISTS_DIR = %(STORAGE_DIR)s/lists/ -# CONF_DIR = %(STORAGE_DIR)s/conf/ -# LOG_DIR = %(STORAGE_DIR)s/log/ -# PID_DIR = %(STORAGE_DIR)s/ +# STORAGE_DIR = $GRAPHITE_STORAGE_DIR +# LOCAL_DATA_DIR = %(STORAGE_DIR)s/whisper/ +# METRIC_FILTERS_DIR = %(STORAGE_DIR)s/lists/ +# CONF_DIR = %(STORAGE_DIR)s/conf/ +# LOG_DIR = %(STORAGE_DIR)s/log/ +# PID_DIR = %(STORAGE_DIR)s/ # # For FHS style directory structures, use: # @@ -224,6 +224,11 @@ WHISPER_FALLOCATE_CREATE = True # page cache memory by posix_fadvise(2) with POSIX_FADVISE_RANDOM option. # WHISPER_FADVISE_RANDOM = False +# Set this to True to enable metric filters with CONF_DIR/allowed_metrics.conf +# and CONF_DIR/blocked_metrics.conf. If the allowed list is missing or empty, +# all metrics will pass through. +# USE_METRIC_FILTERS = False + # By default all nodes stored in Ceres are cached in memory to improve the # throughput of reads and writes to underlying slices. Turning this off will # greatly reduce memory consumption for databases with millions of metrics, at @@ -267,11 +272,6 @@ WHISPER_FALLOCATE_CREATE = True # multiple carbon-cache daemons are writing to the same files. # CERES_LOCK_WRITES = False -# Set this to True to enable whitelisting and blacklisting of metrics in -# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is -# missing or empty, all metrics will pass through -# USE_WHITELIST = False - # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 # seconds. Set CARBON_METRIC_INTERVAL to 0 to disable instrumentation @@ -516,10 +516,10 @@ USE_FLOW_CONTROL = True # metrics have been sent in specified time in seconds #METRIC_CLIENT_IDLE_TIMEOUT = None -# Set this to True to enable whitelisting and blacklisting of metrics in -# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is -# missing or empty, all metrics will pass through -# USE_WHITELIST = False +# Set this to True to enable metric filters with CONF_DIR/allowed_metrics.conf +# and CONF_DIR/blocked_metrics.conf. If the allowed list is missing or empty, +# all metrics will pass through. +# USE_METRIC_FILTERS = False # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 @@ -656,10 +656,10 @@ MAX_AGGREGATION_INTERVALS = 5 # having to first wait rule.frequency seconds. # WRITE_BACK_FREQUENCY = 0 -# Set this to True to enable whitelisting and blacklisting of metrics in -# CONF_DIR/whitelist.conf and CONF_DIR/blacklist.conf. If the whitelist is -# missing or empty, all metrics will pass through -# USE_WHITELIST = False +# Set this to True to enable metric filters with CONF_DIR/allowed_metrics.conf +# and CONF_DIR/blocked_metrics.conf. If the allowed list is missing or empty, +# all metrics will pass through. +# USE_METRIC_FILTERS = False # By default, carbon itself will log statistics (such as a count, # metricsReceived) with the top level prefix of 'carbon' at an interval of 60 diff --git a/lib/carbon/conf.py b/lib/carbon/conf.py index aef5e684f..7886c82a1 100644 --- a/lib/carbon/conf.py +++ b/lib/carbon/conf.py @@ -103,7 +103,7 @@ DESTINATION_POOL_REPLICAS=False, USE_FLOW_CONTROL=True, USE_INSECURE_UNPICKLER=False, - USE_WHITELIST=False, + USE_METRIC_FILTERS=False, CARBON_METRIC_PREFIX='carbon', CARBON_METRIC_INTERVAL=60, CACHE_WRITE_STRATEGY='sorted', @@ -228,8 +228,8 @@ class CarbonCacheOptions(usage.Options): ["config", "c", None, "Use the given config file."], ["instance", "", "a", "Manage a specific carbon instance."], ["logdir", "", None, "Write logs to the given directory."], - ["whitelist", "", None, "List of metric patterns to allow."], - ["blacklist", "", None, "List of metric patterns to disallow."], + ["allowed_metrics", "", None, "List of metric patterns to allow."], + ["blocked_metrics", "", None, "List of metric patterns to block."], ] def postOptions(self): @@ -259,7 +259,7 @@ def cleanpath(path): return os.path.normpath(os.path.expanduser(path)) settings["STORAGE_DIR"] = cleanpath(settings["STORAGE_DIR"]) settings["LOCAL_DATA_DIR"] = cleanpath(settings["LOCAL_DATA_DIR"]) - settings["WHITELISTS_DIR"] = cleanpath(settings["WHITELISTS_DIR"]) + settings["METRIC_FILTERS_DIR"] = os.path.normpath(os.path.expanduser(settings["METRIC_FILTERS_DIR"])) settings["PID_DIR"] = cleanpath(settings["PID_DIR"]) settings["LOG_DIR"] = cleanpath(settings["LOG_DIR"]) settings["pidfile"] = cleanpath(settings["pidfile"]) @@ -319,13 +319,13 @@ def cleanpath(path): os.chown(logdir, self.parent["uid"], self.parent["gid"]) log.logToDir(logdir) - if self["whitelist"] is None: - self["whitelist"] = join(settings["CONF_DIR"], "whitelist.conf") - settings["whitelist"] = self["whitelist"] + if self["allowed_metrics"] is None: + self["allowed_metrics"] = join(settings["METRIC_FILTERS_DIR"], "allowed_metrics.conf") + settings["allowed_metrics"] = self["allowed_metrics"] - if self["blacklist"] is None: - self["blacklist"] = join(settings["CONF_DIR"], "blacklist.conf") - settings["blacklist"] = self["blacklist"] + if self["blocked_metrics"] is None: + self["blocked_metrics"] = join(settings["METRIC_FILTERS_DIR"], "blocked_metrics.conf") + settings["blocked_metrics"] = self["blocked_metrics"] def parseArgs(self, *action): """If an action was provided, store it for further processing.""" @@ -525,13 +525,21 @@ def get_default_parser(usage="%prog [options] "): default=None, help="Use the given config file") parser.add_option( - "--whitelist", + "--allowed_metrics", default=None, - help="Use the given whitelist file") + help="Use the given allowed_metrics file") parser.add_option( - "--blacklist", + "--blocked_metrics", default=None, - help="Use the given blacklist file") + help="Use the given blocked_metrics file") + parser.add_option( + "--whitelist", ##DEPRECATED + default=None, + help="DEPRECATED: Use the '--allowed_metrics' option") + parser.add_option( + "--blacklist", ##DEPRECATED + default=None, + help="DEPRECATED: Use the ''--blocked_metrics' option") parser.add_option( "--logdir", default=None, @@ -636,7 +644,7 @@ def update_STORAGE_DIR_deps(): settings.setdefault( "LOCAL_DATA_DIR", join(settings["STORAGE_DIR"], "whisper")) settings.setdefault( - "WHITELISTS_DIR", join(settings["STORAGE_DIR"], "lists")) + "METRIC_FILTERS_DIR", join(settings["STORAGE_DIR"], "lists")) # Read configuration options from program-specific section. section = program[len("carbon-"):] diff --git a/lib/carbon/instrumentation.py b/lib/carbon/instrumentation.py index 5e4f83e44..2726d017b 100644 --- a/lib/carbon/instrumentation.py +++ b/lib/carbon/instrumentation.py @@ -148,8 +148,8 @@ def recordMetrics(): # common metrics record('activeConnections', len(state.connectedMetricReceiverProtocols)) record('metricsReceived', myStats.get('metricsReceived', 0)) - record('blacklistMatches', myStats.get('blacklistMatches', 0)) - record('whitelistRejects', myStats.get('whitelistRejects', 0)) + record('blocked_metricsMatches', myStats.get('blocked_metricsMatches', 0)) + record('allowed_metricsRejects', myStats.get('allowed_metricsRejects', 0)) record('cpuUsage', getCpuUsage()) # And here preserve count of messages received in the prior period diff --git a/lib/carbon/protocols.py b/lib/carbon/protocols.py index 45eee81a9..997ce975b 100644 --- a/lib/carbon/protocols.py +++ b/lib/carbon/protocols.py @@ -11,7 +11,7 @@ from twisted.protocols.policies import TimeoutMixin from carbon import log, events, state, management from carbon.conf import settings -from carbon.regexlist import WhiteList, BlackList +from carbon.regexlist import allowed_metrics, blocked_metrics from carbon.util import pickle, get_unpickler from carbon.util import PluginRegistrar from six import with_metaclass @@ -166,17 +166,18 @@ def connectionLost(self, reason): events.resumeReceivingMetrics.removeHandler(self.resumeReceiving) def metricReceived(self, metric, datapoint): - if BlackList and metric in BlackList: - instrumentation.increment('blacklistMatches') + if blocked_metrics and metric in blocked_metrics: + instrumentation.increment('blocked_metricsMatches') return - if WhiteList and metric not in WhiteList: - instrumentation.increment('whitelistRejects') + if allowed_metrics and metric not in allowed_metrics: + instrumentation.increment('allowed_metricsRejects') return if datapoint[1] != datapoint[1]: # filter out NaN values return # use current time if none given: https://github.com/graphite-project/carbon/issues/54 if int(datapoint[0]) == -1: datapoint = (time.time(), datapoint[1]) + res = settings.MIN_TIMESTAMP_RESOLUTION if res: datapoint = (int(datapoint[0]) // res * res, datapoint[1]) diff --git a/lib/carbon/regexlist.py b/lib/carbon/regexlist.py index 3fa93ee39..845261eac 100644 --- a/lib/carbon/regexlist.py +++ b/lib/carbon/regexlist.py @@ -5,7 +5,7 @@ class RegexList: - """ Maintain a list of regex for matching whitelist and blacklist """ + """ Maintain a list of regex for matching allowed and blocked metrics. """ def __init__(self): self.regex_list = [] @@ -59,5 +59,11 @@ def __nonzero__(self): __bool__ = __nonzero__ # py2/3 compatibility -WhiteList = RegexList() -BlackList = RegexList() +""" +##DEPRECATION: These were originally called `WhiteList` and `BlackList`. The +new names might fail some styleguide opinions. We can change them later. +""" +allowed_metrics = RegexList() +blocked_metrics = RegexList() +WhiteList = RegexList() ##DEPRECATING +BlackList = RegexList() ##DEPRECATING diff --git a/lib/carbon/service.py b/lib/carbon/service.py index 0c0fb8948..e11cd952a 100644 --- a/lib/carbon/service.py +++ b/lib/carbon/service.py @@ -55,10 +55,21 @@ def createBaseService(config, settings): root_service = CarbonRootService() root_service.setName(settings.program) - if settings.USE_WHITELIST: - from carbon.regexlist import WhiteList, BlackList - WhiteList.read_from(settings.whitelist) - BlackList.read_from(settings.blacklist) + if settings.USE_WHITELIST: # DEPRECATING per https://github.com/graphite-project/carbon/issues/567 + settings.USE_METRIC_FILTERS = True + log.msg("NOTE: 'USE_WHITELIST' is being deprecated in favor of 'USE_METRIC_FILTERS'.") + WhiteList.read_from(settings.whitelist) #DEPRECATING + BlackList.read_from(settings.blacklist) #DEPRECATING + + if settings.USE_METRIC_FILTERS: + from carbon.regexlist import allowed_metrics, blocked_metrics + allowed_metrics.read_from(settings.allowed_metrics) + blocked_metrics.read_from(settings.blocked_metrics) + # I hope that this does what I think. + if WhiteList: #DEPRECATING + allowed_metrics.update(WhiteList) #DEPRECATING + if BlackList: #DEPRECATING + blocked_metrics.update(BlackList) #DEPRECATING # Instantiate an instrumentation service that will record metrics about # this service. diff --git a/lib/carbon/tests/test_conf.py b/lib/carbon/tests/test_conf.py index 8449d3b90..2a4fbfcfe 100644 --- a/lib/carbon/tests/test_conf.py +++ b/lib/carbon/tests/test_conf.py @@ -417,9 +417,9 @@ def test_local_data_dir_depends_on_storage_dir(self): ROOT_DIR="foo") self.assertEqual(join("bar", "whisper"), settings.LOCAL_DATA_DIR) - def test_whitelists_dir_depends_on_storage_dir(self): + def test_metric_filters_dir_depends_on_storage_dir(self): """ - Tests 'STORAGE_DIR' dependency 'WHITELISTS_DIR' + Tests 'STORAGE_DIR' dependency 'METRIC_FILTERS_DIR' """ config = self.makeFile( content=("[foo]\n" @@ -429,4 +429,4 @@ def test_whitelists_dir_depends_on_storage_dir(self): FakeOptions(config=config, instance=None, pidfile=None, logdir=None), ROOT_DIR="foo") - self.assertEqual(join("bar", "lists"), settings.WHITELISTS_DIR) + self.assertEqual(join("bar", "lists"), settings.METRIC_FILTERS_DIR)