Rebasing #1700 (Change Black/White List to Block/Allow list)

docs/config-carbon.rst

@@ -269,13 +269,28 @@ aggregation.
 **Note:** if you plan to use the ``=`` sign in your rewrite rules. Use its octal value: ``\075``.
 For example ``foo=bar = foo.bar`` would be ``foo\075bar = foo.bar``
 
+Metric Filters: allowed and blocked Metrics
+-------------------------------------------
+
+The metric filter functionality allows any of the carbon daemons to only accept
+metrics that are explicitly allowed and/or to reject rejected metrics. The
+functionality can be enabled in carbon.conf with the ``USE_METRIC_FILTERS``
+flag. This can be useful when too many metrics are being sent to a Graphite
+instance or when there are metric senders sending useless or invalid metrics.
+
+``GRAPHITE_CONF_DIR`` is searched for ``allowed_metrics.conf`` and
+``blocked_metrics.conf``.  Each file contains one regular expression per line to
+match against metric values.  If the allowed_metrics configuration is missing or
+empty, all metrics will be passed through by default.
+
 whitelist and blacklist
 -----------------------
-The whitelist functionality allows any of the carbon daemons to only accept metrics that are explicitly
-whitelisted and/or to reject blacklisted metrics. The functionality can be enabled in carbon.conf with
-the ``USE_WHITELIST`` flag. This can be useful when too many metrics are being sent to a Graphite
-instance or when there are metric senders sending useless or invalid metrics.
+The whitelist/blacklist functionality has been renamed to 'allowed' and
+'blocked' so as to use less ambiguous language, and remove possible connotations
+associated with those terms.
+
+The capabilities have been renamed as of this point, but he existing ``whitelist.conf``
+and ``blacklist.conf`` will still be functional for the time being.
 
-``GRAPHITE_CONF_DIR`` is searched for ``whitelist.conf`` and ``blacklist.conf``. Each file contains one regular
-expressions per line to match against metric values. If the whitelist configuration is missing or empty,
-all metrics will be passed through by default.
+Additionally, the ``USE_WHITELIST`` flag in carbon.conf will still be respected,
+but treated as ``USE_METRIC_FILTERS`` until it is deprecated.

docs/config-local-settings.rst

@@ -434,7 +434,7 @@ POOL_MAX_WORKERS
 REMOTE_RETRY_DELAY
   `Default: 60`
 
-  Time in seconds to blacklist a webapp after a timed-out request.
+  Time in seconds to block/filter a webapp after a timed-out request.
 
 FIND_CACHE_DURATION
   `Default: 300`

setup.py

@@ -101,7 +101,7 @@ def read(fname):
         'graphite.url_shortener',
         'graphite.url_shortener.migrations',
         'graphite.version',
-        'graphite.whitelist',
+        'graphite.metric_filters',
         'graphite.worker_pool',
       ],
       package_data={'graphite': ['templates/*', 'local_settings.py.example']},

tox.ini

@@ -11,7 +11,7 @@ envlist =
   lint, docs
 
 [testenv]
-whitelist_externals =
+whitelist_externals =		# Unrelated to graphite project deprecation of white/black
 	mkdir
 setenv =
 	DJANGO_SETTINGS_MODULE=tests.settings

webapp/content/js/composer_widgets.js

@@ -1041,20 +1041,20 @@ var GraphDataWindow = {
 
   addWlSelected: function (item, e) {
     Ext.Ajax.request({
-      url: document.body.dataset.baseUrl + 'whitelist/add',
+      url: document.body.dataset.baseUrl + 'metric_filters/add',
       method: 'POST',
-      success: function () { Ext.Msg.alert('Result', 'Successfully added metrics to whitelist.'); },
-      failure: function () { Ext.Msg.alert('Result', 'Failed to add metrics to whitelist.');   },
+      success: function () { Ext.Msg.alert('Result', 'Successfully added metrics to filter.'); },
+      failure: function () { Ext.Msg.alert('Result', 'Failed to add metrics to filter.');   },
       params: {metrics: this.getSelectedTargets().join('\n') }
     });
   },
 
   removeWlSelected: function (item, e) {
     Ext.Ajax.request({
-      url: document.body.dataset.baseUrl + 'whitelist/remove',
+      url: document.body.dataset.baseUrl + 'metric_filters/remove',
       method: 'POST',
-      success: function () { Ext.Msg.alert('Result', 'Successfully removed metrics from whitelist.'); },
-      failure: function () { Ext.Msg.alert('Result', 'Failed to remove metrics from whitelist.');   },
+      success: function () { Ext.Msg.alert('Result', 'Successfully removed metrics from filter.'); },
+      failure: function () { Ext.Msg.alert('Result', 'Failed to remove metrics from filter.');   },
       params: {metrics: this.getSelectedTargets().join('\n') }
     });
   },

webapp/graphite/app_settings.py

@@ -105,7 +105,7 @@
   'graphite.render',
   'graphite.tags',
   'graphite.url_shortener',
-  'graphite.whitelist',
+  'graphite.metric_filters',
   'django.contrib.auth',
   'django.contrib.sessions',
   'django.contrib.admin',

webapp/graphite/local_settings.py.example

@@ -337,7 +337,7 @@ DEFAULT_XFILES_FACTOR = 0
 #
 #CARBONLINK_HOSTS = ["127.0.0.1:7002:a", "127.0.0.1:7102:b", "127.0.0.1:7202:c"]
 #CARBONLINK_TIMEOUT = 1.0
-#CARBONLINK_RETRY_DELAY = 15 # Seconds to blacklist a failed remote server
+#CARBONLINK_RETRY_DELAY = 15 # Seconds to block a failed remote server
 # Set pickle protocol to use for Carbonlink requests,
 # (default of -1 is HIGHEST_AVAILABLE for your Python version)
 # see more: https://docs.python.org/3/library/pickle.html#data-stream-format

webapp/graphite/whitelist/urls.py → webapp/graphite/metric_filters/urls.py

@@ -16,7 +16,7 @@
 from . import views
 
 urlpatterns = [
-    url(r'^/add$', views.add, name='whitelist_add'),
-    url(r'^/remove$', views.remove, name='whitelist_remove'),
-    url(r'^/?$', views.show, name='whitelist_show'),
+    url(r'^/add$', views.add, name='metric_filters_add'),
+    url(r'^/remove$', views.remove, name='metric_filters_remove'),
+    url(r'^/?$', views.show, name='metric_filters_show'),
 ]

webapp/graphite/whitelist/views.py → webapp/graphite/metric_filters/views.py

@@ -23,43 +23,43 @@
 
 def add(request):
     metrics = set( request.POST['metrics'].split() )
-    whitelist = load_whitelist()
-    new_whitelist = whitelist | metrics
-    save_whitelist(new_whitelist)
+    allowed_metrics = load_allowed_metrics()
+    new_allowed_metrics = allowed_metrics | metrics
+    save_allowed_metrics(new_allowed_metrics)
     return HttpResponse(content_type="text/plain", content="OK")
 
 
 def remove(request):
     metrics = set( request.POST['metrics'].split() )
-    whitelist = load_whitelist()
-    new_whitelist = whitelist - metrics
-    save_whitelist(new_whitelist)
+    allowed_metrics = load_allowed_metrics()
+    new_allowed_metrics = allowed_metrics - metrics
+    save_allowed_metrics(new_allowed_metrics)
     return HttpResponse(content_type="text/plain", content="OK")
 
 
 def show(request):
-    whitelist = load_whitelist()
-    members = '\n'.join( sorted(whitelist) )
+    allowed_metrics = load_allowed_metrics()
+    members = '\n'.join( sorted(allowed_metrics) )
     return HttpResponse(content_type="text/plain", content=members)
 
 
-def load_whitelist():
-    buffer = open(settings.WHITELIST_FILE, 'rb').read()
-    whitelist = unpickle.loads(buffer)
-    return whitelist
+def load_allowed_metrics():
+    buffer = open(settings.METRIC_FILTERS_FILE, 'rb').read()
+    allowed_metrics = unpickle.loads(buffer)
+    return allowed_metrics
 
 
-def save_whitelist(whitelist):
+def save_allowed_metrics(allowed_metrics):
     # do this instead of dump() to raise potential exceptions before open()
-    serialized = pickle.dumps(whitelist, protocol=-1)
-    tmpfile = '%s-%d' % (settings.WHITELIST_FILE, randint(0, 100000))
+    serialized = pickle.dumps(allowed_metrics, protocol=-1)
+    tmpfile = '%s-%d' % (settings.METRIC_FILTERS_FILE, randint(0, 100000))
     try:
         fh = open(tmpfile, 'wb')
         fh.write(serialized)
         fh.close()
-        if os.path.exists(settings.WHITELIST_FILE):
-            os.unlink(settings.WHITELIST_FILE)
-        os.rename(tmpfile, settings.WHITELIST_FILE)
+        if os.path.exists(settings.METRIC_FILTERS_FILE):
+            os.unlink(settings.METRIC_FILTERS_FILE)
+        os.rename(tmpfile, settings.METRIC_FILTERS_FILE)
     finally:
         if os.path.exists(tmpfile):
             os.unlink(tmpfile)

webapp/graphite/settings.py

@@ -50,7 +50,7 @@
 DASHBOARD_CONF = ''
 GRAPHTEMPLATES_CONF = ''
 STORAGE_DIR = ''
-WHITELIST_FILE = ''
+METRIC_FILTERS_FILE = ''
 INDEX_FILE = ''
 LOG_DIR = ''
 CERES_DIR = ''
@@ -266,8 +266,8 @@
 
 if not STORAGE_DIR:
     STORAGE_DIR = os.environ.get('GRAPHITE_STORAGE_DIR', join(GRAPHITE_ROOT, 'storage'))
-if not WHITELIST_FILE:
-    WHITELIST_FILE = join(STORAGE_DIR, 'lists', 'whitelist')
+if not METRIC_FILTERS_FILE:
+    METRIC_FILTERS_FILE = join(STORAGE_DIR, 'lists', 'allowed_metrics')
 if not INDEX_FILE:
     INDEX_FILE = join(STORAGE_DIR, 'index')
 if not LOG_DIR:

webapp/graphite/urls.py

@@ -25,7 +25,7 @@
     url('^browser', include('graphite.browser.urls')),
     url('^account', include('graphite.account.urls')),
     url('^dashboard', include('graphite.dashboard.urls')),
-    url('^whitelist', include('graphite.whitelist.urls')),
+    url('^metric_filters/?', include('graphite.metric_filters.urls')),
     url('^version', include('graphite.version.urls')),
     url('^events', include('graphite.events.urls')),
     url('^tags', include('graphite.tags.urls')),

webapp/tests/test_whitelist.py

@@ -12,108 +12,108 @@
     from django.core.urlresolvers import reverse
 from .base import TestCase
 
-from graphite.whitelist.views import load_whitelist, save_whitelist
+from graphite.metric_filters.views import load_allowed_metrics, save_allowed_metrics
 
 
 class WhitelistTester(TestCase):
-    settings.WHITELIST_FILE = os.path.join(DATA_DIR, 'lists/whitelist')
+    settings.METRIC_FILTERS_FILE = os.path.join(DATA_DIR, 'lists/allowlist')
 
-    def wipe_whitelist(self):
+    def wipe_allowlist(self):
         try:
-            os.remove(settings.WHITELIST_FILE)
+            os.remove(settings.METRIC_FILTERS_FILE)
         except OSError:
             pass
 
-    def create_whitelist(self):
+    def create_allowlist(self):
         try:
-            os.makedirs(settings.WHITELIST_FILE.replace('whitelist', ''))
+            os.makedirs(settings.METRIC_FILTERS_FILE.replace('allowlist', ''))
         except OSError:
             pass
-        fh = open(settings.WHITELIST_FILE, 'wb')
+        fh = open(settings.METRIC_FILTERS_FILE, 'wb')
         pickle.dump({'a.b.c.d', 'e.f.g.h'}, fh)
         fh.close()
 
-    def test_whitelist_show_no_whitelist(self):
-        url = reverse('whitelist_show')
+    def test_allowed_metrics_show_no_allowlist(self):
+        url = reverse('metric_filters_show')
         with self.assertRaises(IOError):
             _ = self.client.get(url)
 
-    def test_whitelist_show(self):
-        url = reverse('whitelist_show')
-        self.create_whitelist()
-        self.addCleanup(self.wipe_whitelist)
+    def test_allowed_metrics_show(self):
+        url = reverse('metric_filters_show')
+        self.create_allowlist()
+        self.addCleanup(self.wipe_allowlist)
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content, b"a.b.c.d\ne.f.g.h")
 
-    def test_whitelist_add(self):
-        self.create_whitelist()
-        self.addCleanup(self.wipe_whitelist)
+    def test_allowed_metrics_add(self):
+        self.create_allowlist()
+        self.addCleanup(self.wipe_allowlist)
 
-        url = reverse('whitelist_add')
+        url = reverse('metric_filters_add')
         response = self.client.post(url, {'metrics': ['i.j.k.l']})
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content, b"OK")
 
-        url = reverse('whitelist_show')
+        url = reverse('metric_filters_show')
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content, b"a.b.c.d\ne.f.g.h\ni.j.k.l")
 
-    def test_whitelist_add_existing(self):
-        self.create_whitelist()
-        self.addCleanup(self.wipe_whitelist)
+    def test_allowed_metrics_add_existing(self):
+        self.create_allowlist()
+        self.addCleanup(self.wipe_allowlist)
 
-        url = reverse('whitelist_add')
+        url = reverse('metric_filters_add')
         response = self.client.post(url, {'metrics': ['a.b.c.d']})
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content, b"OK")
 
-        url = reverse('whitelist_show')
+        url = reverse('metric_filters_show')
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content, b"a.b.c.d\ne.f.g.h")
 
-    def test_whitelist_remove(self):
-        self.create_whitelist()
-        self.addCleanup(self.wipe_whitelist)
+    def test_allowed_metrics_remove(self):
+        self.create_allowlist()
+        self.addCleanup(self.wipe_allowlist)
 
-        url = reverse('whitelist_remove')
+        url = reverse('metric_filters_remove')
         response = self.client.post(url, {'metrics': ['a.b.c.d']})
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content, b"OK")
 
-        url = reverse('whitelist_show')
+        url = reverse('metric_filters_show')
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content, b"e.f.g.h")
 
-    def test_whitelist_remove_missing(self):
-        self.create_whitelist()
-        self.addCleanup(self.wipe_whitelist)
+    def test_allowed_metrics_remove_missing(self):
+        self.create_allowlist()
+        self.addCleanup(self.wipe_allowlist)
 
-        url = reverse('whitelist_remove')
+        url = reverse('metric_filters_remove')
         response = self.client.post(url, {'metrics': ['i.j.k.l']})
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content, b"OK")
 
-        url = reverse('whitelist_show')
+        url = reverse('metric_filters_show')
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content, b"a.b.c.d\ne.f.g.h")
 
-    def test_save_whitelist(self):
+    def test_save_allowed_metrics(self):
         try:
-            os.makedirs(settings.WHITELIST_FILE.replace('whitelist', ''))
+            os.makedirs(settings.METRIC_FILTERS_FILE.replace('allowlist', ''))
         except OSError:
             pass
-        self.addCleanup(self.wipe_whitelist)
-        self.assertEqual(save_whitelist({'a.b.c.d','e.f.g.h'}), None)
-        self.assertEqual(load_whitelist(), {'a.b.c.d','e.f.g.h'})
+        self.addCleanup(self.wipe_allowlist)
+        self.assertEqual(save_allowed_metrics({'a.b.c.d','e.f.g.h'}), None)
+        self.assertEqual(load_allowed_metrics(), {'a.b.c.d','e.f.g.h'})
 
     @mock.patch('os.rename')
-    def test_save_whitelist_rename_failure(self, rename):
-        self.addCleanup(self.wipe_whitelist)
+    def test_save_allowed_metrics_rename_failure(self, rename):
+        self.addCleanup(self.wipe_allowlist)
         rename.side_effect = OSError(errno.EPERM, 'Operation not permitted')
         with self.assertRaises(OSError):
-            save_whitelist({'a.b.c.d','e.f.g.h'})
+            save_allowed_metrics({'a.b.c.d','e.f.g.h'})