diff --git a/docs/tags.rst b/docs/tags.rst index 31ffd2412..a7f222e52 100644 --- a/docs/tags.rst +++ b/docs/tags.rst @@ -113,14 +113,19 @@ The default settings (above) will connect to a local Redis server on the default HTTP(S) TagDB ^^^^^^^^^^^^^ -The HTTP(S) TagDB is used to delegate all tag operations to an external server that implements the Graphite tagging HTTP API. It can be used in clustered graphite scenarios, or with custom data stores. It is selected by setting ``TAGDB='graphite.tags.http.HttpTagDB'`` in `local_settings.py`. There are 3 additional config settings for the HTTP(S) TagDB:: +The HTTP(S) TagDB is used to delegate all tag operations to an external server that implements the Graphite tagging HTTP API. It can be used in clustered graphite scenarios, or with custom data stores. It is selected by setting ``TAGDB='graphite.tags.http.HttpTagDB'`` in `local_settings.py`. There are 4 additional config settings for the HTTP(S) TagDB:: TAGDB_HTTP_URL = 'https://another.server' TAGDB_HTTP_USER = '' TAGDB_HTTP_PASSWORD = '' + TAGDB_HTTP_AUTOCOMPLETE = False The ``TAGDB_HTTP_URL`` is required. ``TAGDB_HTTP_USER`` and ``TAGDB_HTTP_PASSWORD`` are optional and if specified will be used to send a Basic Authorization header in all requests. +``TAGDB_HTTP_AUTOCOMPLETE`` is also optional, if set to ``True`` auto-complete requests will be forwarded to the remote TagDB, otherwise calls to `/tags/findSeries`, `/tags` & `/tags/` will be used to provide auto-complete functionality. + +If ``REMOTE_STORE_FORWARD_HEADERS`` is defined, those headers will also be forwarded to the remote TagDB. + Adding Series to the TagDB -------------------------- Normally `carbon` will take care of this, it submits all new series to the TagDB, and periodically re-submits all series to ensure that the TagDB is kept up to date. There are 2 `carbon` configuration settings related to tagging; the `GRAPHITE_URL` setting specifies the url of your graphite-web installation (default `http://127.0.0.1:8000`), and the `TAG_UPDATE_INTERVAL` setting specifies how often each series should be re-submitted to the TagDB (default is every 100th update). @@ -136,6 +141,22 @@ Series can be submitted via HTTP POST using command-line tools such as ``curl`` This endpoint returns the canonicalized version of the path, with the tags sorted in alphabetical order. +To add multiple series with a single HTTP request, use the ``/tags/tagMultiSeries`` endpoint, which support multiple ``path`` parameters: + +.. code-block:: none + + $ curl -X POST "http://graphite/tags/tagMultiSeries" \ + --data-urlencode 'path=disk.used;rack=a1;datacenter=dc1;server=web01' \ + --data-urlencode 'path=disk.used;rack=a1;datacenter=dc1;server=web02' \ + --data-urlencode 'pretty=1' + + [ + "disk.used;datacenter=dc1;rack=a1;server=web01", + "disk.used;datacenter=dc1;rack=a1;server=web02" + ] + +This endpoint returns a list of the canonicalized paths, in the same order they are specified. + Exploring Tags -------------- You can use the HTTP api to get lists of defined tags, values for each tag, and to find series using the same logic as the `seriesByTag `_ function. @@ -309,3 +330,13 @@ Series can be deleted via HTTP POST to the `/tags/delSeries` endpoint: --data-urlencode 'path=disk.used;datacenter=dc1;rack=a1;server=web01' true + +To delete multiple series at once pass multiple ``path`` parameters: + +.. code-block:: none + + $ curl -X POST "http://graphite/tags/delSeries" \ + --data-urlencode 'path=disk.used;datacenter=dc1;rack=a1;server=web01' \ + --data-urlencode 'path=disk.used;datacenter=dc1;rack=a1;server=web02' + + true diff --git a/webapp/graphite/storage.py b/webapp/graphite/storage.py index ce1bdf85d..28916ad98 100755 --- a/webapp/graphite/storage.py +++ b/webapp/graphite/storage.py @@ -54,7 +54,7 @@ def __init__(self, finders=None, tagdb=None): if tagdb is None: tagdb = settings.TAGDB - self.tagdb = get_tagdb(tagdb) if tagdb else None + self.tagdb = get_tagdb(tagdb) def get_finders(self, local=False): for finder in self.finders: diff --git a/webapp/graphite/tags/base.py b/webapp/graphite/tags/base.py index 5a966149c..78fe0ec9f 100644 --- a/webapp/graphite/tags/base.py +++ b/webapp/graphite/tags/base.py @@ -147,12 +147,26 @@ def tag_series(self, series, requestContext=None): Enter series into database. Accepts a series string, upserts into the TagDB and returns the canonicalized series name. """ + def tag_multi_series(self, seriesList, requestContext=None): + """ + Enter series into database. Accepts a list of series strings, upserts into the TagDB and returns a list of canonicalized series names. + """ + return [self.tag_series(series, requestContext) for series in seriesList] + @abc.abstractmethod def del_series(self, series, requestContext=None): """ Remove series from database. Accepts a series string and returns True """ + def del_multi_series(self, seriesList, requestContext=None): + """ + Remove series from database. Accepts a list of series strings, removes them from the TagDB and returns True + """ + for series in seriesList: + self.del_series(series, requestContext) + return True + def auto_complete_tags(self, exprs, tagPrefix=None, limit=None, requestContext=None): """ Return auto-complete suggestions for tags based on the matches for the specified expressions, optionally filtered by tag prefix diff --git a/webapp/graphite/tags/http.py b/webapp/graphite/tags/http.py index 33ee44f96..db9bda68c 100644 --- a/webapp/graphite/tags/http.py +++ b/webapp/graphite/tags/http.py @@ -1,6 +1,5 @@ from __future__ import absolute_import -from urllib import quote import json from graphite.http_pool import http @@ -19,10 +18,7 @@ def __init__(self, settings, *args, **kwargs): self.username = settings.TAGDB_HTTP_USER self.password = settings.TAGDB_HTTP_PASSWORD - def request(self, method, url, fields=None, requestContext=None): - if not fields: - fields = {} - + def request(self, method, url, fields, requestContext=None): headers = requestContext.get('forwardHeaders') if requestContext else {} if 'Authorization' not in headers and self.username and self.password: headers['Authorization'] = 'Basic ' + ('%s:%s' % (self.username, self.password)).encode('base64') @@ -35,8 +31,11 @@ def request(self, method, url, fields=None, requestContext=None): timeout=self.settings.REMOTE_FIND_TIMEOUT, ) + if result.status == 400: + raise ValueError(json.loads(result.data.decode('utf-8')).get('error')) + if result.status != 200: - raise Exception('HTTP Error from remote tagdb: %s' % result.status) + raise Exception('HTTP Error from remote tagdb: %s %s' % (result.status, result.data)) return json.loads(result.data.decode('utf-8')) @@ -51,8 +50,9 @@ def find_series_cachekey(self, tags, requestContext=None): def _find_series(self, tags, requestContext=None): return self.request( - 'GET', - '/tags/findSeries?' + '&'.join([('expr=%s' % quote(tag)) for tag in tags]), + 'POST', + '/tags/findSeries', + {'expr': tags}, requestContext=requestContext, ) @@ -83,9 +83,15 @@ def list_values(self, tag, valueFilter=None, limit=None, requestContext=None): def tag_series(self, series, requestContext=None): return self.request('POST', '/tags/tagSeries', {'path': series}, requestContext) + def tag_multi_series(self, seriesList, requestContext=None): + return self.request('POST', '/tags/tagMultiSeries', {'path': seriesList}, requestContext) + def del_series(self, series, requestContext=None): return self.request('POST', '/tags/delSeries', {'path': series}, requestContext) + def del_multi_series(self, seriesList, requestContext=None): + return self.request('POST', '/tags/delSeries', {'path': seriesList}, requestContext) + def auto_complete_tags(self, exprs, tagPrefix=None, limit=None, requestContext=None): """ Return auto-complete suggestions for tags based on the matches for the specified expressions, optionally filtered by tag prefix @@ -97,10 +103,13 @@ def auto_complete_tags(self, exprs, tagPrefix=None, limit=None, requestContext=N if limit is None: limit = self.settings.TAGDB_AUTOCOMPLETE_LIMIT - url = '/tags/autoComplete/tags?tagPrefix=' + quote(tagPrefix or '') + '&limit=' + quote(str(limit)) + \ - '&' + '&'.join([('expr=%s' % quote(expr or '')) for expr in exprs]) + fields = { + 'tagPrefix': tagPrefix or '', + 'limit': str(limit), + 'expr': exprs, + } - return self.request('GET', url) + return self.request('POST', '/tags/autoComplete/tags', fields, requestContext) def auto_complete_values(self, exprs, tag, valuePrefix=None, limit=None, requestContext=None): """ @@ -113,7 +122,11 @@ def auto_complete_values(self, exprs, tag, valuePrefix=None, limit=None, request if limit is None: limit = self.settings.TAGDB_AUTOCOMPLETE_LIMIT - url = '/tags/autoComplete/values?tag=' + quote(tag or '') + '&valuePrefix=' + quote(valuePrefix or '') + \ - '&limit=' + quote(str(limit)) + '&' + '&'.join([('expr=%s' % quote(expr or '')) for expr in exprs]) + fields = { + 'tag': tag or '', + 'valuePrefix': valuePrefix or '', + 'limit': str(limit), + 'expr': exprs, + } - return self.request('GET', url) + return self.request('POST', '/tags/autoComplete/values', fields, requestContext) diff --git a/webapp/graphite/tags/urls.py b/webapp/graphite/tags/urls.py index 7789e1810..f720e85f2 100644 --- a/webapp/graphite/tags/urls.py +++ b/webapp/graphite/tags/urls.py @@ -17,6 +17,7 @@ urlpatterns = [ url('tagSeries', views.tagSeries, name='tagSeries'), + url('tagMultiSeries', views.tagMultiSeries, name='tagMultiSeries'), url('delSeries', views.delSeries, name='delSeries'), url('findSeries', views.findSeries, name='findSeries'), url('autoComplete/tags', views.autoCompleteTags, name='tagAutoCompleteTags'), diff --git a/webapp/graphite/tags/views.py b/webapp/graphite/tags/views.py index b14c43f48..7c972e3e7 100644 --- a/webapp/graphite/tags/views.py +++ b/webapp/graphite/tags/views.py @@ -1,56 +1,108 @@ +from functools import wraps + from graphite.compat import HttpResponse from graphite.util import json from graphite.storage import STORE, extractForwardHeaders +def jsonResponse(f): + @wraps(f) + def wrapped_f(request, *args, **kwargs): + if request.method == 'GET': + queryParams = request.GET.copy() + elif request.method == 'POST': + queryParams = request.GET.copy() + queryParams.update(request.POST) + else: + queryParams = {} + + try: + return _jsonResponse(f(request, queryParams, *args, **kwargs), queryParams) + except ValueError as err: + return _jsonError(err.message, queryParams, getattr(err, 'status', 400)) + except Exception as err: + return _jsonError(err.message, queryParams, getattr(err, 'status', 500)) + + return wrapped_f + +class HttpError(Exception): + def __init__(self, message, status=500): + super(HttpError, self).__init__(message) + self.status=status + +def _jsonResponse(data, queryParams, status=200): + if isinstance(data, HttpResponse): + return data + + if not queryParams: + queryParams = {} + + return HttpResponse( + json.dumps( + data, + indent=(2 if queryParams.get('pretty') else None), + sort_keys=bool(queryParams.get('pretty')) + ) if data is not None else 'null', + content_type='application/json', + status=status + ) + +def _jsonError(message, queryParams, status=500): + return _jsonResponse({'error': message}, queryParams, status=status) + def _requestContext(request): return { 'forwardHeaders': extractForwardHeaders(request), } -def tagSeries(request): +@jsonResponse +def tagSeries(request, queryParams): if request.method != 'POST': return HttpResponse(status=405) - path = request.POST.get('path') + path = queryParams.get('path') if not path: - return HttpResponse( - json.dumps({'error': 'no path specified'}), - content_type='application/json', - status=400 - ) + raise HttpError('no path specified', status=400) - return HttpResponse( - json.dumps( - STORE.tagdb.tag_series(path, requestContext=_requestContext(request)), - ) if STORE.tagdb else 'null', - content_type='application/json' - ) + return STORE.tagdb.tag_series(path, requestContext=_requestContext(request)) -def delSeries(request): +@jsonResponse +def tagMultiSeries(request, queryParams): if request.method != 'POST': return HttpResponse(status=405) - path = request.POST.get('path') - if not path: - return HttpResponse( - json.dumps({'error': 'no path specified'}), - content_type='application/json', - status=400 - ) + paths = [] + # Normal format: ?path=name;tag1=value1;tag2=value2&path=name;tag1=value2;tag2=value2 + if len(queryParams.getlist('path')) > 0: + paths = queryParams.getlist('path') + # Rails/PHP/jQuery common practice format: ?path[]=...&path[]=... + elif len(queryParams.getlist('path[]')) > 0: + paths = queryParams.getlist('path[]') + else: + raise HttpError('no paths specified',status=400) - return HttpResponse( - json.dumps( - STORE.tagdb.del_series(path, requestContext=_requestContext(request)), - ) if STORE.tagdb else 'null', - content_type='application/json' - ) + return STORE.tagdb.tag_multi_series(paths, requestContext=_requestContext(request)) -def findSeries(request): - if request.method not in ['GET', 'POST']: +@jsonResponse +def delSeries(request, queryParams): + if request.method != 'POST': return HttpResponse(status=405) - queryParams = request.GET.copy() - queryParams.update(request.POST) + paths = [] + # Normal format: ?path=name;tag1=value1;tag2=value2&path=name;tag1=value2;tag2=value2 + if len(queryParams.getlist('path')) > 0: + paths = queryParams.getlist('path') + # Rails/PHP/jQuery common practice format: ?path[]=...&path[]=... + elif len(queryParams.getlist('path[]')) > 0: + paths = queryParams.getlist('path[]') + else: + raise HttpError('no path specified', status=400) + + return STORE.tagdb.del_multi_series(paths, requestContext=_requestContext(request)) + +@jsonResponse +def findSeries(request, queryParams): + if request.method not in ['GET', 'POST']: + return HttpResponse(status=405) exprs = [] # Normal format: ?expr=tag1=value1&expr=tag2=value2 @@ -61,66 +113,38 @@ def findSeries(request): exprs = queryParams.getlist('expr[]') if not exprs: - return HttpResponse( - json.dumps({'error': 'no tag expressions specified'}), - content_type='application/json', - status=400 - ) + raise HttpError('no tag expressions specified', status=400) - return HttpResponse( - json.dumps( - STORE.tagdb.find_series( - exprs, - requestContext=_requestContext(request), - ) if STORE.tagdb else [], - indent=(2 if queryParams.get('pretty') else None), - sort_keys=bool(queryParams.get('pretty')) - ), - content_type='application/json' - ) + return STORE.tagdb.find_series(exprs, requestContext=_requestContext(request)) -def tagList(request): +@jsonResponse +def tagList(request, queryParams): if request.method != 'GET': return HttpResponse(status=405) - return HttpResponse( - json.dumps( - STORE.tagdb.list_tags( - tagFilter=request.GET.get('filter'), - limit=request.GET.get('limit'), - requestContext=_requestContext(request), - ) if STORE.tagdb else [], - indent=(2 if request.GET.get('pretty') else None), - sort_keys=bool(request.GET.get('pretty')) - ), - content_type='application/json' + return STORE.tagdb.list_tags( + tagFilter=request.GET.get('filter'), + limit=request.GET.get('limit'), + requestContext=_requestContext(request), ) -def tagDetails(request, tag): +@jsonResponse +def tagDetails(request, queryParams, tag): if request.method != 'GET': return HttpResponse(status=405) - return HttpResponse( - json.dumps( - STORE.tagdb.get_tag( - tag, - valueFilter=request.GET.get('filter'), - limit=request.GET.get('limit'), - requestContext=_requestContext(request), - ) if STORE.tagdb else None, - indent=(2 if request.GET.get('pretty') else None), - sort_keys=bool(request.GET.get('pretty')) - ), - content_type='application/json' + return STORE.tagdb.get_tag( + tag, + valueFilter=queryParams.get('filter'), + limit=queryParams.get('limit'), + requestContext=_requestContext(request), ) -def autoCompleteTags(request): +@jsonResponse +def autoCompleteTags(request, queryParams): if request.method not in ['GET', 'POST']: return HttpResponse(status=405) - queryParams = request.GET.copy() - queryParams.update(request.POST) - exprs = [] # Normal format: ?expr=tag1=value1&expr=tag2=value2 if len(queryParams.getlist('expr')) > 0: @@ -129,27 +153,18 @@ def autoCompleteTags(request): elif len(queryParams.getlist('expr[]')) > 0: exprs = queryParams.getlist('expr[]') - return HttpResponse( - json.dumps( - STORE.tagdb.auto_complete_tags( - exprs, - tagPrefix=queryParams.get('tagPrefix'), - limit=queryParams.get('limit'), - requestContext=_requestContext(request) - ) if STORE.tagdb else [], - indent=(2 if queryParams.get('pretty') else None), - sort_keys=bool(queryParams.get('pretty')) - ), - content_type='application/json' + return STORE.tagdb.auto_complete_tags( + exprs, + tagPrefix=queryParams.get('tagPrefix'), + limit=queryParams.get('limit'), + requestContext=_requestContext(request) ) -def autoCompleteValues(request): +@jsonResponse +def autoCompleteValues(request, queryParams): if request.method not in ['GET', 'POST']: return HttpResponse(status=405) - queryParams = request.GET.copy() - queryParams.update(request.POST) - exprs = [] # Normal format: ?expr=tag1=value1&expr=tag2=value2 if len(queryParams.getlist('expr')) > 0: @@ -160,23 +175,12 @@ def autoCompleteValues(request): tag = queryParams.get('tag') if not tag: - return HttpResponse( - json.dumps({'error': 'no tag specified'}), - content_type='application/json', - status=400 - ) - - return HttpResponse( - json.dumps( - STORE.tagdb.auto_complete_values( - exprs, - tag, - valuePrefix=queryParams.get('valuePrefix'), - limit=queryParams.get('limit'), - requestContext=_requestContext(request) - ) if STORE.tagdb else [], - indent=(2 if queryParams.get('pretty') else None), - sort_keys=bool(queryParams.get('pretty')) - ), - content_type='application/json' + raise HttpError('no tag specified', status=400) + + return STORE.tagdb.auto_complete_values( + exprs, + tag, + valuePrefix=queryParams.get('valuePrefix'), + limit=queryParams.get('limit'), + requestContext=_requestContext(request) ) diff --git a/webapp/tests/test_tags.py b/webapp/tests/test_tags.py index fd4c29c18..8b261972a 100644 --- a/webapp/tests/test_tags.py +++ b/webapp/tests/test_tags.py @@ -178,10 +178,27 @@ def _test_tagdb(self, db): with self.assertRaises(ValueError): db.find_series('test=') + # tag multiple series + result = db.tag_multi_series([ + 'test.a;blah=blah;hello=lion', + 'test.b;hello=lion;blah=blah', + 'test.c;blah=blah;hello=lion', + ]) + self.assertEqual(result, [ + 'test.a;blah=blah;hello=lion', + 'test.b;blah=blah;hello=lion', + 'test.c;blah=blah;hello=lion', + ]) + # delete series we added self.assertTrue(db.del_series('test.a;blah=blah;hello=tiger')) self.assertTrue(db.del_series('test.a;blah=blah;hello=lion')) + self.assertTrue(db.del_multi_series([ + 'test.b;blah=blah;hello=lion', + 'test.c;blah=blah;hello=lion', + ])) + def test_local_tagdb(self): return self._test_tagdb(LocalDatabaseTagDB(settings)) @@ -529,3 +546,75 @@ def test_tag_views(self): self.assertEqual(response.status_code, 200) self.assertEqual(response['Content-Type'], 'application/json') self.assertEqual(response.content, json.dumps(expected, indent=2, sort_keys=True)) + + # tag multiple series + + # get should fail + response = self.client.get(url + '/tagMultiSeries', {'path': 'test.a;hello=tiger;blah=blah'}) + self.assertEqual(response.status_code, 405) + + # post without path should fail + response = self.client.post(url + '/tagMultiSeries', {}) + self.assertEqual(response.status_code, 400) + self.assertEqual(response['Content-Type'], 'application/json') + + # multiple path should succeed + expected = [ + 'test.a;blah=blah;hello=tiger', + 'test.b;blah=blah;hello=tiger', + ] + + response = self.client.post(url + '/tagMultiSeries', { + 'path': [ + 'test.a;hello=tiger;blah=blah', + 'test.b;hello=tiger;blah=blah', + ], + 'pretty': '1', + }) + self.assertEqual(response.status_code, 200) + self.assertEqual(response['Content-Type'], 'application/json') + self.assertEqual(response.content, json.dumps(expected, indent=2, sort_keys=True)) + + # multiple path[] should succeed + expected = [ + 'test.a;blah=blah;hello=tiger', + 'test.b;blah=blah;hello=tiger', + ] + + response = self.client.post(url + '/tagMultiSeries', { + 'path[]': [ + 'test.a;hello=tiger;blah=blah', + 'test.b;hello=tiger;blah=blah', + ], + 'pretty': '1', + }) + self.assertEqual(response.status_code, 200) + self.assertEqual(response['Content-Type'], 'application/json') + self.assertEqual(response.content, json.dumps(expected, indent=2, sort_keys=True)) + + # remove multiple series + expected = True + + response = self.client.post(url + '/delSeries', { + 'path': [ + 'test.a;hello=tiger;blah=blah', + 'test.b;hello=tiger;blah=blah', + ], + 'pretty': '1', + }) + self.assertEqual(response.status_code, 200) + self.assertEqual(response['Content-Type'], 'application/json') + self.assertEqual(response.content, json.dumps(expected, indent=2, sort_keys=True)) + + expected = True + + response = self.client.post(url + '/delSeries', { + 'path[]': [ + 'test.a;hello=tiger;blah=blah', + 'test.b;hello=tiger;blah=blah', + ], + 'pretty': '1', + }) + self.assertEqual(response.status_code, 200) + self.assertEqual(response['Content-Type'], 'application/json') + self.assertEqual(response.content, json.dumps(expected, indent=2, sort_keys=True))