diff --git a/.travis.yml b/.travis.yml index c1ed08a20..e1c85093e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,46 +9,53 @@ matrix: - python: pypy env: - TOXENV=pypy-django111-pyparsing2 - - python: 3.4 - env: - - TOXENV=py34-django20-pyparsing2 - python: 3.5 + sudo: true + dist: xenial env: - TOXENV=py35-django21-pyparsing2 - python: 3.5 + sudo: true + dist: xenial env: - - TOXENV=py35-django21-pyparsing2 + - TOXENV=py35-django22-pyparsing2 - python: 3.6 + sudo: true + dist: xenial env: - - TOXENV=py36-django21-pyparsing2 + - TOXENV=py36-django22-pyparsing2 - python: 3.7 sudo: true dist: xenial env: - - TOXENV=py37-django21-pyparsing2-msgpack + - TOXENV=py37-django22-pyparsing2-msgpack - python: 3.7 sudo: true dist: xenial env: - - TOXENV=py37-django21-pyparsing2-pyhash + - TOXENV=py37-django22-pyparsing2-pyhash - python: 3.7 sudo: true dist: xenial env: - - TOXENV=py37-django21-pyparsing2-mysql TEST_MYSQL_PASSWORD=graphite + - TOXENV=py37-django22-pyparsing2-mysql TEST_MYSQL_PASSWORD=graphite - python: 3.7 sudo: true dist: xenial env: - - TOXENV=py37-django21-pyparsing2-postgresql TEST_POSTGRESQL_PASSWORD=graphite - - python: 3.7 + - TOXENV=py37-django22-pyparsing2-postgresql TEST_POSTGRESQL_PASSWORD=graphite + - python: "3.8-dev" sudo: true dist: xenial env: - TOXENV=lint + - python: "3.8-dev" + sudo: true + dist: xenial + env: + - TOXENV=py38-django22-pyparsing2 env: - - TOXENV=py27-django18-pyparsing2 - TOXENV=py27-django111-pyparsing2-msgpack - TOXENV=py27-django111-pyparsing2-pyhash - TOXENV=py27-django111-pyparsing2-mysql TEST_MYSQL_PASSWORD=graphite diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..a8286c0f4 --- /dev/null +++ b/ISSUE_TEMPLATE.md @@ -0,0 +1,39 @@ + + + + +**I'm submitting a ...** + +[ ] bug report +[ ] feature request + + + +**Current behavior:** + + +**Expected behavior:** + + +**Steps to reproduce:** + + +**Related code:** + + + +``` +insert short code snippets here +``` + +**Other information:** + + diff --git a/MANIFEST.in b/MANIFEST.in index bd94f4e0d..18ccc8e90 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include LICENSE +include README.md include MANIFEST.in include check-dependencies.py include examples/* diff --git a/docs/ceres.rst b/docs/ceres.rst index cc3fb6cf2..1ec57980f 100644 --- a/docs/ceres.rst +++ b/docs/ceres.rst @@ -6,7 +6,7 @@ for Graphite. In contrast with Whisper, Ceres is not a fixed-size database and i better support sparse data of arbitrary fixed-size resolutions. This allows Graphite to distribute individual time-series across multiple servers or mounts. -Ceres is not actively developped at the moment. For alternatives to whisper look at :doc:`alternative storage backends `. +Ceres is not actively developed at the moment. For alternatives to whisper look at :doc:`alternative storage backends `. Storage Overview ---------------- diff --git a/docs/client-apis.rst b/docs/client-apis.rst index 2e220abc2..c8011c773 100644 --- a/docs/client-apis.rst +++ b/docs/client-apis.rst @@ -27,5 +27,5 @@ txCarbonClient .. _Cubism.js: http://square.github.io/cubism/ .. _Graphitejs: https://github.com/prestontimmons/graphitejs .. _Scales: https://github.com/Cue/scales -.. _structured_metrics: https://github.com/vimeo/graph-explorer/tree/master/structured_metrics +.. _structured_metrics: https://github.com/vimeo/graph-explorer/tree/master/graph_explorer/structured_metrics .. _txCarbonClient: https://github.com/fdChasm/txCarbonClient diff --git a/docs/config-database-setup.rst b/docs/config-database-setup.rst index 87b309fcb..18ec781ff 100644 --- a/docs/config-database-setup.rst +++ b/docs/config-database-setup.rst @@ -20,7 +20,10 @@ To set up a new database and create the initial schema, run: .. code-block:: none - PYTHONPATH=$GRAPHITE_ROOT/webapp django-admin.py migrate --settings=graphite.settings --run-syncdb + PYTHONPATH=$GRAPHITE_ROOT/webapp django-admin.py migrate --settings=graphite.settings + +.. note :: + Graphite-Web 1.0 and earlier had some models without migrations, and with Django 1.9 or later, the ``--run-syncdb`` option was needed for migrate to create tables for these models. (Django 1.8 and earlier did not have this option, but always exhibited this behavior.) In Graphite-Web 1.1 and later all models have migrations, so ``--run-syncdb`` is no longer needed. If upgrading a database created by Graphite-Web 1.0 or earlier, you need to use the ``--fake-initial`` option for migrate: it considers an initial migration to already be applied if the tables it creates already exist. If you are experiencing problems, uncomment the following line in /opt/graphite/webapp/graphite/local_settings.py: diff --git a/docs/config-local-settings.rst b/docs/config-local-settings.rst index 4af89463e..f5580100e 100644 --- a/docs/config-local-settings.rst +++ b/docs/config-local-settings.rst @@ -208,6 +208,10 @@ TAGDB_REDIS_DB `Default: 0` Redis database to use with `TAGDB = 'graphite.tags.redis.RedisTagDB'` +TAGDB_REDIS_PASSWORD + `Default: ''` + Redis password to use with `TAGDB = 'graphite.tags.redis.RedisTagDB'` + Configure Webserver (Apache) ---------------------------- There is an example ``example-graphite-vhost.conf`` file in the examples directory of the graphite web source code. You can use this to configure apache. Different distributions have different ways of configuring Apache. Please refer to your distribution's documentation on the subject. @@ -373,7 +377,7 @@ The following configures the Django database settings. Graphite uses the databas See the `Django documentation `_ for full documentation of the DATABASES setting. .. note :: - Remember, setting up a new database requires running ``PYTHONPATH=$GRAPHITE_ROOT/webapp django-admin.py migrate --settings=graphite.settings --run-syncdb`` to create the initial schema. + Remember, setting up a new database requires running ``PYTHONPATH=$GRAPHITE_ROOT/webapp django-admin.py migrate --settings=graphite.settings`` to create the initial schema. .. note :: If you are using a custom database backend (other than SQLite) you must first create a $GRAPHITE_ROOT/webapp/graphite/local_settings.py file that overrides the database related settings from settings.py. Use $GRAPHITE_ROOT/webapp/graphite/local_settings.py.example as a template. diff --git a/docs/install.rst b/docs/install.rst index a9c4dea0a..b730783dc 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -8,14 +8,14 @@ Try Graphite in Docker and have it running in seconds: .. code-block:: none - docker run -d\ - --name graphite\ - --restart=always\ - -p 80:80\ - -p 2003-2004:2003-2004\ - -p 2023-2024:2023-2024\ - -p 8125:8125/udp\ - -p 8126:8126\ + docker run -d \ + --name graphite \ + --restart=always \ + -p 80:80 \ + -p 2003-2004:2003-2004 \ + -p 2023-2024:2023-2024 \ + -p 8125:8125/udp \ + -p 8126:8126 \ graphiteapp/graphite-statsd Check `docker repo`_ for details. diff --git a/docs/releases/1_1_1.rst b/docs/releases/1_1_1.rst index 31eae4cc8..66a1393fc 100644 --- a/docs/releases/1_1_1.rst +++ b/docs/releases/1_1_1.rst @@ -176,7 +176,9 @@ If you're not already running from the *master* branch, Graphite-Web's applicati sudo cp /opt/graphite/storage/graphite.db \ /opt/graphite/storage/graphite.db.backup-`date +%Y%m%d_%H%M%S` sudo PYTHONPATH=/opt/graphite/webapp django-admin.py migrate \ - --noinput --settings=graphite.settings --run-syncdb + --noinput --settings=graphite.settings --fake-initial + +In this release of Graphite-Web, migrations have been added for any Django models that did not have them. Previously, if using Django 1.9 or later, one needed the ``--run-syncdb`` option for migrate to create tables for Graphite-Web models without migrations (and Django 1.8 or earlier did not have this option but always exhibited this behavior). Django keeps track of which migrations have been applied and attempts to run any that have not, and these new initial migrations try to create tables that already exist from previous versions of Graphite-Web, and fail. This common Django situation is resolved by the ``--fake-initial`` option for migrate: it considers an initial migration to already be applied if the tables it creates already exist. Other Changes diff --git a/docs/render_api.rst b/docs/render_api.rst index 171882205..2db88c2ac 100644 --- a/docs/render_api.rst +++ b/docs/render_api.rst @@ -531,7 +531,7 @@ darkgray 111,111,111 darkgrey 111,111,111 ============ ============= -RGB can be passed directly in the format #RRGGBB[AA] where RR, GG, and BB are 2-digit hex vaules for red, green and blue, respectively. AA is an optional addition describing the opacity ("alpha"). Where FF is fully opaque, 00 fully transparent. +RGB can be passed directly in the format #RRGGBB[AA] where RR, GG, and BB are 2-digit hex values for red, green and blue, respectively. AA is an optional addition describing the opacity ("alpha"). Where FF is fully opaque, 00 fully transparent. Examples: diff --git a/docs/requirements.txt b/docs/requirements.txt index caa60a862..137c8c3e5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,6 @@ ## Requirements for documentation -Django>=1.4,<1.8 -django-tagging==0.3.1 +Django>=1.11.19,<2.3 +django-tagging==0.4.6 sphinx sphinx_rtd_theme pytz diff --git a/docs/tags.rst b/docs/tags.rst index 647b2943f..bae542882 100644 --- a/docs/tags.rst +++ b/docs/tags.rst @@ -17,6 +17,8 @@ Carbon will automatically decode the tags, normalize the tag order, and register .. _querying-tagged-series: +Tag names must have a length >= 1 and may contain any ascii characters except ``;!^=``. Tag values must also have a length >= 1, and they may contain any ascii characters except ``;~``. UTF-8 characters may work for names and values, but they are not well tested and it is not recommended to use non-ascii characters in metric names or tags. + Querying -------- @@ -88,6 +90,8 @@ Finally, the `aliasByTags # web01.disk.used # web02.disk.used +If a tag name or value contains quotes (``'"``), then they will need to be escaped properly. For example a series with a tag ``tagName='quotedValue'`` could be queried with ``seriesByTag('tagName=\'quotedValue\'')`` or alternatively ``seriesByTag("tagName='quotedValue'")``. + Database Storage ---------------- As Whisper and other storage backends are designed to hold simple time-series data (metric key, value, and timestamp), Graphite stores tag information in a separate tag database (TagDB). The TagDB is a pluggable store, by default it uses the Graphite SQLite, MySQL or PostgreSQL database, but it can also be configured to use an external Redis server or a custom plugin. @@ -104,13 +108,14 @@ The Local TagDB stores tag information in tables inside the graphite-web databas Redis TagDB ^^^^^^^^^^^ -The Redis TagDB will store the tag information on a Redis server, and is selected by setting ``TAGDB='graphite.tags.redis.RedisTagDB'`` in `local_settings.py`. There are 3 additional config settings for the Redis TagDB:: +The Redis TagDB will store the tag information on a Redis server, and is selected by setting ``TAGDB='graphite.tags.redis.RedisTagDB'`` in `local_settings.py`. There are 4 additional config settings for the Redis TagDB:: TAGDB_REDIS_HOST = 'localhost' TAGDB_REDIS_PORT = 6379 TAGDB_REDIS_DB = 0 + TAGDB_REDIS_PASSWORD = '' -The default settings (above) will connect to a local Redis server on the default port, and use the default database. +The default settings (above) will connect to a local Redis server on the default port, and use the default database without password. HTTP(S) TagDB ^^^^^^^^^^^^^ diff --git a/docs/tools.rst b/docs/tools.rst index cb86d00a7..4decef334 100644 --- a/docs/tools.rst +++ b/docs/tools.rst @@ -57,6 +57,9 @@ Collection `metrics-sampler`_ A java program which regularly queries metrics from a configured set of inputs, selects and renames them using regular expressions and sends them to a configured set of outputs. It supports JMX and JDBC as inputs and Graphite as output out of the box. +`netdata`_ + A fast and efficient monitoring agent that supports graphite backends. It has collection, forwarding, visualization and monitoring features. Netdata collects common system metrics and a variety of other sources through plugins. + `Sensu`_ A monitoring framework that can route metrics to Graphite. Servers subscribe to sets of checks, so getting metrics from a new server to Graphite is as simple as installing the Sensu client and subscribing. @@ -125,6 +128,9 @@ Forwarding `statsd`_ A simple daemon for easy stats aggregation, developed by the folks at Etsy. A list of forks and alternative implementations can be found at +`Statusengine`_ + A daemon written in PHP to store Nagios and Naemon performance data to Graphite. + Visualization ------------- @@ -381,6 +387,7 @@ Other .. _metrics-sampler: https://github.com/dimovelev/metrics-sampler .. _metrictank: https://github.com/grafana/metrictank .. _Moira: http://moira.readthedocs.io +.. _netdata: https://github.com/netdata/netdata .. _New Relic: https://newrelic.com/platform .. _Pencil: https://github.com/fetep/pencil .. _pipe-to-graphite: https://github.com/iFixit/pipe-to-graphite @@ -399,6 +406,7 @@ Other .. _SqlToGraphite: https://github.com/perryofpeek/SqlToGraphite .. _SSC Serv: https://ssc-serv.com .. _statsd: https://github.com/etsy/statsd +.. _Statusengine: https://github.com/statusengine/worker .. _Tasseo: https://github.com/obfuscurity/tasseo .. _Targets-io: https://github.com/dmoll1974/targets-io .. _telegraf: https://github.com/influxdata/telegraf diff --git a/requirements.txt b/requirements.txt index 56f7560c2..63f74f315 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ # deactivate # -Django>=1.8,<2.1.99 +Django>=1.8,<2.3 python-memcached==1.58 txAMQP==0.8 django-tagging==0.4.6 diff --git a/setup.py b/setup.py index 8e4ee9805..463192928 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,10 @@ conf_files = [ ('conf', glob('conf/*.example')) ] examples = [ ('examples', glob('examples/example-*')) ] +def read(fname): + with open(os.path.join(os.path.dirname(__file__), fname)) as f: + return f.read() + try: setup( name='graphite-web', @@ -80,6 +84,8 @@ author_email='chrismd@gmail.com', license='Apache Software License 2.0', description='Enterprise scalable realtime graphing', + long_description=read('README.md'), + long_description_content_type='text/markdown', package_dir={'' : 'webapp'}, packages=[ 'graphite', @@ -109,7 +115,7 @@ ['templates/*', 'local_settings.py.example']}, scripts=glob('bin/*'), data_files=list(webapp_content.items()) + storage_dirs + conf_files + examples, - install_requires=['Django>=1.8,<2.1', 'django-tagging==0.4.3', 'pytz', 'pyparsing', 'cairocffi', 'urllib3', 'scandir', 'six'], + install_requires=['Django>=1.8,<2.3', 'django-tagging==0.4.3', 'pytz', 'pyparsing', 'cairocffi', 'urllib3', 'scandir', 'six'], classifiers=[ 'Intended Audience :: Developers', 'Natural Language :: English', @@ -122,6 +128,7 @@ 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', ], diff --git a/tox.ini b/tox.ini index b06adf085..61ea9e74d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,7 @@ [tox] -# Django 21 does not support Python 3.4 -# Django 1.11 has a bug and does not work with Python 3.7 (this will be fixed) envlist = - py{27,py}-django1{8,11}-pyparsing2{,-rrdtool,-msgpack,-pyhash}, - py{34}-django{18,111,20}-pyparsing2{,-rrdtool,-msgpack,-pyhash}, - py{35,36}-django{18,111,20,21}-pyparsing2{,-rrdtool,-msgpack,-pyhash}, - py37-django{20,21}-pyparsing2{,-rrdtool,-msgpack,-pyhash}, + py{27,py}-django111-pyparsing2{,-msgpack,-pyhash}, + py{35,36,37,38}-django{111,21,22}-pyparsing2{,-msgpack,-pyhash}, lint, docs [testenv] @@ -32,12 +28,10 @@ deps = git+git://github.com/graphite-project/whisper.git#egg=whisper git+git://github.com/graphite-project/ceres.git#egg=ceres pyparsing2: pyparsing - django18: Django>=1.8,<1.8.99 - django19: Django>=1.9,<1.9.99 - django110: Django>=1.10,<1.10.99 django111: Django>=1.11,<1.11.99 django20: Django>=2.0,<2.0.99 django21: Django>=2.1,<2.1.99 + django22: Django>=2.2,<2.2.99 scandir urllib3 redis @@ -56,7 +50,7 @@ deps = pytz git+git://github.com/graphite-project/whisper.git#egg=whisper git+git://github.com/graphite-project/ceres.git#egg=ceres - Django<2.0 + Django<2.3 pyparsing Sphinx<1.4 sphinx_rtd_theme @@ -67,7 +61,7 @@ commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html [testenv:lint] -basepython = python3.7 +basepython = python3.8 deps = flake8 commands = diff --git a/webapp/content/js/composer_widgets.js b/webapp/content/js/composer_widgets.js index aafc0bd8e..e1ce36c7a 100644 --- a/webapp/content/js/composer_widgets.js +++ b/webapp/content/js/composer_widgets.js @@ -1085,28 +1085,32 @@ function createFunctionsMenu() { }, { text: 'Transform', menu: [ - {text: 'Scale', handler: applyFuncToEachWithInput('scale', 'Please enter a scale factor')}, - {text: 'ScaleToSeconds', handler: applyFuncToEachWithInput('scaleToSeconds', 'Please enter a number of seconds to scale to')}, + {text: 'Absolute Value', handler: applyFuncToEach('absolute')}, + {text: 'Add', handler: applyFuncToEachWithInput('add', 'Please enter a constant')}, + {text: 'Delay', handler: applyFuncToEachWithInput('delay', 'Please enter the number of steps to delay')}, + {text: 'Derivative', handler: applyFuncToEach('derivative')}, + {text: 'Exp', handler: applyFuncToEach('exp')}, + {text: 'Hit Count', handler: applyFuncToEachWithInput('hitcount', 'Please enter a summary interval (examples: 10min, 1h, 7d)', {quote: true})}, + {text: 'Interpolate', handler: applyFuncToEach('interpolate')}, + {text: 'Integral', handler: applyFuncToEach('integral')}, + {text: 'Integral by Interval', handler: applyFuncToEachWithInput('integralByInterval', 'Integral this metric with a reset every ___ (examples: 1d, 1h, 10min)', {quote: true})}, + {text: 'Invert', handler: applyFuncToEach('invert')}, + {text: 'Log', handler: applyFuncToEachWithInput('log', 'Please enter a base')}, + {text: 'Logit', handler: applyFuncToEach('logit')}, + {text: 'Non-negative Derivative', handler: applyFuncToEachWithInput('nonNegativeDerivative', 'Please enter a maximum value if this metric is a wrapping counter (or just leave this blank)', {allowBlank: true})}, {text: 'Offset', handler: applyFuncToEachWithInput('offset', 'Please enter the value to offset Y-values by')}, {text: 'OffsetToZero', handler: applyFuncToEach('offsetToZero')}, - {text: 'Interpolate', handler: applyFuncToEach('interpolate')}, - {text: 'Derivative', handler: applyFuncToEach('derivative')}, + {text: 'Percentile Values', handler: applyFuncToEachWithInput('percentileOfSeries', 'Please enter the percentile to use')}, {text: 'Power', handler: applyFuncToEachWithInput('pow', 'Please enter a power factor')}, {text: 'Power Series', handler: applyFuncToEachWithInput('powSeries', 'Please enter at least 2 series')}, + {text: 'Summarize', handler: applyFuncToEachWithInput('summarize', 'Please enter a summary interval (examples: 10min, 1h, 7d)', {quote: true})}, + {text: 'Scale', handler: applyFuncToEachWithInput('scale', 'Please enter a scale factor')}, + {text: 'ScaleToSeconds', handler: applyFuncToEachWithInput('scaleToSeconds', 'Please enter a number of seconds to scale to')}, + {text: 'Sigmoid', handler: applyFuncToEach('sigmoid')}, {text: 'Square Root', handler: applyFuncToEach('squareRoot')}, - {text: 'Time-adjusted Derivative', handler: applyFuncToEachWithInput('perSecond', 'Please enter a maximum value if this metric is a wrapping counter (or just leave this blank)', {allowBlank: true})}, - {text: 'Delay', handler: applyFuncToEachWithInput('delay', 'Please enter the number of steps to delay')}, - {text: 'Integral', handler: applyFuncToEach('integral')}, - {text: 'Integral by Interval', handler: applyFuncToEachWithInput('integralByInterval', 'Integral this metric with a reset every ___ (examples: 1d, 1h, 10min)', {quote: true})}, - {text: 'Percentile Values', handler: applyFuncToEachWithInput('percentileOfSeries', 'Please enter the percentile to use')}, - {text: 'Non-negative Derivative', handler: applyFuncToEachWithInput('nonNegativeDerivative', 'Please enter a maximum value if this metric is a wrapping counter (or just leave this blank)', {allowBlank: true})}, - {text: 'Log', handler: applyFuncToEachWithInput('log', 'Please enter a base')}, - {text: 'Invert', handler: applyFuncToEach('invert')}, - {text: 'Absolute Value', handler: applyFuncToEach('absolute')}, {text: 'timeShift', handler: applyFuncToEachWithInput('timeShift', 'Shift this metric ___ back in time (examples: 10min, 7d, 2w)', {quote: true})}, {text: 'timeSlice', handler: applyFuncToEachWithInput('timeSlice', 'Start showing metric at (example: 14:57 20150115)', {quote: true})}, - {text: 'Summarize', handler: applyFuncToEachWithInput('summarize', 'Please enter a summary interval (examples: 10min, 1h, 7d)', {quote: true})}, - {text: 'Hit Count', handler: applyFuncToEachWithInput('hitcount', 'Please enter a summary interval (examples: 10min, 1h, 7d)', {quote: true})} + {text: 'Time-adjusted Derivative', handler: applyFuncToEachWithInput('perSecond', 'Please enter a maximum value if this metric is a wrapping counter (or just leave this blank)', {allowBlank: true})} ] }, { text: 'Calculate', diff --git a/webapp/content/js/dashboard.js b/webapp/content/js/dashboard.js index 6c981ddb7..1be7c640b 100644 --- a/webapp/content/js/dashboard.js +++ b/webapp/content/js/dashboard.js @@ -4,7 +4,7 @@ /*global AUTOCOMPLETE_DELAY CALENDAR_ICON CLOCK_ICON CONTEXT_FIELD_WIDTH FINDER_QUERY_DELAY*/ /*global HELP_ICON NEW_DASHBOARD_REMOVE_GRAPHS REFRESH_ICON REMOVE_ICON RESIZE_ICON*/ /*global SHARE_ICON UI_CONFIG initialState initialError permissions queryString userName*/ -/*global permissionsUnauthenticated schemes*/ +/*global UP_ICON DOWN_ICON TRASH_ICON permissionsUnauthenticated schemes*/ // Defined in composer_widgets.js /*global createFunctionsMenu createOptionsMenu updateCheckItems*/ @@ -792,7 +792,7 @@ function initDashboard () { if(window.location.hash != '') { if (window.location.hash.indexOf('/') != -1) { - var nameVal = window.location.hash.substr(1).split('#'); + var nameVal = window.location.hash.substr(1).split('/'); sendLoadTemplateRequest(nameVal[0],nameVal[1]); } else { sendLoadRequest(window.location.hash.substr(1)); @@ -1400,7 +1400,7 @@ function newEmptyGraph() { var record = new GraphRecord({ target: graphTargetString, params: myParams, - url: '/render?' + Ext.urlEncode(urlParams), + url: document.body.dataset.baseUrl + 'render?' + Ext.urlEncode(urlParams), 'width': GraphSize.width, 'height': GraphSize.height, }); @@ -1566,7 +1566,7 @@ function newFromMetric() { var record = new GraphRecord({ target: graphTargetString, params: myParams, - url: '/render?' + Ext.urlEncode(urlParams) + url: document.body.dataset.baseUrl + 'render?' + Ext.urlEncode(urlParams) }); graphStore.add([record]); updateGraphRecords(); @@ -1916,7 +1916,7 @@ function graphClicked(graphView, graphIndex, element, evt) { width: 30, sortable: false, items: [{ - icon: '/static/img/move_up.png', + icon: UP_ICON, tooltip: 'Move Up', handler: function(grid, rowIndex, colIndex) { var record = targetStore.getAt(rowIndex); @@ -1935,7 +1935,7 @@ function graphClicked(graphView, graphIndex, element, evt) { width: 30, sortable: false, items: [{ - icon: '/static/img/move_down.png', + icon: DOWN_ICON, tooltip: 'Move Down', handler: function(grid, rowIndex, colIndex) { var record = targetStore.getAt(rowIndex); @@ -1954,7 +1954,7 @@ function graphClicked(graphView, graphIndex, element, evt) { width: 30, sortable: false, items: [{ - icon: '/static/img/trash.png', + icon: TRASH_ICON, tooltip: 'Delete Row', handler: function(grid, rowIndex, colIndex) { var record = targetStore.getAt(rowIndex); diff --git a/webapp/graphite/app_settings.py b/webapp/graphite/app_settings.py index 9b39c990f..a037aa896 100644 --- a/webapp/graphite/app_settings.py +++ b/webapp/graphite/app_settings.py @@ -109,6 +109,7 @@ 'django.contrib.sessions', 'django.contrib.admin', 'django.contrib.contenttypes', + 'django.contrib.messages', 'django.contrib.staticfiles', 'tagging', ) diff --git a/webapp/graphite/carbonlink.py b/webapp/graphite/carbonlink.py index e50d2ec3d..378d0402f 100644 --- a/webapp/graphite/carbonlink.py +++ b/webapp/graphite/carbonlink.py @@ -7,7 +7,7 @@ from graphite.render.hashing import ConsistentHashRing from graphite.logger import log -from graphite.util import load_module, unpickle +from graphite.util import load_module, unpickle, parseHosts from graphite.singleton import ThreadSafeSingleton @@ -83,10 +83,8 @@ def get_connection(self, host): pass #nothing left in the pool, gotta make a new connection log.cache("CarbonLink creating a new socket for %s" % str(host)) - connection = socket.socket() - connection.settimeout(self.timeout) try: - connection.connect((server, port)) + connection = socket.create_connection((server, port), self.timeout) except socket.error: self.last_failure[host] = time.time() raise @@ -193,16 +191,7 @@ def recv_exactly(conn, num_bytes): @ThreadSafeSingleton class GlobalCarbonLinkPool(CarbonLinkPool): def __init__(self): - hosts = [] - for host in settings.CARBONLINK_HOSTS: - parts = host.split(':') - server = parts[0] - port = int(parts[1]) - if len(parts) > 2: - instance = parts[2] - else: - instance = None - hosts.append((server, int(port), instance)) + hosts = parseHosts(settings.CARBONLINK_HOSTS) timeout = settings.CARBONLINK_TIMEOUT CarbonLinkPool.__init__(self, hosts, timeout) diff --git a/webapp/graphite/composer/urls.py b/webapp/graphite/composer/urls.py index c741c7b56..d3faefea7 100644 --- a/webapp/graphite/composer/urls.py +++ b/webapp/graphite/composer/urls.py @@ -16,7 +16,6 @@ from . import views urlpatterns = [ - url(r'^/send_email', views.send_email, name='composer_send_email'), url(r'^/mygraph', views.mygraph, name='composer_mygraph'), url(r'^/?$', views.composer, name='composer'), ] diff --git a/webapp/graphite/composer/views.py b/webapp/graphite/composer/views.py index 129c8b187..4f61931b4 100644 --- a/webapp/graphite/composer/views.py +++ b/webapp/graphite/composer/views.py @@ -13,15 +13,6 @@ limitations under the License.""" import os -from smtplib import SMTP -from socket import gethostname -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText -from email.mime.image import MIMEImage -from six.moves.http_client import HTTPConnection -from six.moves.urllib.parse import urlsplit -from time import ctime, strftime -from traceback import format_exc from graphite.user_util import getProfile from graphite.logger import log from graphite.account.models import MyGraph @@ -90,35 +81,3 @@ def mygraph(request): else: return HttpResponse("Invalid operation '%s'" % action) - - -def send_email(request): - try: - recipients = request.GET['to'].split(',') - url = request.GET['url'] - proto, server, path, query, frag = urlsplit(url) - if query: path += '?' + query - conn = HTTPConnection(server) - conn.request('GET',path) - try: # Python 2.7+, use buffering of HTTP responses - resp = conn.getresponse(buffering=True) - except TypeError: # Python 2.6 and older - resp = conn.getresponse() - assert resp.status == 200, "Failed HTTP response %s %s" % (resp.status, resp.reason) - rawData = resp.read() - conn.close() - message = MIMEMultipart() - message['Subject'] = "Graphite Image" - message['To'] = ', '.join(recipients) - message['From'] = 'composer@%s' % gethostname() - text = MIMEText( "Image generated by the following graphite URL at %s\r\n\r\n%s" % (ctime(),url) ) - image = MIMEImage( rawData ) - image.add_header('Content-Disposition', 'attachment', filename="composer_" + strftime("%b%d_%I%M%p.png")) - message.attach(text) - message.attach(image) - s = SMTP(settings.SMTP_SERVER) - s.sendmail('composer@%s' % gethostname(),recipients,message.as_string()) - s.quit() - return HttpResponse( "OK" ) - except Exception: - return HttpResponse(format_exc()) diff --git a/webapp/graphite/errors.py b/webapp/graphite/errors.py index 1349153d0..2a11e74ab 100644 --- a/webapp/graphite/errors.py +++ b/webapp/graphite/errors.py @@ -1,3 +1,7 @@ class NormalizeEmptyResultError(Exception): # throw error for normalize() when empty pass + + +class InputParameterError(ValueError): + pass diff --git a/webapp/graphite/functions/params.py b/webapp/graphite/functions/params.py index 68a0c7ab3..47630c472 100644 --- a/webapp/graphite/functions/params.py +++ b/webapp/graphite/functions/params.py @@ -1,3 +1,9 @@ +import six + +from graphite.errors import InputParameterError +from graphite.render.attime import parseTimeOffset + + class ParamTypes(object): pass @@ -21,6 +27,41 @@ class ParamTypes(object): setattr(ParamTypes, paramType, paramType) +def validateBoolean(value): + return isinstance(value, bool) + + +def validateFloat(value): + return isinstance(value, float) or validateInteger(value) + + +def validateInteger(value): + return isinstance(value, six.integer_types) + + +def validateInterval(value): + try: + parseTimeOffset(value) + except Exception: + return False + return True + + +def validateSeriesList(value): + return isinstance(value, list) + + +typeValidators = { + 'boolean': validateBoolean, + 'float': validateFloat, + 'integer': validateInteger, + 'interval': validateInterval, + 'node': validateInteger, + 'seriesList': validateSeriesList, + 'seriesLists': validateSeriesList, +} + + class Param(object): __slots__ = ('name', 'type', 'required', 'default', 'multiple', 'options', 'suggestions') @@ -52,3 +93,59 @@ def toJSON(self): if self.suggestions: jsonVal['suggestions'] = self.suggestions return jsonVal + + def validateValue(self, value): + # None is ok for optional params + if not self.required and value is None: + return True + + validator = typeValidators.get(self.type, None) + # if there's no validator for the type we assume True + if validator is None: + return True + + return validator(value) + + +def validateParams(func, params, args, kwargs): + valid_args = [] + + if len(params) == 0 or params[len(params)-1].multiple is False: + if len(args) + len(kwargs) > len(params): + raise InputParameterError( + 'Too many parameters specified for function "{func}"'.format(func=func)) + + for i in range(len(params)): + if len(args) <= i: + # requirement is satisfied from "kwargs" + value = kwargs.get(params[i].name, None) + if value is None: + if params[i].required: + # required parameter is missing + raise InputParameterError( + 'Missing required parameter "{param}" for function "{func}"'.format( + param=params[i].name, func=func)) + else: + # got multiple values for keyword argument + if params[i].name in valid_args: + raise InputParameterError( + 'Keyword parameter "{param}" specified multiple times for function "{func}"'.format( + param=params[i].name, func=func)) + else: + # requirement is satisfied from "args" + value = args[i] + + # parameter is restricted to a defined set of values, but value is not in it + if params[i].options and value not in params[i].options: + raise InputParameterError( + 'Invalid option specified for function "{func}" parameter "{param}"'.format( + param=params[i].name, func=func)) + + if not params[i].validateValue(value): + raise InputParameterError( + 'Invalid {type} value specified for function "{func}" parameter "{param}"'.format( + type=params[i].type, param=params[i].name, func=func)) + + valid_args.append(params[i].name) + + return True diff --git a/webapp/graphite/local_settings.py.example b/webapp/graphite/local_settings.py.example index b5378fae4..ac4ee53ea 100644 --- a/webapp/graphite/local_settings.py.example +++ b/webapp/graphite/local_settings.py.example @@ -374,6 +374,7 @@ DEFAULT_XFILES_FACTOR = 0 #TAGDB_REDIS_HOST = 'localhost' #TAGDB_REDIS_PORT = 6379 #TAGDB_REDIS_DB = 0 +#TAGDB_REDIS_PASSWORD = '' # Settings for HTTP TagDB #TAGDB_HTTP_URL = '' diff --git a/webapp/graphite/render/evaluator.py b/webapp/graphite/render/evaluator.py index dde13f1a3..abf5db262 100644 --- a/webapp/graphite/render/evaluator.py +++ b/webapp/graphite/render/evaluator.py @@ -1,10 +1,14 @@ import re import six -from graphite.errors import NormalizeEmptyResultError +from graphite.errors import NormalizeEmptyResultError, InputParameterError from graphite.functions import SeriesFunction +from graphite.logger import log from graphite.render.grammar import grammar from graphite.render.datalib import fetchData, TimeSeries, prefetchData +from graphite.functions.params import validateParams + +from django.conf import settings def evaluateTarget(requestContext, targets): @@ -76,12 +80,20 @@ def evaluateTokens(requestContext, tokens, replacements=None, pipedArg=None): if tokens.call.funcname == 'template': # if template propagates down here, it means the grammar didn't match the invocation # as tokens.template. this generally happens if you try to pass non-numeric/string args - raise ValueError("invalid template() syntax, only string/numeric arguments are allowed") + raise InputParameterError("invalid template() syntax, only string/numeric arguments are allowed") if tokens.call.funcname == 'seriesByTag': return fetchData(requestContext, tokens.call.raw) - func = SeriesFunction(tokens.call.funcname) + try: + func = SeriesFunction(tokens.call.funcname) + except KeyError: + msg = 'Received request for unknown function: {func}'.format(func=tokens.call.funcname) + log.warning(msg) + + # even if input validation enforcement is disabled, there's nothing else we can do here + raise InputParameterError(msg) + rawArgs = tokens.call.args or [] if pipedArg is not None: rawArgs.insert(0, pipedArg) @@ -89,10 +101,40 @@ def evaluateTokens(requestContext, tokens, replacements=None, pipedArg=None): requestContext['args'] = rawArgs kwargs = dict([(kwarg.argname, evaluateTokens(requestContext, kwarg.args[0], replacements)) for kwarg in tokens.call.kwargs]) + + def handleInvalidParameters(e): + if not getattr(handleInvalidParameters, 'alreadyLogged', False): + log.warning( + 'Received invalid parameters ({msg}): {func} ({args})'.format( + msg=str(e.message), + func=tokens.call.funcname, + args=', '.join( + argList + for argList in [ + ', '.join(str(arg) for arg in args), + ', '.join('{k}={v}'.format(k=str(k), v=str(v)) for k, v in kwargs.items()), + ] if argList + ) + )) + + # only log invalid parameters once + setattr(handleInvalidParameters, 'alreadyLogged', True) + + if settings.ENFORCE_INPUT_VALIDATION: + raise + + if hasattr(func, 'params'): + try: + validateParams(tokens.call.funcname, func.params, args, kwargs) + except InputParameterError as e: + handleInvalidParameters(e) + try: return func(requestContext, *args, **kwargs) except NormalizeEmptyResultError: return [] + except InputParameterError as e: + handleInvalidParameters(e) return evaluateScalarTokens(tokens) @@ -106,7 +148,7 @@ def evaluateScalarTokens(tokens): if tokens.number.scientific: return float(tokens.number.scientific[0]) - raise ValueError("unknown numeric type in target evaluator") + raise InputParameterError("unknown numeric type in target evaluator") if tokens.string: return tokens.string[1:-1] @@ -117,7 +159,7 @@ def evaluateScalarTokens(tokens): if tokens.none: return None - raise ValueError("unknown token in target evaluator") + raise InputParameterError("unknown token in target evaluator") def extractPathExpressions(requestContext, targets): diff --git a/webapp/graphite/render/functions.py b/webapp/graphite/render/functions.py index 9497e4ff5..03a041b6e 100644 --- a/webapp/graphite/render/functions.py +++ b/webapp/graphite/render/functions.py @@ -34,8 +34,7 @@ from os import environ from django.conf import settings - -from graphite.errors import NormalizeEmptyResultError +from graphite.errors import NormalizeEmptyResultError, InputParameterError from graphite.events import models from graphite.functions import SeriesFunction, ParamTypes, Param from graphite.logger import log @@ -85,6 +84,13 @@ def safeDiv(a, b): return a / b +def safeExp(a): + try: + return math.exp(a) + except TypeError: + return None + + def safePow(a, b): if a is None: return None if b is None: return None @@ -280,7 +286,7 @@ def formatPathExpressions(seriesList): 'current': aggFuncs['last'], } -aggFuncNames = sorted(aggFuncs.keys()) +aggFuncNames = sorted(aggFuncs.keys()) + sorted(aggFuncAliases.keys()) def getAggFunc(func, rawFunc=None): @@ -288,7 +294,7 @@ def getAggFunc(func, rawFunc=None): return aggFuncs[func] if func in aggFuncAliases: return aggFuncAliases[func] - raise ValueError('Unsupported aggregation function: %s' % (rawFunc or func)) + raise InputParameterError('Unsupported aggregation function: %s' % (rawFunc or func)) def aggregate(requestContext, seriesList, func, xFilesFactor=None): @@ -307,8 +313,9 @@ def aggregate(requestContext, seriesList, func, xFilesFactor=None): &target=sumSeries(host.cpu-[0-7].cpu-{user,system}.value) - This function can be used with aggregation functions ``average``, ``median``, ``sum``, ``min``, - ``max``, ``diff``, ``stddev``, ``count``, ``range``, ``multiply`` & ``last``. + This function can be used with aggregation functions ``average`` (or ``avg``), ``avg_zero``, + ``median``, ``sum`` (or ``total``), ``min``, ``max``, ``diff``, ``stddev``, ``count``, + ``range`` (or ``rangeOf``) , ``multiply`` & ``last`` (or ``current``). """ # strip Series from func if func was passed like sumSeries rawFunc = func @@ -664,7 +671,7 @@ def percentileOfSeries(requestContext, seriesList, n, interpolate=False): supplied series. """ if n <= 0: - raise ValueError('The requested percent is required to be greater than 0') + raise InputParameterError('The requested percent is required to be greater than 0') # if seriesList is empty then just short-circuit if not seriesList: @@ -937,7 +944,7 @@ def asPercent(requestContext, seriesList, total=None, *nodes): totalSeries[key] = TimeSeries(name,start,end,step,totalValues,xFilesFactor=xFilesFactor) # trying to use nodes with a total value, which isn't supported because it has no effect else: - raise ValueError('total must be None or a seriesList') + raise InputParameterError('total must be None or a seriesList') resultList = [] for key in keys: @@ -972,7 +979,7 @@ def asPercent(requestContext, seriesList, total=None, *nodes): totalText = "sumSeries(%s)" % formatPathExpressions(seriesList) elif type(total) is list: if len(total) != 1 and len(total) != len(seriesList): - raise ValueError("asPercent second argument must be missing, a single digit, reference exactly 1 series or reference the same number of series as the first argument") + raise InputParameterError("asPercent second argument must be missing, a single digit, reference exactly 1 series or reference the same number of series as the first argument") if len(total) == 1: normalize([seriesList, total]) @@ -1016,11 +1023,10 @@ def divideSeriesLists(requestContext, dividendSeriesList, divisorSeriesList): """ if len(dividendSeriesList) != len(divisorSeriesList): - raise ValueError("dividendSeriesList and divisorSeriesList argument must have equal length") + raise InputParameterError("dividendSeriesList and divisorSeriesList argument must have equal length") results = [] - #for dividendSeries in dividendSeriesList: for i in range(0, len(dividendSeriesList)): dividendSeries = dividendSeriesList[i] divisorSeries = divisorSeriesList[i] @@ -1074,7 +1080,7 @@ def divideSeries(requestContext, dividendSeriesList, divisorSeries): series[i] = None return dividendSeriesList if len(divisorSeries) > 1: - raise ValueError("divideSeries second argument must reference exactly 1 series (got {0})".format(len(divisorSeries))) + raise InputParameterError("divideSeries second argument must reference exactly 1 series (got {0})".format(len(divisorSeries))) divisorSeries = divisorSeries[0] results = [] @@ -1239,6 +1245,7 @@ def movingWindow(requestContext, seriesList, windowSize, func='average', xFilesF # ignore original data and pull new, including our preview # data from earlier is needed to calculate the early results newContext = requestContext.copy() + newContext['prefetch'] = {} newContext['startTime'] = requestContext['startTime'] - timedelta(seconds=previewSeconds) previewList = evaluateTarget(newContext, requestContext['args'][0]) result = [] @@ -1324,6 +1331,7 @@ def exponentialMovingAverage(requestContext, seriesList, windowSize): # ignore original data and pull new, including our preview # data from earlier is needed to calculate the early results newContext = requestContext.copy() + newContext['prefetch'] = {} newContext['startTime'] = requestContext['startTime'] - timedelta(seconds=previewSeconds) previewList = evaluateTarget(newContext, requestContext['args'][0]) result = [] @@ -1453,6 +1461,132 @@ def scaleToSeconds(requestContext, seriesList, seconds): ] +def exp(requestContext, seriesList): + """ + Raise e to the power of the datapoint, + where e = 2.718281... is the base of natural logarithms. + + Example: + + .. code-block:: none + + &target=exp(Server.instance01.threads.busy) + + """ + for series in seriesList: + series.tags['exp'] = 'e' + series.name = "exp(%s)" % (series.name) + series.pathExpression = series.name + for i, value in enumerate(series): + series[i] = safeExp(value) + + return seriesList + + +exp.group = 'Transform' +exp.params = [ + Param('seriesList', ParamTypes.seriesList, required=True), +] + + +def add(requestContext, seriesList, constant): + """ + Takes one metric or a wildcard seriesList followed by a constant, and adds the + constant to each datapoint. Also works for negative numbers. + + Example: + + .. code-block:: none + + &target=add(Server.instance01.threads.busy, 10) + &target=add(Server.instance*.threads.busy, 10) + + """ + for series in seriesList: + series.tags['add'] = constant + series.name = "add(%s,%d)" % (series.name, constant) + series.pathExpression = series.name + for i, value in enumerate(series): + try: + series[i] = value + constant + except TypeError: + series[i] = None + + return seriesList + + +add.group = 'Transform' +add.params = [ + Param('seriesList', ParamTypes.seriesList, required=True), + Param('constant', ParamTypes.float, required=True), +] + + +def sigmoid(requestContext, seriesList): + """ + Takes one metric or a wildcard seriesList and applies the sigmoid + function `1 / (1 + exp(-x))` to each datapoint. + + Example: + + .. code-block:: none + + &target=sigmoid(Server.instance01.threads.busy) + &target=sigmoid(Server.instance*.threads.busy) + + """ + for series in seriesList: + series.tags['sigmoid'] = 'sigmoid' + series.name = "sigmoid(%s)" % series.name + series.pathExpression = series.name + for i, value in enumerate(series): + try: + log.info(value) + series[i] = 1 / (1 + safeExp(-value)) + except (TypeError, ValueError, ZeroDivisionError): + series[i] = None + + return seriesList + + +sigmoid.group = 'Transform' +sigmoid.params = [ + Param('seriesList', ParamTypes.seriesList, required=True), +] + + +def logit(requestContext, seriesList): + """ + Takes one metric or a wildcard seriesList and applies the logit + function `log(x / (1 - x))` to each datapoint. + + Example: + + .. code-block:: none + + &target=logit(Server.instance01.threads.busy) + &target=logit(Server.instance*.threads.busy) + + """ + for series in seriesList: + series.tags['logit'] = 'logit' + series.name = "logit(%s)" % series.name + series.pathExpression = series.name + for i, value in enumerate(series): + try: + series[i] = math.log(value / (1 - value)) + except (TypeError, ValueError, ZeroDivisionError): + series[i] = None + + return seriesList + + +logit.group = 'Transform' +logit.params = [ + Param('seriesList', ParamTypes.seriesList, required=True), +] + + def pow(requestContext, seriesList, factor): """ Takes one metric or a wildcard seriesList followed by a constant, and raises the datapoint @@ -2338,12 +2472,12 @@ def areaBetween(requestContext, seriesList): def aliasSub(requestContext, seriesList, search, replace): - """ + r""" Runs series names through a regex search/replace. .. code-block:: none - &target=aliasSub(ip.*TCP*,"^.*TCP(\d+)","\\1") + &target=aliasSub(ip.*TCP*,"^.*TCP(\d+)","\1") """ try: seriesList.name = re.sub(search, replace, seriesList.name) @@ -2362,12 +2496,12 @@ def aliasSub(requestContext, seriesList, search, replace): def aliasQuery(requestContext, seriesList, search, replace, newName): - """ + r""" Performs a query to alias the metrics in seriesList. .. code-block:: none - &target=aliasQuery(channel.power.*,"channel\.power\.([0-9]+)","channel.frequency.\\1", "Channel %d MHz") + &target=aliasQuery(channel.power.*,"channel\.power\.([0-9]+)","channel.frequency.\1", "Channel %d MHz") The series in seriesList will be aliased by first translating the series names using the search & replace parameters, then using the last value of the resulting series @@ -2375,12 +2509,14 @@ def aliasQuery(requestContext, seriesList, search, replace, newName): """ for series in seriesList: newQuery = re.sub(search, replace, series.name) - newSeriesList = evaluateTarget(requestContext, newQuery) + newContext = requestContext.copy() + newContext['prefetch'] = {} + newSeriesList = evaluateTarget(newContext, newQuery) if newSeriesList is None or len(newSeriesList) == 0: - raise Exception('No series found with query: ' + newQuery) + raise InputParameterError('No series found with query: ' + newQuery) current = safeLast(newSeriesList[0]) if current is None: - raise Exception('Cannot get last value of series: ' + newSeriesList[0]) + raise InputParameterError('Cannot get last value of series: ' + newSeriesList[0]) series.name = newName % current return seriesList @@ -2753,7 +2889,7 @@ def filterSeries(requestContext, seriesList, func, operator, threshold): consolidationFunc = getAggFunc(func) if operator not in operatorFuncs: - raise Exception('Unsupported operator: %s' % (operator)) + raise InputParameterError('Unsupported operator: %s' % (operator)) operatorFunc = operatorFuncs[operator] # if seriesList is empty then just short-circuit @@ -2844,7 +2980,7 @@ def maximumBelow(requestContext, seriesList, n): def minimumBelow(requestContext, seriesList, n): """ Takes one metric or a wildcard seriesList followed by a constant n. - Draws only the metrics with a minimum value below n. + Draws only the metrics with a minimum value below or equal to n. Example: @@ -3515,7 +3651,9 @@ def useSeriesAbove(requestContext, seriesList, value, search, replace): if not newNames: return [] - newSeries = evaluateTarget(requestContext, 'group(%s)' % ','.join(newNames)) + newContext = requestContext.copy() + newContext['prefetched'] = {} + newSeries = evaluateTarget(newContext, 'group(%s)' % ','.join(newNames)) return [n for n in newSeries if n is not None and len(n) > 0] @@ -3810,6 +3948,7 @@ def holtWintersForecast(requestContext, seriesList, bootstrapInterval='7d', seas # ignore original data and pull new, including our preview newContext = requestContext.copy() + newContext['prefetch'] = {} newContext['startTime'] = requestContext['startTime'] - timedelta(seconds=previewSeconds) previewList = evaluateTarget(newContext, requestContext['args'][0]) results = [] @@ -3844,6 +3983,7 @@ def holtWintersConfidenceBands(requestContext, seriesList, delta=3, bootstrapInt # ignore original data and pull new, including our preview newContext = requestContext.copy() + newContext['prefetch'] = {} newContext['startTime'] = requestContext['startTime'] - timedelta(seconds=previewSeconds) previewList = evaluateTarget(newContext, requestContext['args'][0]) results = [] @@ -4004,6 +4144,7 @@ def linearRegression(requestContext, seriesList, startSourceAt=None, endSourceAt """ results = [] sourceContext = requestContext.copy() + sourceContext['prefetch'] = {} if startSourceAt is not None: sourceContext['startTime'] = parseATTime(startSourceAt) if endSourceAt is not None: sourceContext['endTime'] = parseATTime(endSourceAt) @@ -4158,6 +4299,7 @@ def timeStack(requestContext, seriesList, timeShiftUnit='1d', timeShiftStart=0, for shft in range(timeShiftStartint,timeShiftEndint): myContext = requestContext.copy() + myContext['prefetch'] = {} innerDelta = delta * shft myContext['startTime'] = requestContext['startTime'] + innerDelta myContext['endTime'] = requestContext['endTime'] + innerDelta @@ -4218,6 +4360,7 @@ def timeShift(requestContext, seriesList, timeShift, resetEnd=True, alignDST=Fal timeShift = '-' + timeShift delta = parseTimeOffset(timeShift) myContext = requestContext.copy() + myContext['prefetch'] = {} myContext['startTime'] = requestContext['startTime'] + delta myContext['endTime'] = requestContext['endTime'] + delta @@ -4423,9 +4566,9 @@ def verticalLine(requestContext, ts, label=None, color=None): start = int(timestamp( requestContext['startTime'] )) end = int(timestamp( requestContext['endTime'] )) if ts < start: - raise ValueError("verticalLine(): timestamp %s exists before start of range" % ts) + raise InputParameterError("verticalLine(): timestamp %s exists before start of range" % ts) elif ts > end: - raise ValueError("verticalLine(): timestamp %s exists after end of range" % ts) + raise InputParameterError("verticalLine(): timestamp %s exists after end of range" % ts) start = end = ts step = 1.0 series = TimeSeries(label, start, end, step, [1.0, 1.0], xFilesFactor=requestContext.get('xFilesFactor')) @@ -4802,8 +4945,10 @@ def applyByNode(requestContext, seriesList, nodeNum, templateFunction, newName=N prefix = '.'.join(series.name.split('.')[:nodeNum + 1]) prefixes.add(prefix) results = [] + newContext = requestContext.copy() + newContext['prefetch'] = {} for prefix in sorted(prefixes): - for resultSeries in evaluateTarget(requestContext, templateFunction.replace('%', prefix)): + for resultSeries in evaluateTarget(newContext, templateFunction.replace('%', prefix)): if newName: resultSeries.name = newName.replace('%', prefix) resultSeries.pathExpression = prefix @@ -4848,7 +4993,7 @@ def groupByNode(requestContext, seriesList, nodeNum, callback='average'): groupByNode.params = [ Param('seriesList', ParamTypes.seriesList, required=True), Param('nodeNum', ParamTypes.nodeOrTag, required=True), - Param('callback', ParamTypes.aggFunc, default='average', options=aggFuncNames, required=True), + Param('callback', ParamTypes.aggFunc, default='average', options=aggFuncNames), ] @@ -4969,6 +5114,7 @@ def smartSummarize(requestContext, seriesList, intervalString, func='sum', align if alignTo is not None: alignToUnit = getUnitString(alignTo) requestContext = requestContext.copy() + requestContext['prefetch'] = {} s = requestContext['startTime'] if alignToUnit == YEARS_STRING: requestContext['startTime'] = datetime(s.year, 1, 1, tzinfo = s.tzinfo) @@ -5139,6 +5285,7 @@ def hitcount(requestContext, seriesList, intervalString, alignToInterval = False if alignToInterval: requestContext = requestContext.copy() + requestContext['prefetch'] = {} s = requestContext['startTime'] if interval >= DAY: requestContext['startTime'] = datetime(s.year, s.month, s.day, tzinfo = s.tzinfo) @@ -5316,7 +5463,7 @@ def removeEmptySeries(requestContext, seriesList, xFilesFactor=None): removeEmptySeries.group = 'Filter Series' removeEmptySeries.params = [ Param('seriesList', ParamTypes.seriesList, required=True), - Param('xFilesFactor', ParamTypes.float, required=True), + Param('xFilesFactor', ParamTypes.float), ] @@ -5442,15 +5589,16 @@ def groupByTags(requestContext, seriesList, callback, *tags): averageSeries(seriesByTag("name=cpu","dc=dc1")),averageSeries(seriesByTag("name=cpu","dc=dc2")),... This function can be used with all aggregation functions supported by - :py:func:`aggregate `: ``average``, ``median``, ``sum``, ``min``, ``max``, ``diff``, - ``stddev``, ``range`` & ``multiply``. + :py:func:`aggregate `: ``average`` (or ``avg``), ``avg_zero``, + ``median``, ``sum`` (or ``total``), ``min``, ``max``, ``diff``, ``stddev``, ``count``, + ``range`` (or ``rangeOf``) , ``multiply`` & ``last`` (or ``current``). """ if STORE.tagdb is None: log.info('groupByTags called but no TagDB configured') return [] if not tags: - raise ValueError("groupByTags(): no tags specified") + raise InputParameterError("groupByTags(): no tags specified") # if all series have the same "name" tag use that for results, otherwise use the callback # if we're grouping by name, then the name is always used (see below) @@ -5671,9 +5819,11 @@ def pieMinimum(requestContext, series): 'weightedAverage': weightedAverage, # Transform functions + 'add': add, 'absolute': absolute, 'delay': delay, 'derivative': derivative, + 'exp': exp, 'hitcount': hitcount, 'integral': integral, 'integralByInterval' : integralByInterval, @@ -5681,6 +5831,7 @@ def pieMinimum(requestContext, series): 'invert': invert, 'keepLastValue': keepLastValue, 'log': logarithm, + 'logit': logit, 'minMax': minMax, 'nonNegativeDerivative': nonNegativeDerivative, 'offset': offset, @@ -5691,6 +5842,7 @@ def pieMinimum(requestContext, series): 'round': roundFunction, 'scale': scale, 'scaleToSeconds': scaleToSeconds, + 'sigmoid': sigmoid, 'smartSummarize': smartSummarize, 'squareRoot': squareRoot, 'summarize': summarize, diff --git a/webapp/graphite/render/views.py b/webapp/graphite/render/views.py index 27790b457..b7ca5cf9b 100755 --- a/webapp/graphite/render/views.py +++ b/webapp/graphite/render/views.py @@ -19,10 +19,10 @@ from datetime import datetime from time import time from random import shuffle -from six.moves.urllib.parse import urlencode, urlsplit, urlunsplit -from cgi import parse_qs +from six.moves.urllib.parse import urlencode, urlsplit, urlunsplit, parse_qs from graphite.compat import HttpResponse +from graphite.errors import InputParameterError from graphite.user_util import getProfileByUsername from graphite.util import json, unpickle, pickle, msgpack, BytesIO from graphite.storage import extractForwardHeaders @@ -34,7 +34,7 @@ from graphite.render.glyph import GraphTypes from graphite.tags.models import Series, Tag, TagValue, SeriesTag # noqa # pylint: disable=unused-import -from django.http import HttpResponseServerError, HttpResponseRedirect +from django.http import HttpResponseServerError, HttpResponseRedirect, HttpResponseBadRequest from django.template import Context, loader from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist @@ -46,6 +46,17 @@ loadFunctions() +def handleInputParameterError(f): + def new_f(*args, **kwargs): + try: + return f(*args, **kwargs) + except InputParameterError as e: + return HttpResponseBadRequest('Bad Request: {err}'.format(err=e)) + + return new_f + + +@handleInputParameterError def renderView(request): start = time() (graphOptions, requestOptions) = parseOptions(request) @@ -63,6 +74,7 @@ def renderView(request): 'data' : [], 'prefetched' : {}, 'xFilesFactor' : requestOptions['xFilesFactor'], + 'maxDataPoints' : requestOptions.get('maxDataPoints', None), } data = requestContext['data'] diff --git a/webapp/graphite/settings.py b/webapp/graphite/settings.py index 269308478..fe641acc1 100644 --- a/webapp/graphite/settings.py +++ b/webapp/graphite/settings.py @@ -124,7 +124,6 @@ REMOTE_RENDER_CONNECT_TIMEOUT = 1.0 #Miscellaneous settings -SMTP_SERVER = "localhost" DOCUMENTATION_VERSION = 'latest' if 'dev' in WEBAPP_VERSION else WEBAPP_VERSION DOCUMENTATION_URL = 'https://graphite.readthedocs.io/en/{}/'.format(DOCUMENTATION_VERSION) ALLOW_ANONYMOUS_CLI = True @@ -145,6 +144,7 @@ TAGDB_REDIS_HOST = 'localhost' TAGDB_REDIS_PORT = 6379 TAGDB_REDIS_DB = 0 +TAGDB_REDIS_PASSWORD = '' TAGDB_HTTP_URL = '' TAGDB_HTTP_USER = '' @@ -182,6 +182,13 @@ # Django 1.5 requires this so we set a default but warn the user SECRET_KEY = 'UNSAFE_DEFAULT' +# Input validation +# - When False we still validate the received input parameters, but if validation +# detects an issue it only logs an error and doesn't directly reject the request +# - When True we reject requests of which the input validation detected an issue with the +# provided arguments and return an error message to the user +ENFORCE_INPUT_VALIDATION = False + # Django 1.5 requires this to be set. Here we default to prior behavior and allow all ALLOWED_HOSTS = [ '*' ] diff --git a/webapp/graphite/tags/redis.py b/webapp/graphite/tags/redis.py index 7ede6ca3d..c034122b7 100644 --- a/webapp/graphite/tags/redis.py +++ b/webapp/graphite/tags/redis.py @@ -32,6 +32,7 @@ def __init__(self, settings, *args, **kwargs): host=settings.TAGDB_REDIS_HOST, port=settings.TAGDB_REDIS_PORT, db=settings.TAGDB_REDIS_DB, + password=settings.TAGDB_REDIS_PASSWORD, decode_responses=(sys.version_info[0] >= 3), ) diff --git a/webapp/graphite/tags/utils.py b/webapp/graphite/tags/utils.py index 37854b8e7..632cd753f 100644 --- a/webapp/graphite/tags/utils.py +++ b/webapp/graphite/tags/utils.py @@ -5,6 +5,22 @@ class TaggedSeries(object): + prohibitedTagChars = ';!^=' + prohibitedValueChars = ';~' + + @classmethod + def validateCharacters(cls, tag, value): + """validate that there are no prohibited characters in given tag/value""" + for char in cls.prohibitedTagChars: + if char in tag: + return False + + for char in cls.prohibitedValueChars: + if char in value: + return False + + return True + @classmethod def parse(cls, path): # if path is in openmetrics format: metric{tag="value",...} @@ -31,7 +47,13 @@ def parse_openmetrics(cls, path): if not m: raise Exception('Cannot parse path %s, invalid segment %s' % (path, rawtags)) - tags[m.group(1)] = m.group(2).replace(r'\"', '"').replace(r'\\', '\\') + tag = m.group(1) + value = m.group(2).replace(r'\"', '"').replace(r'\\', '\\') + + if not cls.validateCharacters(tag, value): + raise Exception('Tag/Value contains invalid characters: %s/%s' % (tag, value)) + + tags[tag] = value rawtags = rawtags[len(m.group(0)):] tags['name'] = metric @@ -53,6 +75,9 @@ def parse_carbon(cls, path): if len(tag) != 2 or not tag[0]: raise Exception('Cannot parse path %s, invalid segment %s' % (path, segment)) + if not cls.validateCharacters(*tag): + raise Exception('Tag/Value contains invalid characters: %s/%s' % (tag[0], tag[1])) + tags[tag[0]] = tag[1] tags['name'] = metric diff --git a/webapp/graphite/templates/dashboard.html b/webapp/graphite/templates/dashboard.html index 688a3fde2..8a9a15b36 100644 --- a/webapp/graphite/templates/dashboard.html +++ b/webapp/graphite/templates/dashboard.html @@ -19,14 +19,12 @@ var REMOVE_ICON = '{% static "js/ext/resources/icons/fam/cross.gif" %}'; var REFRESH_ICON = '{% static "js/ext/resources/icons/fam/table_refresh.png" %}'; var SHARE_ICON = '{% static "js/ext/resources/icons/fam/application_go.png" %}'; + var HELP_ICON = '{% static "js/ext/resources/icons/fam/information.png" %}'; var CALENDAR_ICON = '{% static "js/ext/resources/images/default/shared/calendar.gif" %}'; var CLOCK_ICON = '{% static "img/clock_16.png" %}'; - var HELP_ICON = '{% static "js/ext/resources/icons/fam/information.png" %}'; - - // Prefetch images for use in Target grid - (new Image()).src = '{% static "img/move_up.png" %}'; - (new Image()).src = '{% static "img/move_down.png" %}'; - (new Image()).src = '{% static "img/trash.png" %}'; + var UP_ICON = '{% static "img/move_up.png" %}'; + var DOWN_ICON = '{% static "img/move_down.png" %}'; + var TRASH_ICON = '{% static "img/trash.png" %}'; {% if initialState %} var initialState = JSON.parse('{{ initialState|escapejs }}'); diff --git a/webapp/graphite/util.py b/webapp/graphite/util.py index 3adaa6edd..d0651daf4 100644 --- a/webapp/graphite/util.py +++ b/webapp/graphite/util.py @@ -280,7 +280,7 @@ def wrapped_f(*args, **kwargs): return wrapped_f -class BufferedHTTPReader(io.IOBase): +class BufferedHTTPReader(io.FileIO): def __init__(self, response, buffer_size=1048576): self.response = response self.buffer_size = buffer_size @@ -366,3 +366,28 @@ def _jsonResponse(data, queryParams, status=200, encoder=None, default=None): def _jsonError(message, queryParams, status=500, encoder=None, default=None): return _jsonResponse( {'error': message}, queryParams, status=status, encoder=encoder, default=default) + + +def parseHost(host_string): + s = host_string.strip() + bidx = s.rfind(']:') # find closing bracket and following colon. + cidx = s.find(':') + if s.startswith('[') and bidx is not None: + server = s[1:bidx] + port = s[bidx + 2:] + elif cidx is not None: + server = s[:cidx] + port = s[cidx + 1:] + else: + raise ValueError("Invalid host string \"%s\"" % host_string) + + if ':' in port: + port, _, instance = port.partition(':') + else: + instance = None + + return server, int(port), instance + + +def parseHosts(host_strings): + return [parseHost(host_string) for host_string in host_strings] diff --git a/webapp/tests/test_composer.py b/webapp/tests/test_composer.py deleted file mode 100644 index f7dcc270d..000000000 --- a/webapp/tests/test_composer.py +++ /dev/null @@ -1,34 +0,0 @@ -import mock - -from urllib3.response import HTTPResponse -from graphite.util import BytesIO - -from .base import TestCase -try: - from django.urls import reverse -except ImportError: # Django < 1.10 - from django.core.urlresolvers import reverse - - -class ComposerTest(TestCase): - @mock.patch('six.moves.http_client.HTTPConnection.request') - @mock.patch('six.moves.http_client.HTTPConnection.getresponse') - @mock.patch('graphite.composer.views.SMTP') - @mock.patch('django.conf.settings.SMTP_SERVER', 'localhost') - def test_send_email(self, mock_smtp, http_response, http_request): - url = reverse('composer_send_email') - request = { "to": "noreply@localhost", - "url": 'https://localhost:8000/render?target=sumSeries(a.b.c.d)&title=Test&width=500&from=-55minutes&until=now&height=400'} - - response = self.client.get(reverse('render'), {'target': 'test'}) - self.assertEqual(response['Content-Type'], 'image/png') - data = response.content - responseObject = HTTPResponse(body=BytesIO(data), status=200, preload_content=False) - http_request.return_value = responseObject - http_response.return_value = responseObject - - instance = mock_smtp.return_value - instance.sendmail.return_value = {} - - response = self.client.get(url, request) - self.assertEqual(response.content, b'OK') diff --git a/webapp/tests/test_functions.py b/webapp/tests/test_functions.py index df6ec4441..e461cc8fd 100644 --- a/webapp/tests/test_functions.py +++ b/webapp/tests/test_functions.py @@ -1310,7 +1310,8 @@ def test_divideSeries_error(self): ] ) - with self.assertRaisesRegexp(ValueError, "divideSeries second argument must reference exactly 1 series \(got 2\)"): + message = r"divideSeries second argument must reference exactly 1 series \(got 2\)" + with self.assertRaisesRegexp(ValueError, message): functions.divideSeries({}, seriesList, seriesList2) def test_divideSeries_seriesList2_single(self): @@ -2609,7 +2610,8 @@ def test_vertical_line_before_start(self): endTime=datetime(1971,1,1,1,2,0,0,pytz.timezone(settings.TIME_ZONE)), tzinfo=pytz.utc ) - with self.assertRaisesRegexp(ValueError, "verticalLine\(\): timestamp 3600 exists before start of range"): + message = r"verticalLine\(\): timestamp 3600 exists before start of range" + with self.assertRaisesRegexp(ValueError, message): result = functions.verticalLine(requestContext, "01:0019700101", "foo") def test_vertical_line_after_end(self): @@ -2618,7 +2620,8 @@ def test_vertical_line_after_end(self): endTime=datetime(1970,1,1,1,2,0,0,pytz.timezone(settings.TIME_ZONE)), tzinfo=pytz.utc ) - with self.assertRaisesRegexp(ValueError, "verticalLine\(\): timestamp 31539600 exists after end of range"): + message = r"verticalLine\(\): timestamp 31539600 exists after end of range" + with self.assertRaisesRegexp(ValueError, message): result = functions.verticalLine(requestContext, "01:0019710101", "foo") def test_line_width(self): @@ -2821,7 +2824,7 @@ def test_alias(self): def test_alias_sub(self): seriesList = self._generate_series_list() substitution = "Shrubbery" - results = functions.aliasSub({}, seriesList, "^\w+", substitution) + results = functions.aliasSub({}, seriesList, r"^\w+", substitution) for series in results: self.assertTrue(series.name.startswith(substitution), "aliasSub should replace the name with {0}".format(substitution), @@ -2865,10 +2868,10 @@ def mock_evaluateTarget(requestContext, target): with patch('graphite.render.functions.evaluateTarget', mock_evaluateTarget): # Perform query - this one will not find a matching metric with self.assertRaises(Exception): - functions.aliasQuery({}, seriesList, 'chan\.pow\.([0-9]+)', 'chan.fred.\\1', 'Channel %d MHz') + functions.aliasQuery({}, seriesList, r'chan\.pow\.([0-9]+)', 'chan.fred.\\1', 'Channel %d MHz') # Perform query - this one will find a matching metric - results = functions.aliasQuery({}, seriesList, 'chan\.pow\.([0-9]+)', 'chan.freq.\\1', 'Channel %d MHz') + results = functions.aliasQuery({}, seriesList, r'chan\.pow\.([0-9]+)', 'chan.freq.\\1', 'Channel %d MHz') # Check results self.assertEqual(results, expectedResult) @@ -2880,7 +2883,7 @@ def noneSafeLast(x): # Perform query - this one will fail to return a current value for the matched metric with self.assertRaises(Exception): - functions.aliasQuery({}, seriesList, 'chan\.pow\.([0-9]+)', 'chan.freq.\\1', 'Channel %d MHz') + functions.aliasQuery({}, seriesList, r'chan\.pow\.([0-9]+)', 'chan.freq.\\1', 'Channel %d MHz') # TODO: Add tests for * globbing and {} matching to this def test_alias_by_node(self): @@ -3786,6 +3789,60 @@ def test_reduceSeries_asPercent(self): results = functions.reduceSeries({}, copy.deepcopy(mappedResult), "asPercent", 2, "bytes_used", "total_bytes") self.assertEqual(results,expectedResult) + def test_add(self): + seriesList = self._generate_series_list() + # Leave the original seriesList undisturbed for verification + results = functions.add({}, copy.deepcopy(seriesList), 1.23) + for i, series in enumerate(results): + for counter, value in enumerate(series): + if value is None: + continue + original_value = seriesList[i][counter] + expected_value = original_value + 1.23 + self.assertEqual(value, expected_value) + + def test_sigmoid(self): + seriesList = self._generate_series_list() + # Leave the original seriesList undisturbed for verification + results = functions.sigmoid({}, copy.deepcopy(seriesList)) + for i, series in enumerate(results): + for counter, value in enumerate(series): + if value is None: + continue + original_value = seriesList[i][counter] + try: + expected_value = 1 / (1 + math.exp(-original_value)) + except (TypeError, ValueError, ZeroDivisionError): + expected_value = None + self.assertEqual(value, expected_value) + + def test_logit(self): + seriesList = self._generate_series_list() + # Leave the original seriesList undisturbed for verification + results = functions.logit({}, copy.deepcopy(seriesList)) + for i, series in enumerate(results): + for counter, value in enumerate(series): + if value is None: + continue + original_value = seriesList[i][counter] + try: + expected_value = math.log(original_value / (1 - original_value)) + except (TypeError, ValueError, ZeroDivisionError): + expected_value = None + self.assertEqual(value, expected_value) + + def test_exp(self): + seriesList = self._generate_series_list() + # Leave the original seriesList undisturbed for verification + results = functions.exp({}, copy.deepcopy(seriesList)) + for i, series in enumerate(results): + for counter, value in enumerate(series): + if value is None: + continue + original_value = seriesList[i][counter] + expected_value = math.exp(original_value) + self.assertEqual(value, expected_value) + def test_pow(self): seriesList = self._generate_series_list() factor = 2 diff --git a/webapp/tests/test_params.py b/webapp/tests/test_params.py new file mode 100644 index 000000000..a4795085d --- /dev/null +++ b/webapp/tests/test_params.py @@ -0,0 +1,194 @@ +import unittest + +from graphite.errors import InputParameterError +from graphite.functions.params import Param, ParamTypes, validateParams + + +class TestParam(unittest.TestCase): + params = [ + Param('one', ParamTypes.string, required=True), + Param('two', ParamTypes.string, required=True), + Param('three', ParamTypes.string, required=True), + ] + + def test_simple_args(self): + self.assertTrue(validateParams( + 'TestParam', + self.params, + ['arg1', 'arg2', 'arg3'], + {}, + )) + + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', + self.params, + ['arg1', 'arg2'], + {}, + ) + + def test_simple_kwargs(self): + self.assertTrue(validateParams( + 'TestParam', + self.params, + [], + {'one': '1', 'two': '2', 'three': '3'}, + )) + + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', + self.params, + [], + {'one': '1', 'two': '2'}, + ) + + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', + self.params, + [], + {'one': '1', 'two': '2', 'four': '4'}, + ) + + def test_mixed_cases(self): + self.assertTrue(validateParams( + 'TestParam', + self.params, + ['one', 'two'], + {'three': '3'}, + )) + + self.assertTrue(validateParams( + 'TestParam', + self.params, + ['one'], + {'three': '3', 'two': '2'}, + )) + + # positional args don't check the name + self.assertTrue(validateParams( + 'TestParam', + self.params, + ['one', 'two', 'four'], + {}, + )) + + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', + self.params, + [], + {'three': '3', 'two': '2'}, + ) + + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', + self.params, + ['one', 'three'], + {'two': '2'}, + ) + + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', + self.params, + ['three'], + {'one': '1', 'two': '2'}, + ) + + def test_repeated_args(self): + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', + self.params, + ['one'], + {'three': '3', 'one': '1'}, + ) + + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', + self.params, + ['one', 'two'], + {'three': '3', 'two': '2'}, + ) + + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', + self.params, + ['one', 'two', 'three'], + {'one': '1'}, + ) + + def test_multiple_property(self): + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', + [ + Param('one', ParamTypes.string, required=True), + Param('two', ParamTypes.string, required=True), + Param('three', ParamTypes.string, required=True, multiple=False), + ], + ['one', 'two', 'three', 'four'], + {}, + ) + + self.assertTrue(validateParams( + 'TestParam', + [ + Param('one', ParamTypes.string, required=True), + Param('two', ParamTypes.string, required=True), + Param('three', ParamTypes.string, required=True, multiple=True), + ], + ['one', 'two', 'three', 'four'], + {}, + )) + + def test_options_property(self): + self.assertTrue(validateParams( + 'TestParam', + [ + Param('one', ParamTypes.string, required=True), + Param('two', ParamTypes.string, required=True), + Param('three', ParamTypes.string, required=True, options=['3', 'three']), + ], + ['one', 'two', '3'], + {}, + )) + + self.assertTrue(validateParams( + 'TestParam', + [ + Param('one', ParamTypes.string, required=True), + Param('two', ParamTypes.string, required=True), + Param('three', ParamTypes.string, required=True, options=['3', 'three']), + ], + ['one', 'two', 'three'], + {}, + )) + + self.assertRaises( + InputParameterError, + validateParams, + 'TestParam', + [ + Param('one', ParamTypes.string, required=True), + Param('two', ParamTypes.string, required=True), + Param('three', ParamTypes.string, required=True, options=['3', 'three']), + ], + ['one', 'two', 'foud'], + {}, + ) diff --git a/webapp/tests/test_readers_remote.py b/webapp/tests/test_readers_remote.py index 17a5589aa..f12132b45 100644 --- a/webapp/tests/test_readers_remote.py +++ b/webapp/tests/test_readers_remote.py @@ -28,7 +28,7 @@ def test_RemoteReader_init_repr_get_intervals(self): bulk_query=['a.b.c.d']) self.assertIsNotNone(reader) - self.assertRegexpMatches(str(reader), "") + self.assertRegexpMatches(str(reader), r"") self.assertEqual(reader.get_intervals(), []) # @@ -163,7 +163,7 @@ def test_RemoteReader_fetch_multi(self, http_request): ) http_request.return_value = responseObject - with self.assertRaisesRegexp(Exception, 'Invalid render response from http://[^ ]+: KeyError\(\'name\',?\)'): + with self.assertRaisesRegexp(Exception, r'Invalid render response from http://[^ ]+: KeyError\(\'name\',?\)'): reader.fetch(startTime, endTime) # non-200 response diff --git a/webapp/tests/test_render.py b/webapp/tests/test_render.py index 3b69d29a4..bc517e95a 100644 --- a/webapp/tests/test_render.py +++ b/webapp/tests/test_render.py @@ -129,7 +129,8 @@ def test_render_evaluateTokens_template(self): 'template(target.$test, test=foo.bar)', ] - with self.assertRaisesRegexp(ValueError, 'invalid template\(\) syntax, only string/numeric arguments are allowed'): + message = r'invalid template\(\) syntax, only string/numeric arguments are allowed' + with self.assertRaisesRegexp(ValueError, message): evaluateTarget({}, test_input) @patch('graphite.render.evaluator.prefetchData', lambda *_: None) diff --git a/webapp/tests/test_render_datalib.py b/webapp/tests/test_render_datalib.py index df1767401..950b7db49 100644 --- a/webapp/tests/test_render_datalib.py +++ b/webapp/tests/test_render_datalib.py @@ -16,9 +16,9 @@ class TimeSeriesTest(TestCase): def test_TimeSeries_init_no_args(self): if sys.version_info[0] >= 3: - msg = '__init__\(\) missing 5 required positional arguments' + msg = r'__init__\(\) missing 5 required positional arguments' else: - msg = '__init__\(\) takes at least 6 arguments \(1 given\)' + msg = r'__init__\(\) takes at least 6 arguments \(1 given\)' with self.assertRaisesRegexp(TypeError, msg): TimeSeries() diff --git a/webapp/tests/test_storage.py b/webapp/tests/test_storage.py index 610674582..5be951abd 100644 --- a/webapp/tests/test_storage.py +++ b/webapp/tests/test_storage.py @@ -70,7 +70,7 @@ def test_fetch_pool_timeout(self): def mock_pool_exec(pool, jobs, timeout): raise PoolTimeoutError() - message = 'Timed out after [-.e0-9]+s for fetch for \[\'a\'\]' + message = r'Timed out after [-.e0-9]+s for fetch for \[\'a\'\]' with patch('graphite.storage.pool_exec', mock_pool_exec): with patch('graphite.storage.log.info') as log_info: with self.assertRaisesRegexp(Exception, message): @@ -84,21 +84,29 @@ def test_fetch_all_failed(self): finders=[TestFinder()] ) + message = r'All requests failed for fetch for \[\'a\'\] \(1\)' with patch('graphite.storage.log.info') as log_info: - with self.assertRaisesRegexp(Exception, 'All requests failed for fetch for \[\'a\'\] \(1\)'): + with self.assertRaisesRegexp(Exception, message): list(store.fetch(['a'], 1, 2, 3, {})) self.assertEqual(log_info.call_count, 1) - self.assertRegexpMatches(log_info.call_args[0][0], 'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes') + self.assertRegexpMatches( + log_info.call_args[0][0], + r'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes' + ) store = Store( finders=[TestFinder(), TestFinder()] ) + message = r'All requests failed for fetch for \[\'a\'\] \(2\)' with patch('graphite.storage.log.info') as log_info: - with self.assertRaisesRegexp(Exception, 'All requests failed for fetch for \[\'a\'\] \(2\)'): + with self.assertRaisesRegexp(Exception, message): list(store.fetch(['a'], 1, 2, 3, {})) self.assertEqual(log_info.call_count, 2) - self.assertRegexpMatches(log_info.call_args[0][0], 'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes') + self.assertRegexpMatches( + log_info.call_args[0][0], + r'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes' + ) def test_fetch_some_failed(self): # some finders failed @@ -114,11 +122,15 @@ def test_fetch_some_failed(self): finders=[TestFinder(), TestFinder()] ) + message = r'All requests failed for fetch for \[\'a\'\] \(2\)' with patch('graphite.storage.log.info') as log_info: - with self.assertRaisesRegexp(Exception, 'All requests failed for fetch for \[\'a\'\] \(2\)'): + with self.assertRaisesRegexp(Exception, message): list(store.fetch(['a'], 1, 2, 3, {})) self.assertEqual(log_info.call_count, 2) - self.assertRegexpMatches(log_info.call_args[0][0], 'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes') + self.assertRegexpMatches( + log_info.call_args[0][0], + r'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes' + ) @override_settings(STORE_FAIL_ON_ERROR=True) def test_fetch_some_failed_hard_fail_enabled(self): @@ -127,21 +139,29 @@ def test_fetch_some_failed_hard_fail_enabled(self): finders=[TestFinder(), RemoteFinder()] ) + message = r'1 request\(s\) failed for fetch for \[\'a\'\] \(2\)' with patch('graphite.storage.log.info') as log_info: - with self.assertRaisesRegexp(Exception, '1 request\(s\) failed for fetch for \[\'a\'\] \(2\)'): + with self.assertRaisesRegexp(Exception, message): list(store.fetch(['a'], 1, 2, 3, {})) self.assertEqual(log_info.call_count, 1) - self.assertRegexpMatches(log_info.call_args[0][0], 'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes') + self.assertRegexpMatches( + log_info.call_args[0][0], + r'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes' + ) store = Store( finders=[TestFinder(), TestFinder()] ) + message = r'All requests failed for fetch for \[\'a\'\] \(2\)' with patch('graphite.storage.log.info') as log_info: - with self.assertRaisesRegexp(Exception, 'All requests failed for fetch for \[\'a\'\] \(2\)'): + with self.assertRaisesRegexp(Exception, message): list(store.fetch(['a'], 1, 2, 3, {})) self.assertEqual(log_info.call_count, 2) - self.assertRegexpMatches(log_info.call_args[0][0], 'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes') + self.assertRegexpMatches( + log_info.call_args[0][0], + r'Exception during fetch for \[\'a\'\] after [-.e0-9]+s: TestFinder.find_nodes' + ) def test_find(self): disabled_finder = DisabledFinder() @@ -174,8 +194,9 @@ def test_find(self): self.assertTrue(node.path in ['a.b.c.d', 'a.b.c.e']) # failure threshold + message = r'Query a yields too many results and failed \(failure threshold is 1\)' with self.settings(METRICS_FIND_FAILURE_THRESHOLD=1): - with self.assertRaisesRegexp(Exception, 'Query a yields too many results and failed \(failure threshold is 1\)'): + with self.assertRaisesRegexp(Exception, message): list(store.find('a')) # warning threshold @@ -197,7 +218,7 @@ def test_find_pool_timeout(self): def mock_pool_exec(pool, jobs, timeout): raise PoolTimeoutError() - message = 'Timed out after [-.e0-9]+s for find ' + message = r'Timed out after [-.e0-9]+s for find ' with patch('graphite.storage.pool_exec', mock_pool_exec): with patch('graphite.storage.log.info') as log_info: with self.assertRaisesRegexp(Exception, message): @@ -211,14 +232,14 @@ def test_find_all_failed(self): finders=[TestFinder()] ) - message = 'All requests failed for find ' + message = r'All requests failed for find ' with patch('graphite.storage.log.info') as log_info: with self.assertRaisesRegexp(Exception, message): list(store.find('a')) self.assertEqual(log_info.call_count, 1) self.assertRegexpMatches( log_info.call_args[0][0], - 'Exception during find after [-.e0-9]+s: TestFinder.find_nodes' + r'Exception during find after [-.e0-9]+s: TestFinder.find_nodes' ) store = Store( @@ -231,7 +252,7 @@ def test_find_all_failed(self): self.assertEqual(log_info.call_count, 2) self.assertRegexpMatches( log_info.call_args[0][0], - 'Exception during find after [-.e0-9]+s: TestFinder.find_nodes' + r'Exception during find after [-.e0-9]+s: TestFinder.find_nodes' ) @override_settings(REMOTE_STORE_FORWARD_HEADERS=['X-Test1', 'X-Test2']) @@ -290,17 +311,23 @@ def test_get_index_all_failed(self): with self.assertRaisesRegexp(Exception, 'All requests failed for get_index'): store.get_index() self.assertEqual(log_info.call_count, 1) - self.assertRegexpMatches(log_info.call_args[0][0], 'Exception during get_index after [-.e0-9]+s: TestFinder.find_nodes') + self.assertRegexpMatches( + log_info.call_args[0][0], + 'Exception during get_index after [-.e0-9]+s: TestFinder.find_nodes' + ) store = Store( finders=[TestFinder(), TestFinder()] ) with patch('graphite.storage.log.info') as log_info: - with self.assertRaisesRegexp(Exception, 'All requests failed for get_index \(2\)'): + with self.assertRaisesRegexp(Exception, r'All requests failed for get_index \(2\)'): store.get_index() self.assertEqual(log_info.call_count, 2) - self.assertRegexpMatches(log_info.call_args[0][0], 'Exception during get_index after [-.e0-9]+s: TestFinder.find_nodes') + self.assertRegexpMatches( + log_info.call_args[0][0], + 'Exception during get_index after [-.e0-9]+s: TestFinder.find_nodes' + ) @override_settings(USE_WORKER_POOL=False) def test_fetch_tag_support(self): @@ -311,7 +338,11 @@ def find_nodes(self, query): pass def fetch(self, patterns, start_time, end_time, now=None, requestContext=None): - if patterns != ['seriesByTag("hello=tiger")', 'seriesByTag("name=notags")', 'seriesByTag("name=testtags")', 'testtags;hello=tiger']: + if patterns != [ + 'seriesByTag("hello=tiger")', + 'seriesByTag("name=notags")', + 'seriesByTag("name=testtags")', + 'testtags;hello=tiger']: raise Exception('Unexpected patterns %s' % str(patterns)) return [ @@ -546,29 +577,29 @@ def mockAutoCompleteValues(exprs, tag, valuePrefix=None, limit=None, requestCont # test exception handling with one finder store = mockStore([TestFinderTagsException()]) - with self.assertRaisesRegexp(Exception, 'All requests failed for tags for \[\'tag1=value1\'\] test.*'): + with self.assertRaisesRegexp(Exception, r'All requests failed for tags for \[\'tag1=value1\'\] test.*'): store.tagdb_auto_complete_tags(['tag1=value1'], 'test', 100, request_context) - with self.assertRaisesRegexp(Exception, 'All requests failed for values for \[\'tag1=value1\'\] tag2 test.*'): + with self.assertRaisesRegexp(Exception, r'All requests failed for values for \[\'tag1=value1\'\] tag2 test.*'): store.tagdb_auto_complete_values(['tag1=value1'], 'tag2', 'test', 100, request_context) # test exception handling with more than one finder store = mockStore([TestFinderTagsException(), TestFinderTagsException()]) - with self.assertRaisesRegexp(Exception, 'All requests failed for tags for \[\'tag1=value1\'\] test'): + with self.assertRaisesRegexp(Exception, r'All requests failed for tags for \[\'tag1=value1\'\] test'): store.tagdb_auto_complete_tags(['tag1=value1'], 'test', 100, request_context) - with self.assertRaisesRegexp(Exception, 'All requests failed for values for \[\'tag1=value1\'\] tag2 test'): + with self.assertRaisesRegexp(Exception, r'All requests failed for values for \[\'tag1=value1\'\] tag2 test'): store.tagdb_auto_complete_values(['tag1=value1'], 'tag2', 'test', 100, request_context) # test pool timeout handling store = mockStore([TestFinderTagsTimeout()]) with self.settings(USE_WORKER_POOL=True, FIND_TIMEOUT=0): - with self.assertRaisesRegexp(Exception, 'Timed out after [-.e0-9]+s for tags for \[\'tag1=value1\'\]'): + with self.assertRaisesRegexp(Exception, r'Timed out after [-.e0-9]+s for tags for \[\'tag1=value1\'\]'): store.tagdb_auto_complete_tags(['tag1=value1'], 'test', 100, request_context) - with self.assertRaisesRegexp(Exception, 'Timed out after [-.e0-9]+s for values for \[\'tag1=value1\'\] tag2 test'): + with self.assertRaisesRegexp(Exception, r'Timed out after [-.e0-9]+s for values for \[\'tag1=value1\'\] tag2 test'): store.tagdb_auto_complete_values(['tag1=value1'], 'tag2', 'test', 100, request_context) # test write_index diff --git a/webapp/tests/test_tags.py b/webapp/tests/test_tags.py index 2b452db65..831a6682a 100644 --- a/webapp/tests/test_tags.py +++ b/webapp/tests/test_tags.py @@ -294,7 +294,7 @@ def test_tagdb_cached(self): self.assertEqual(mockLog.info.call_count, 1) self.assertRegexpMatches( mockLog.info.call_args[0][0], - 'graphite\.tags\.localdatabase\.LocalDatabaseTagDB\.find_series :: completed \(cached\) in [-.e0-9]+s' + r'graphite\.tags\.localdatabase\.LocalDatabaseTagDB\.find_series :: completed \(cached\) in [-.e0-9]+s' ) def test_http_tagdb(self): diff --git a/webapp/tests/test_util.py b/webapp/tests/test_util.py index ca0b08947..a916084c8 100644 --- a/webapp/tests/test_util.py +++ b/webapp/tests/test_util.py @@ -28,14 +28,20 @@ def test_epoch_naive(self, mock_log): self.assertEqual(util.epoch(dt), 600) self.assertEqual(mock_log.call_count, 1) self.assertEqual(len(mock_log.call_args[0]), 1) - self.assertRegexpMatches(mock_log.call_args[0][0], 'epoch\(\) called with non-timezone-aware datetime in test_epoch_naive at .+/webapp/tests/test_util\.py:[0-9]+') + self.assertRegexpMatches( + mock_log.call_args[0][0], + r'epoch\(\) called with non-timezone-aware datetime in test_epoch_naive at .+/webapp/tests/test_util\.py:[0-9]+' + ) with self.settings(TIME_ZONE='Europe/Berlin'): dt = datetime(1970, 1, 1, 1, 10, 0, 0) self.assertEqual(util.epoch(dt), 600) self.assertEqual(mock_log.call_count, 2) self.assertEqual(len(mock_log.call_args[0]), 1) - self.assertRegexpMatches(mock_log.call_args[0][0], 'epoch\(\) called with non-timezone-aware datetime in test_epoch_naive at .+/webapp/tests/test_util\.py:[0-9]+') + self.assertRegexpMatches( + mock_log.call_args[0][0], + r'epoch\(\) called with non-timezone-aware datetime in test_epoch_naive at .+/webapp/tests/test_util\.py:[0-9]+' + ) def test_epoch_to_dt(self): dt = pytz.utc.localize(datetime(1970, 1, 1, 0, 10, 0, 0)) @@ -43,8 +49,8 @@ def test_epoch_to_dt(self): def test_is_local_interface_ipv4(self): addresses = ['127.0.0.1', '127.0.0.1:8080', '8.8.8.8'] - results = [ util.is_local_interface(a) for a in addresses ] - self.assertEqual( results, [True, True, False] ) + results = [util.is_local_interface(a) for a in addresses] + self.assertEqual(results, [True, True, False]) def test_is_local_interface_ipv6(self): # we need to know whether the host provides an ipv6 callback address @@ -69,20 +75,20 @@ def test_is_local_interface_dns(self): self.assertEqual( results, [True, True, False] ) def test_is_escaped_pattern(self): - self.assertFalse( util.is_escaped_pattern('asdf') ) - self.assertTrue( util.is_escaped_pattern('a\*b') ) - self.assertTrue( util.is_escaped_pattern('a\?b') ) - self.assertTrue( util.is_escaped_pattern('a\[b') ) - self.assertTrue( util.is_escaped_pattern('a\{b') ) - self.assertFalse( util.is_escaped_pattern('a*b') ) - self.assertFalse( util.is_escaped_pattern('a?b') ) - self.assertFalse( util.is_escaped_pattern('a[b') ) - self.assertFalse( util.is_escaped_pattern('a{b') ) + self.assertFalse(util.is_escaped_pattern(r'asdf')) + self.assertTrue(util.is_escaped_pattern(r'a\*b')) + self.assertTrue(util.is_escaped_pattern(r'a\?b')) + self.assertTrue(util.is_escaped_pattern(r'a\[b')) + self.assertTrue(util.is_escaped_pattern(r'a\{b')) + self.assertFalse(util.is_escaped_pattern(r'a*b')) + self.assertFalse(util.is_escaped_pattern(r'a?b')) + self.assertFalse(util.is_escaped_pattern(r'a[b')) + self.assertFalse(util.is_escaped_pattern(r'a{b')) def test_find_escaped_pattern_fields(self): - self.assertEqual( list(util.find_escaped_pattern_fields('a.b.c.d')), []) - self.assertEqual( list(util.find_escaped_pattern_fields('a\*.b.c.d')), [0]) - self.assertEqual( list(util.find_escaped_pattern_fields('a.b.\[c].d')), [2]) + self.assertEqual(list(util.find_escaped_pattern_fields(r'a.b.c.d')), []) + self.assertEqual(list(util.find_escaped_pattern_fields(r'a\*.b.c.d')), [0]) + self.assertEqual(list(util.find_escaped_pattern_fields(r'a.b.\[c].d')), [2]) hostcpu = os.path.join(settings.WHISPER_DIR, 'hosts/hostname/cpu.wsp')