Merge pull request #2399 from deniszh/backport/1.1.x/pr-2370_pr-2371_…

…pr-2372_pr-2372_pr-2383_pr-2380_pr-2380_pr-2380_pr-2380_pr-2380_pr-2381_pr-2383_pr-2383_pr-2383_pr-2375_pr-2174

[1.1.x] Fix carbonlink queries for tagged series. | flake8 cleanup | With PR #2295 the top nav dashboard fails to render the default tree | Also have to override the input box value of '' with '*' because que

.travis.yml

@@ -49,7 +49,6 @@ matrix:
 
 env:
   - TOXENV=py27-django18-pyparsing2
-  - TOXENV=py27-django111-pyparsing2-rrdtool
   - TOXENV=py27-django111-pyparsing2-msgpack
   - TOXENV=py27-django111-pyparsing2-pyhash
   - TOXENV=py27-django111-pyparsing2-mysql TEST_MYSQL_PASSWORD=graphite

docs/config-carbon.rst

@@ -30,7 +30,7 @@ Important notes before continuing:
 
 * There can be many sections in this file.
 * The sections are applied in order from the top (first) and bottom (last).
-* The patterns are regular expressions, as opposed to the wildcards used in the URL API.
+* The patterns are `regular expressions <https://docs.python.org/3/library/re.html#regular-expression-syntax>`_, as opposed to the wildcards used in the URL API.
 * The first pattern that matches the metric name is used.
 * This retention is set at the time the first metric is sent.
 * Changing this file will not affect already-created .wsp files. Use whisper-resize.py to change those.
@@ -41,7 +41,7 @@ A given rule is made up of 3 lines:
 * A regex, specified after "pattern="
 * A retention rate line, specified after "retentions="
 
-The retentions line can specify multiple retentions. Each retention of ``frequency:history`` is separated by a comma. 
+The retentions line can specify multiple retentions. Each retention of ``frequency:history`` is separated by a comma.
 
 Frequencies and histories are specified using the following suffixes:
 
@@ -61,9 +61,9 @@ Here's a simple, single retention example:
  pattern = garbageCollections$
  retentions = 10s:14d
 
-The name ``[garbage_collection]`` is mainly for documentation purposes, and will show up in ``creates.log`` when metrics matching this section are created. 
+The name ``[garbage_collection]`` is mainly for documentation purposes, and will show up in ``creates.log`` when metrics matching this section are created.
 
-The regular expression pattern will match any metric that ends with ``garbageCollections``. For example, ``com.acmeCorp.instance01.jvm.memory.garbageCollections`` would match, but ``com.acmeCorp.instance01.jvm.memory.garbageCollections.full`` would not.
+The regular expression ``pattern`` will match any metric that ends with ``garbageCollections``. For example, ``com.acmeCorp.instance01.jvm.memory.garbageCollections`` would match, but ``com.acmeCorp.instance01.jvm.memory.garbageCollections.full`` would not. Graphite is using the `Python Regular Expression Syntax <https://docs.python.org/3/library/re.html#regular-expression-syntax>`_, for an introduction to regular expressions consult the `Regular Expression HOWTO <https://docs.python.org/3/howto/regex.html#regex-howto>`_.
 
 The ``retentions`` line is saying that each datapoint represents 10 seconds, and we want to keep enough datapoints so that they add up to 14 days of data.
 

docs/render_api.rst

@@ -473,7 +473,7 @@ areaAlpha
 ---------
 *Default: 1.0*
 
-Takes a floating point number between 0.0 and 1.0 
+Takes a floating point number between 0.0 and 1.0
 
 Sets the alpha (transparency) value of filled areas when using an areaMode_
 
@@ -484,7 +484,7 @@ areaMode
 *Default: none*
 
 Enables filling of the area below the graphed lines. Fill area is the same color as
-the line color associated with it. See areaAlpha_ to make this area transparent. 
+the line color associated with it. See areaAlpha_ to make this area transparent.
 Takes one of the following parameters which determines the fill mode to use:
 
 ``none``
@@ -498,7 +498,7 @@ Takes one of the following parameters which determines the fill mode to use:
   Each target line is displayed as the sum of all previous lines plus the value of the current line.
 
 .. _param-bgcolor:
-  
+
 bgcolor
 -------
 *Default: value from the [default] template in graphTemplates.conf*
@@ -573,7 +573,7 @@ drawNullAsZero
 
 Converts any None (null) values in the displayed metrics to zero at render time.
 
-.. _param-fgcolor: 
+.. _param-fgcolor:
 
 fgcolor
 -------
@@ -1076,7 +1076,7 @@ Example:
   &title=Apache Busy Threads, All Servers, Past 24h
 
 .. _param-tz:
-  
+
 tz
 --
 *Default: The timezone specified in local_settings.py*
@@ -1202,23 +1202,23 @@ Sets the time format used when displaying the X-axis. See
 for format specification details.
 
 .. _param-yAxisSide:
-  
+
 yAxisSide
 ---------
 *Default: left*
 
 Sets the side of the graph on which to render the Y-axis. Accepts values of ``left`` or ``right``
 
 .. _param-yDivisors:
-  
+
 yDivisors
 ---------
 *Default: 4,5,6*
 
 Sets the preferred number of intermediate values to display on the Y-axis (Y values between the
-minimum and maximum).  Note that Graphite will ultimately choose what values (and how many) to 
-display based on a 'pretty' factor, which tries to maintain a sensible scale (e.g. preferring 
-intermediary values like 25%,50%,75% over 33.3%,66.6%).  To explicitly set the Y-axis values, 
+minimum and maximum).  Note that Graphite will ultimately choose what values (and how many) to
+display based on a 'pretty' factor, which tries to maintain a sensible scale (e.g. preferring
+intermediary values like 25%,50%,75% over 33.3%,66.6%).  To explicitly set the Y-axis values,
 see `yStep`_
 
 .. _param-yLimit:
@@ -1295,7 +1295,7 @@ yMinRight
 In dual Y-axis mode, sets the lower bound of the right Y-Axis (See: `yMin`_)
 
 .. _param-yStep:
-  
+
 yStep
 -----
 *Default: Calculated automatically*

setup.cfg

@@ -58,4 +58,6 @@ obsoletes = graphite <= 0.9.9
 # E731 do not assign a lambda expression, use a def
 # E114 indentation is not a multiple of four (comment)
 # E266 too many leading '#' for block comment
-ignore=E128,E111,E501,E231,E201,E202,E203,E251,E124,E265,E121,E261,E226,E262,E701,E241,E221,E703,E502,F841,E401,E222,E126,E122,E222,E225,E228,E127,E123,E713,E125,E227,E271,E272,N802,N803,N806,E731,E114,E266
+# W504 line break after binary operator
+# W605 invalid escape sequence
+ignore=E128,E111,E501,E231,E201,E202,E203,E251,E124,E265,E121,E261,E226,E262,E701,E241,E221,E703,E502,F841,E401,E222,E126,E122,E222,E225,E228,E127,E123,E713,E125,E227,E271,E272,N802,N803,N806,E731,E114,E266,W504,W605

webapp/content/js/dashboard.js

@@ -331,7 +331,7 @@ function initDashboard () {
         url: document.body.dataset.baseUrl + 'metrics/find/',
         autoLoad: true,
         baseParams: {
-          query: '',
+          query: '*',
           format: 'completer',
           automatic_variants: (UI_CONFIG.automatic_variants) ? '1' : '0'
         },
@@ -383,6 +383,9 @@ function initDashboard () {
   var autocompleteTask = new Ext.util.DelayedTask(function () {
     var query = metricSelectorTextField.getValue();
     var store = metricSelectorGrid.getStore();
+    if (query === '') {
+      query = '*'
+    }
     store.setBaseParam('query', query);
     store.load();
   });

webapp/graphite/finders/standard.py

@@ -69,13 +69,14 @@ def find_nodes(self, query):
                     datasource_pattern = None
 
                 absolute_path = join(root_dir, relative_path)
-                metric_path = fs_to_metric(relative_path)
-                real_metric_path = get_real_metric_path(absolute_path, metric_path)
 
                 # if we're finding by tag, return the proper metric path
                 if tagged:
                   metric_path = query.pattern
+                  real_metric_path = query.pattern
                 else:
+                  metric_path = fs_to_metric(relative_path)
+                  real_metric_path = get_real_metric_path(absolute_path, metric_path)
                   metric_path_parts = metric_path.split('.')
                   for field_index in find_escaped_pattern_fields(query.pattern):
                       metric_path_parts[field_index] = pattern_parts[field_index].replace('\\', '')

webapp/graphite/readers/rrd.py

@@ -1,7 +1,6 @@
 import sys
 import os
 import time
-import six
 
 # Use the built-in version of scandir/stat if possible, otherwise
 # use the scandir module version
@@ -25,8 +24,12 @@ class RRDReader(BaseReader):
 
     @staticmethod
     def _convert_fs_path(fs_path):
-        if isinstance(fs_path, six.text_type):
-            fs_path = fs_path.encode(sys.getfilesystemencoding())
+        # Only Python 2 'unicode' needs to be converted to str/bytes
+        try:
+            if isinstance(fs_path, unicode):
+                fs_path = fs_path.encode(sys.getfilesystemencoding())
+        except NameError:
+            pass
         return os.path.realpath(fs_path)
 
     def __init__(self, fs_path, datasource_name):

webapp/graphite/readers/utils.py

@@ -1,3 +1,5 @@
+from __future__ import division
+
 import abc
 
 from graphite.logger import log
@@ -40,10 +42,12 @@ def consolidate(func, values):
         usable = [v for v in values if v is not None]
         if not usable:
             return None
+        if func == 'avg_zero':
+            return sum([0 if v is None else v for v in values]) / len(values)
         if func == 'sum':
             return sum(usable)
         if func == 'average':
-            return float(sum(usable)) / len(usable)
+            return sum(usable) / len(usable)
         if func == 'max':
             return max(usable)
         if func == 'min':

webapp/graphite/render/datalib.py

@@ -76,6 +76,7 @@ def consolidate(self, valuesPerPoint):
   __consolidation_functions = {
     'sum': sum,
     'average': lambda usable: sum(usable) / len(usable),
+    'avg_zero': lambda usable: sum(usable) / len(usable),
     'max': max,
     'min': min,
     'first': lambda usable: usable[0],
@@ -88,24 +89,29 @@ def __consolidatingGenerator(self, gen):
     except KeyError:
       raise Exception("Invalid consolidation function: '%s'" % self.consolidationFunc)
 
-    buf = []  # only the not-None values
+    buf = []
     valcnt = 0
+    nonNull = 0
 
     for x in gen:
       valcnt += 1
       if x is not None:
         buf.append(x)
+        nonNull += 1
+      elif self.consolidationFunc == 'avg_zero':
+        buf.append(0)
 
       if valcnt == self.valuesPerPoint:
-        if buf and (len(buf) / self.valuesPerPoint) >= self.xFilesFactor:
+        if nonNull and (nonNull / self.valuesPerPoint) >= self.xFilesFactor:
           yield cf(buf)
         else:
           yield None
         buf = []
         valcnt = 0
+        nonNull = 0
 
     if valcnt > 0:
-      if buf and (len(buf) / self.valuesPerPoint) >= self.xFilesFactor:
+      if nonNull and (nonNull / self.valuesPerPoint) >= self.xFilesFactor:
         yield cf(buf)
       else:
         yield None

webapp/graphite/render/evaluator.py

@@ -64,7 +64,7 @@ def evaluateTokens(requestContext, tokens, replacements=None, pipedArg=None):
           val = replacements[name]
           if not isinstance(val, six.string_types):
             return val
-          elif re.match('^-?[\d.]+$', val):
+          elif re.match(r'^-?[\d.]+$', val):
             return float(val)
           else:
             return val