From 33f36deb308ba2d545f69a528342748ee353afe7 Mon Sep 17 00:00:00 2001 From: deniszh Date: Sun, 23 Feb 2020 22:49:15 +0100 Subject: [PATCH 1/4] Adding Hisser and Go-graphite buckytools in tools documentation --- docs/tools.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/tools.rst b/docs/tools.rst index e2a9921b4..4f292ff61 100644 --- a/docs/tools.rst +++ b/docs/tools.rst @@ -270,6 +270,9 @@ If you wish to use a backend to graphite other than Whisper, there are some opti `Cyanite`_ A highly available, elastic, and low-latency time-series storage wirtten on top of Cassandra +`hisser`_ + Time series database, backend for graphite, fast alternative to carbon + whisper. + `graphite-clickhouse`_ Graphite-web backend with `ClickHouse`_ support. Please also see `carbon-clickhouse`_. @@ -305,6 +308,9 @@ Other `carbonate`_ Utilities for managing graphite clusters. +`go-graphite buckytools`_ + Fork of `buckytools`_, with more features. + `graphite-dl4j`_ Experimental: Train a model for predictions. Machine learning using data from graphite. @@ -354,6 +360,7 @@ Other .. _Gdash: https://github.com/ripienaar/gdash.git .. _Giraffe: http://kenhub.github.com/giraffe .. _go-carbon: https://github.com/lomik/go-carbon +.. _go-graphite buckytools: https://github.com/go-graphite/buckytools .. _Grafana: http://grafana.org .. _Grafsy: https://github.com/leoleovich/grafsy .. _Graphene: http://jondot.github.com/graphene @@ -378,6 +385,7 @@ Other .. _Grockets: https://github.com/disqus/grockets .. _Gruffalo: https://github.com/outbrain/gruffalo .. _HoardD: https://github.com/coredump/hoardd +.. _hisser: https://github.com/baverman/hisser .. _Host sFlow: http://host-sflow.sourceforge.net .. _Hubot: https://github.com/github/hubot .. _hubot-scripts: https://github.com/github/hubot-scripts From 114097e86e5915bfb383c648f4f3c5b953906d49 Mon Sep 17 00:00:00 2001 From: Pierce Lopez Date: Fri, 28 Feb 2020 19:08:18 -0500 Subject: [PATCH 2/4] reformat flake8 ignore list --- setup.cfg | 126 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 43 deletions(-) diff --git a/setup.cfg b/setup.cfg index 2123d23f7..23b1944c4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,46 +18,86 @@ provides = graphite obsoletes = graphite <= 0.9.9 [flake8] -# E501 line too long -# E111 indentation is not a multiple of four -# E128 continuation line under-indented for visual indent -# E201 whitespace after '(' -# E202 whitespace before ')' -# E203 whitespace before ':' -# E231 missing whitespace after ',' -# E251 unexpected spaces around keyword / parameter equals -# E124 closing bracket does not match visual indentation -# E265 block comment should start with '# ' -# E121 continuation line under-indented for hanging indent -# E261 at least two spaces before inline comment -# E226 missing whitespace around arithmetic operator -# E262 inline comment should start with '# ' -# E701 multiple statements on one line (colon) -# E241 multiple spaces after ':' -# E221 multiple spaces before operator -# E703 statement ends with a semicolon -# E502 the backslash is redundant between brackets -# F841 local variable 'stuff' is assigned to but never used -# E401 multiple imports on one line -# E222 multiple spaces after operator -# E126 continuation line over-indented for hanging indent -# E122 continuation line missing indentation or outdented -# E222 multiple spaces after operator -# E225 missing whitespace around operator -# E228 missing whitespace around modulo operator -# E127 continuation line over-indented for visual indent -# E123 closing bracket does not match indentation of opening bracket's line -# E713 test for membership should be 'not in' -# E125 continuation line with same indent as next logical line -# E227 missing whitespace around bitwise or shift operator -# E271 multiple spaces after keyword -# E272 multiple spaces before keyword -# N802 function name should be lowercase -# N803 argument name should be lowercase -# N806 variable in function should be lowercase -# E731 do not assign a lambda expression, use a def -# E114 indentation is not a multiple of four (comment) -# E266 too many leading '#' for block comment -# W504 line break after binary operator -# W605 invalid escape sequence -ignore=E128,E111,E501,E231,E201,E202,E203,E251,E124,E265,E121,E261,E226,E262,E701,E241,E221,E703,E502,F841,E401,E222,E126,E122,E222,E225,E228,E127,E123,E713,E125,E227,E271,E272,N802,N803,N806,E731,E114,E266,W504,W605 +ignore = + # E111 indentation is not a multiple of four + E111, + # E114 indentation is not a multiple of four (comment) + E114, + # E121 continuation line under-indented for hanging indent + E121, + # E122 continuation line missing indentation or outdented + E122, + # E123 closing bracket does not match indentation of opening bracket's line + E123, + # E124 closing bracket does not match visual indentation + E124, + # E125 continuation line with same indent as next logical line + E125, + # E126 continuation line over-indented for hanging indent + E126, + # E127 continuation line over-indented for visual indent + E127, + # E128 continuation line under-indented for visual indent + E128, + # E201 whitespace after '(' + E201, + # E202 whitespace before ')' + E202, + # E203 whitespace before ':' + E203, + # E221 multiple spaces before operator + E221, + # E222 multiple spaces after operator + E222, + # E225 missing whitespace around operator + E225, + # E226 missing whitespace around arithmetic operator + E226, + # E227 missing whitespace around bitwise or shift operator + E227, + # E228 missing whitespace around modulo operator + E228, + # E231 missing whitespace after ',' + E231, + # E241 multiple spaces after ':' + E241, + # E251 unexpected spaces around keyword / parameter equals + E251, + # E261 at least two spaces before inline comment + E261, + # E262 inline comment should start with '# ' + E262, + # E265 block comment should start with '# ' + E265, + # E266 too many leading '#' for block comment + E266, + # E271 multiple spaces after keyword + E271, + # E272 multiple spaces before keyword + E272, + # E401 multiple imports on one line + E401, + # E501 line too long + E501, + # E502 the backslash is redundant between brackets + E502, + # W504 line break after binary operator + W504, + # W605 invalid escape sequence + W605, + # E701 multiple statements on one line (colon) + E701, + # E703 statement ends with a semicolon + E703, + # E713 test for membership should be 'not in' + E713, + # E731 do not assign a lambda expression, use a def + E731, + # N802 function name should be lowercase + N802, + # N803 argument name should be lowercase + N803, + # N806 variable in function should be lowercase + N806, + # F841 local variable 'stuff' is assigned to but never used + F841, From 2c816d3fea39a18ea913384bd8a1dc03ceac09b8 Mon Sep 17 00:00:00 2001 From: Pierce Lopez Date: Fri, 28 Feb 2020 19:32:15 -0500 Subject: [PATCH 3/4] flake8: re-enable some rules which need the least churn to fix see removed lines from setup.cfg for the specific rules re-enabled The N8xx ignored rules were for the "pep8-naming" plugin for flake8. It is not installed by default, it probably has other warnings that this codebase would trigger, and there are probably other flake8 plugins that would cause lots of warnings, so just remove. --- setup.cfg | 28 ------------------------- webapp/graphite/account/ldapBackend.py | 4 +++- webapp/graphite/dashboard/views.py | 4 ++-- webapp/graphite/logger.py | 3 ++- webapp/graphite/render/functions.py | 25 +++++++++++----------- webapp/graphite/render/glyph.py | 29 ++++++++++++++------------ webapp/graphite/settings.py | 2 +- webapp/tests/test_functions.py | 6 +++--- webapp/tests/test_readers_util.py | 2 +- 9 files changed, 40 insertions(+), 63 deletions(-) diff --git a/setup.cfg b/setup.cfg index 23b1944c4..bfd07e2b0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,16 +27,10 @@ ignore = E121, # E122 continuation line missing indentation or outdented E122, - # E123 closing bracket does not match indentation of opening bracket's line - E123, # E124 closing bracket does not match visual indentation E124, - # E125 continuation line with same indent as next logical line - E125, # E126 continuation line over-indented for hanging indent E126, - # E127 continuation line over-indented for visual indent - E127, # E128 continuation line under-indented for visual indent E128, # E201 whitespace after '(' @@ -53,10 +47,6 @@ ignore = E225, # E226 missing whitespace around arithmetic operator E226, - # E227 missing whitespace around bitwise or shift operator - E227, - # E228 missing whitespace around modulo operator - E228, # E231 missing whitespace after ',' E231, # E241 multiple spaces after ':' @@ -71,33 +61,15 @@ ignore = E265, # E266 too many leading '#' for block comment E266, - # E271 multiple spaces after keyword - E271, - # E272 multiple spaces before keyword - E272, - # E401 multiple imports on one line - E401, # E501 line too long E501, - # E502 the backslash is redundant between brackets - E502, # W504 line break after binary operator W504, - # W605 invalid escape sequence - W605, # E701 multiple statements on one line (colon) E701, - # E703 statement ends with a semicolon - E703, # E713 test for membership should be 'not in' E713, # E731 do not assign a lambda expression, use a def E731, - # N802 function name should be lowercase - N802, - # N803 argument name should be lowercase - N803, - # N806 variable in function should be lowercase - N806, # F841 local variable 'stuff' is assigned to but never used F841, diff --git a/webapp/graphite/account/ldapBackend.py b/webapp/graphite/account/ldapBackend.py index 68f1fd455..cc897ed3c 100644 --- a/webapp/graphite/account/ldapBackend.py +++ b/webapp/graphite/account/ldapBackend.py @@ -12,7 +12,9 @@ See the License for the specific language governing permissions and limitations under the License.""" -import ldap, traceback +import traceback + +import ldap from django.conf import settings from django.contrib.auth.models import User diff --git a/webapp/graphite/dashboard/views.py b/webapp/graphite/dashboard/views.py index 576c9a4c9..5e5d69183 100644 --- a/webapp/graphite/dashboard/views.py +++ b/webapp/graphite/dashboard/views.py @@ -233,7 +233,7 @@ def save(request, name): dashboard = Dashboard.objects.create(name=name, state=state) else: dashboard.state = state - dashboard.save(); + dashboard.save() return json_response( dict(success=True) ) @@ -252,7 +252,7 @@ def save_template(request, name, key): template.save() else: template.setState(state, key) - template.save(); + template.save() return json_response( dict(success=True) ) diff --git a/webapp/graphite/logger.py b/webapp/graphite/logger.py index 9cbc46992..4baafa442 100644 --- a/webapp/graphite/logger.py +++ b/webapp/graphite/logger.py @@ -12,7 +12,8 @@ See the License for the specific language governing permissions and limitations under the License.""" -import os, logging +import os +import logging from logging.handlers import TimedRotatingFileHandler as Rotater try: from logging import NullHandler diff --git a/webapp/graphite/render/functions.py b/webapp/graphite/render/functions.py index 0fcbbccec..6d32efc6b 100644 --- a/webapp/graphite/render/functions.py +++ b/webapp/graphite/render/functions.py @@ -72,7 +72,7 @@ def safeFunc(values): def gcd(a, b): if b == 0: return a - return gcd(b, a%b) + return gcd(b, a % b) # Least common multiple @@ -2246,7 +2246,7 @@ def stacked(requestContext,seriesLists,stackName='__DEFAULT__'): totalStack = requestContext['totalStack'].get(stackName, []) else: requestContext['totalStack'] = {} - totalStack = []; + totalStack = [] results = [] for series in seriesLists: newValues = [] @@ -2450,9 +2450,10 @@ def cactiStyle(requestContext, seriesList, system=None, units=None): fmt = lambda x:"%.2f%s" % format_units(x,system=system) else: if units: - fmt = lambda x:"%.2f %s"%(x,units) + fmt = lambda x: "%.2f %s" % (x, units) else: - fmt = lambda x:"%.2f"%x + fmt = lambda x: "%.2f" % x + nameLen = max([0] + [len(getattr(series,"name")) for series in seriesList]) lastLen = max([0] + [len(fmt(int(safe.safeLast(series) or 3))) for series in seriesList]) + 3 maxLen = max([0] + [len(fmt(int(safe.safeMax(series) or 3))) for series in seriesList]) + 3 @@ -2477,9 +2478,9 @@ def cactiStyle(requestContext, seriesList, system=None, units=None): series.name = "%*s Current:%*s Max:%*s Min:%*s " % \ (-nameLen, series.name, - -lastLen, last, - -maxLen, maximum, - -minLen, minimum) + -lastLen, last, + -maxLen, maximum, + -minLen, minimum) return seriesList @@ -3207,7 +3208,7 @@ def averageOutsidePercentile(requestContext, seriesList, n): averages = [safe.safeAvg(s) for s in seriesList] if n < 50: - n = 100 - n; + n = 100 - n lowPercentile = _getPercentile(averages, 100 - n) highPercentile = _getPercentile(averages, n) @@ -3642,9 +3643,7 @@ def stdev(requestContext, seriesList, points, windowTolerance=0.1): currentSum += newValue currentSumOfSquares += newValue**2 - if validPoints > 0 and \ - float(validPoints)/points >= windowTolerance: - + if validPoints > 0 and float(validPoints)/points >= windowTolerance: try: deviation = math.sqrt(validPoints * currentSumOfSquares - currentSum**2)/validPoints except ValueError: @@ -4013,12 +4012,12 @@ def linearRegression(requestContext, seriesList, startSourceAt=None, endSourceAt series.tags['linearRegressions'] = '%s, %s' % ( int(time.mktime(sourceContext['startTime'].timetuple())), int(time.mktime(sourceContext['endTime'].timetuple())) - ) + ) newName = 'linearRegression(%s, %s, %s)' % ( series.name, int(time.mktime(sourceContext['startTime'].timetuple())), int(time.mktime(sourceContext['endTime'].timetuple())) - ) + ) forecast = linearRegressionAnalysis(source) if forecast is None: continue diff --git a/webapp/graphite/render/glyph.py b/webapp/graphite/render/glyph.py index 4bd2f6a91..6f75b449c 100644 --- a/webapp/graphite/render/glyph.py +++ b/webapp/graphite/render/glyph.py @@ -12,8 +12,11 @@ See the License for the specific language governing permissions and limitations under the License.""" -import math, itertools, re +import itertools +import math +import re from datetime import datetime, timedelta + from six.moves import range, zip from six.moves.urllib.parse import unquote_plus from six.moves.configparser import SafeConfigParser @@ -517,8 +520,8 @@ def getLabelValues(self): class Graph: - customizable = ('width','height','margin','bgcolor','fgcolor', \ - 'fontName','fontSize','fontBold','fontItalic', \ + customizable = ('width','height','margin','bgcolor','fgcolor', + 'fontName','fontSize','fontBold','fontItalic', 'colorList','template','yAxisSide','outputFormat') def __init__(self,**params): @@ -916,16 +919,16 @@ def onHeaderPath(match): class LineGraph(Graph): customizable = Graph.customizable + \ - ('title','vtitle','lineMode','lineWidth','hideLegend', \ - 'hideAxes','minXStep','hideGrid','majorGridLineColor', \ - 'minorGridLineColor','thickness','min','max', \ - 'graphOnly','yMin','yMax','yLimit','yStep','areaMode', \ - 'areaAlpha','drawNullAsZero','tz', 'yAxisSide','pieMode', \ - 'yUnitSystem', 'logBase','yMinLeft','yMinRight','yMaxLeft', \ - 'yMaxRight', 'yLimitLeft', 'yLimitRight', 'yStepLeft', \ - 'yStepRight', 'rightWidth', 'rightColor', 'rightDashed', \ - 'leftWidth', 'leftColor', 'leftDashed', 'xFormat', 'minorY', \ - 'hideYAxis', 'uniqueLegend', 'vtitleRight', 'yDivisors', \ + ('title','vtitle','lineMode','lineWidth','hideLegend', + 'hideAxes','minXStep','hideGrid','majorGridLineColor', + 'minorGridLineColor','thickness','min','max', + 'graphOnly','yMin','yMax','yLimit','yStep','areaMode', + 'areaAlpha','drawNullAsZero','tz', 'yAxisSide','pieMode', + 'yUnitSystem', 'logBase','yMinLeft','yMinRight','yMaxLeft', + 'yMaxRight', 'yLimitLeft', 'yLimitRight', 'yStepLeft', + 'yStepRight', 'rightWidth', 'rightColor', 'rightDashed', + 'leftWidth', 'leftColor', 'leftDashed', 'xFormat', 'minorY', + 'hideYAxis', 'uniqueLegend', 'vtitleRight', 'yDivisors', 'connectedLimit', 'hideXAxis', 'hideNullFromLegend') validLineModes = ('staircase','slope','connected') validAreaModes = ('none','first','all','stacked') diff --git a/webapp/graphite/settings.py b/webapp/graphite/settings.py index d7a1deadd..657af8441 100644 --- a/webapp/graphite/settings.py +++ b/webapp/graphite/settings.py @@ -298,7 +298,7 @@ # Default sqlite db file # This is set here so that a user-set STORAGE_DIR is available if 'sqlite3' in DATABASES.get('default',{}).get('ENGINE','') \ - and not DATABASES.get('default',{}).get('NAME'): + and not DATABASES.get('default',{}).get('NAME'): DATABASES['default']['NAME'] = join(STORAGE_DIR, 'graphite.db') # Caching shortcuts diff --git a/webapp/tests/test_functions.py b/webapp/tests/test_functions.py index 0010dfd52..fb0289b7e 100644 --- a/webapp/tests/test_functions.py +++ b/webapp/tests/test_functions.py @@ -5163,7 +5163,7 @@ def test_holtWintersConfidenceBands(self): def hw_range(x,y,jump): while x Date: Fri, 28 Feb 2020 20:37:21 -0500 Subject: [PATCH 4/4] flake8: enable for all subdirs except contrib/ and also exclude .tox/ which has other packages source installed in it also print flake8 and plugin versions in test output, and on failures include the source line for a bit of context --- bin/run-graphite-devel-server.py | 3 ++- check-dependencies.py | 18 ++++++++++++++++++ docs/conf.py | 17 +++++++++++------ examples/example-client.py | 12 +++++++----- setup.cfg | 1 + setup.py | 2 +- tox.ini | 4 ++-- 7 files changed, 42 insertions(+), 15 deletions(-) diff --git a/bin/run-graphite-devel-server.py b/bin/run-graphite-devel-server.py index 79f2851e1..deb26eb49 100755 --- a/bin/run-graphite-devel-server.py +++ b/bin/run-graphite-devel-server.py @@ -1,6 +1,7 @@ #!/usr/bin/env python -import sys, os +import os +import sys from optparse import OptionParser from django.core import management diff --git a/check-dependencies.py b/check-dependencies.py index 78cbacc1e..f89c4ae6b 100755 --- a/check-dependencies.py +++ b/check-dependencies.py @@ -156,3 +156,21 @@ sys.exit(1) else: print("All necessary dependencies are met.") + + +# suppress unused-import warnings +__all__ = [ + 'whisper', + 'ceres', + 'cairo', + 'django', + 'pytz', + 'pyparsing', + 'tagging', + 'memcache', + 'ldap', + 'txamqp', + 'rrdtool', + 'whitenoise', + 'pyhash', +] diff --git a/docs/conf.py b/docs/conf.py index e6d0a7677..f75aac269 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,7 +11,14 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import os +import re +import sys + +from sphinx.ext import autodoc + +# Bring in the new ReadTheDocs sphinx theme +import sphinx_rtd_theme # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -22,7 +29,7 @@ os.environ['DJANGO_SETTINGS_MODULE'] = "graphite.settings" # Prevent graphite logger from complaining about missing log dir. -from graphite import settings +from graphite import settings # noqa: E402 settings.LOG_DIR = os.path.abspath('.') try: @@ -32,13 +39,9 @@ else: setup() -# Bring in the new ReadTheDocs sphinx theme -import sphinx_rtd_theme # Define a custom autodoc documenter for the render.functions module # This will remove the requestContext parameter which doesnt make sense in the context of the docs -import re -from sphinx.ext import autodoc class RenderFunctionDocumenter(autodoc.FunctionDocumenter): priority = 10 # Override FunctionDocumenter @@ -53,9 +56,11 @@ def format_args(self): # Really, a regex sub here is by far the easiest way return re.sub('requestContext, ','',args) + def setup(app): app.add_autodocumenter(RenderFunctionDocumenter) + # -- General configuration ----------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be extensions diff --git a/examples/example-client.py b/examples/example-client.py index 044b0e6dc..708963b07 100644 --- a/examples/example-client.py +++ b/examples/example-client.py @@ -16,22 +16,23 @@ import sys import time import os -import platform +import platform import subprocess from socket import socket CARBON_SERVER = '127.0.0.1' CARBON_PORT = 2003 -delay = 60 +delay = 60 if len(sys.argv) > 1: delay = int( sys.argv[1] ) + def get_loadavg(): - # For more details, "man proc" and "man uptime" + # For more details, "man proc" and "man uptime" if platform.system() == "Linux": return open('/proc/loadavg').read().strip().split()[:3] - else: + else: command = "uptime" process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) os.waitpid(process.pid, 0) @@ -39,10 +40,11 @@ def get_loadavg(): length = len(output) return output[length - 3:length] + sock = socket() try: sock.connect( (CARBON_SERVER,CARBON_PORT) ) -except: +except Exception: print("Couldn't connect to %(server)s on port %(port)d, is carbon-agent.py running?" % { 'server':CARBON_SERVER, 'port':CARBON_PORT }) sys.exit(1) diff --git a/setup.cfg b/setup.cfg index bfd07e2b0..4adb04e4b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,6 +18,7 @@ provides = graphite obsoletes = graphite <= 0.9.9 [flake8] +exclude = .tox,contrib ignore = # E111 indentation is not a multiple of four E111, diff --git a/setup.py b/setup.py index 058778d53..b94509505 100644 --- a/setup.py +++ b/setup.py @@ -134,7 +134,7 @@ def read(fname): 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', - ], + ], **setup_kwargs ) finally: diff --git a/tox.ini b/tox.ini index 1a5cbd1d1..3303703a7 100644 --- a/tox.ini +++ b/tox.ini @@ -65,5 +65,5 @@ basepython = python3.8 deps = flake8 commands = - flake8 {toxinidir}/webapp/graphite - flake8 {toxinidir}/webapp/tests + flake8 --version + flake8 --show-source {toxinidir}