Merge branch 'feature/counter' into 'master'

Feature/counter

See merge request grafolean/grafolean-collector-snmp!7

.gitlab-ci.yml

@@ -15,7 +15,7 @@ pytest:
   image: python:3.6-slim-stretch
   before_script:
     - apt-get update
-    - apt-get install --no-install-recommends -q -y libsnmp-dev build-essential
+    - apt-get install --no-install-recommends -q -y libsnmp-dev build-essential git
     - pip install --no-cache-dir pipenv
     - pipenv install --dev
   script:

Dockerfile

@@ -31,7 +31,7 @@ LABEL org.label-schema.vendor="Grafolean" \
 COPY --from=python-requirements /requirements.txt /requirements.txt
 RUN \
     apt-get update && \
-    apt-get install --no-install-recommends -q -y libsnmp-dev build-essential&& \
+    apt-get install --no-install-recommends -q -y libsnmp-dev build-essential git && \
     pip install --no-cache-dir -r /requirements.txt && \
     apt-get purge -y build-essential && \
     apt-get clean autoclean && \

Pipfile

@@ -12,7 +12,7 @@ requests = "*"
 python-dotenv = "*"
 apscheduler = "*"
 ansicolors = "*"
-easysnmp = "*"
+easysnmp = {editable = true,git = "http://github.com/grafolean/easysnmp"}
 mathjspy = "*"
 numpy = "*"
 

snmpcollector.py

@@ -2,11 +2,12 @@
 import dotenv
 import logging
 import json
+import time
 from pytz import utc
 from colors import color
 import requests
 
-from easysnmp import Session
+from easysnmp import Session, SNMPVariable
 from mathjspy import MathJS
 
 from collector import Collector
@@ -25,6 +26,50 @@ class NoValueForOid(Exception):
     pass
 
 
+previous_counter_values = {}
+
+
+def _get_previous_counter_value(counter_ident):
+    prev_value = previous_counter_values.get(counter_ident)
+    if prev_value is None:
+        return None, None
+    return prev_value
+
+
+def _save_current_counter_value(new_value, now, counter_ident):
+    previous_counter_values[counter_ident] = (new_value, now)
+
+
+def _convert_counters_to_values(results, now, counter_ident_prefix):
+    new_results = []
+    for i, v in enumerate(results):
+        if isinstance(v, list):
+            new_results.append(_convert_counters_to_values(v, now, counter_ident_prefix + f'/{i}'))
+            continue
+        if v.snmp_type not in ['COUNTER', 'COUNTER64']:
+            new_results.append(v)
+            continue
+        # counter - deal with it:
+        counter_ident = counter_ident_prefix + f'/{i}/{v.oid}/{v.oid_index}'
+        old_value, t = _get_previous_counter_value(counter_ident)
+        new_value = float(v.value)
+        _save_current_counter_value(new_value, now, counter_ident)
+        if old_value is None:
+            new_results.append(SNMPVariable(oid=v.oid, oid_index=v.oid_index, value=None, snmp_type='COUNTER_PER_S'))
+            continue
+
+        # it seems like the counter overflow happened, discard result:
+        if new_value < old_value:
+            new_results.append(SNMPVariable(oid=v.oid, oid_index=v.oid_index, value=None, snmp_type='COUNTER_PER_S'))
+            log.warning(f"Counter overflow detected for oid {v.oid}, oid index {v.oid_index}, discarding value - if this happens often, consider using OIDS with 64bit counters (if available) or decreasing polling interval.")
+            continue
+
+        dt = now - t
+        dv = (new_value - old_value) / dt
+        new_results.append(SNMPVariable(oid=v.oid, oid_index=v.oid_index, value=dv, snmp_type='COUNTER_PER_S'))
+    return new_results
+
+
 def _apply_expression_to_results(snmp_results, methods, expression, output_path):
     if 'walk' in methods:
         """
@@ -48,7 +93,9 @@ def _apply_expression_to_results(snmp_results, methods, expression, output_path)
                 mjs = MathJS()
                 for i, r in enumerate(addressable_results):
                     v = r.get(oid_index)
-                    if v is None:
+                    if v is None:  # oid index wasn't present
+                        raise NoValueForOid()
+                    if v.value is None:  # no value (probably the first time we're asking for a counter)
                         raise NoValueForOid()
                     mjs.set('${}'.format(i + 1), float(v.value))
                 value = mjs.eval(expression)
@@ -61,13 +108,19 @@ def _apply_expression_to_results(snmp_results, methods, expression, output_path)
         return result
 
     else:
-        mjs = MathJS()
-        for i, r in enumerate(snmp_results):
-            mjs.set('${}'.format(i + 1), float(r.value))
-        value = mjs.eval(expression)
-        return [
-            {'p': output_path, 'v': value},
-        ]
+        try:
+            mjs = MathJS()
+            for i, v in enumerate(snmp_results):
+                if v.value is None:  # no value (probably the first time we're asking for a counter)
+                    raise NoValueForOid()
+                mjs.set('${}'.format(i + 1), float(v.value))
+            value = mjs.eval(expression)
+            return [
+                {'p': output_path, 'v': value},
+            ]
+        except NoValueForOid:
+            log.warning(f'Missing OID value (counter?)')
+            return []
 
 
 def send_results_to_grafolean(backend_url, bot_token, account_id, values):
@@ -178,15 +231,17 @@ def do_snmp(*args, **job_info):
                     # while we are at it, save the indexes of the results:
                     if not walk_indexes:
                         walk_indexes = [r.oid_index for r in result]
-            oids_results = list(zip(oids, methods, results))
-            log.info("Results: {}".format(oids_results))
+            log.info("Results: {}".format(list(zip(oids, methods, results))))
+
+            counter_ident_prefix = f'{job_info["entity_id"]}/{sensor["sensor_details"]["id"]}'
+            results_no_counters = _convert_counters_to_values(results, time.time(), counter_ident_prefix)
 
             # We have SNMP results and expression - let's calculate value(s). The trick here is that
             # if some of the data is fetched via SNMP WALK, we will have many results; if only SNMP
             # GET was used, we get one.
             expression = sensor["sensor_details"]["expression"]
             output_path = f'entity.{job_info["entity_id"]}.snmp.{sensor["sensor_details"]["output_path"]}'
-            values = _apply_expression_to_results(results, methods, expression, output_path)
+            values = _apply_expression_to_results(results_no_counters, methods, expression, output_path)
             send_results_to_grafolean(job_info['backend_url'], job_info['bot_token'], job_info['account_id'], values)
 
 

test_snmpcollector.py

@@ -1,11 +1,11 @@
 from easysnmp import SNMPVariable
 
-from snmpcollector import _apply_expression_to_results
+from snmpcollector import _apply_expression_to_results, _convert_counters_to_values
 
 
-def test_snmpget():
+def test_apply_expression_snmpget():
     results = [
-        SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='5', value=68000, snmp_type='GAUGE'),
+        SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='5', value='68000', snmp_type='GAUGE'),
     ]
     methods = ['walk' if isinstance(x, list) else 'get' for x in results]
     expression = '$1'
@@ -15,10 +15,10 @@ def test_snmpget():
     ]
     assert _apply_expression_to_results(results, methods, expression, output_path) == expected_result
 
-def test_snmpget_add():
+def test_apply_expression_snmpget_add():
     results = [
-        SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='5', value=68000, snmp_type='GAUGE'),
-        SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='1', value=200, snmp_type='GAUGE'),
+        SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='5', value='68000', snmp_type='GAUGE'),
+        SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='1', value='200', snmp_type='GAUGE'),
     ]
     methods = ['walk' if isinstance(x, list) else 'get' for x in results]
     expression = '$1 + $2'
@@ -28,12 +28,12 @@ def test_snmpget_add():
     ]
     assert _apply_expression_to_results(results, methods, expression, output_path) == expected_result
 
-def test_snmpwalk():
+def test_apply_expression_snmpwalk():
     results = [
         [
-            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='1', value=60000, snmp_type='GAUGE'),
-            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='2', value=61000, snmp_type='GAUGE'),
-            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='3', value=62000, snmp_type='GAUGE'),
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='1', value='60000', snmp_type='GAUGE'),
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='2', value='61000', snmp_type='GAUGE'),
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='3', value='62000', snmp_type='GAUGE'),
         ],
     ]
     methods = ['walk' if isinstance(x, list) else 'get' for x in results]
@@ -46,13 +46,13 @@ def test_snmpwalk():
     ]
     assert _apply_expression_to_results(results, methods, expression, output_path) == expected_result
 
-def test_expression_add():
+def test_apply_expression_expression_add():
     results = [
-        SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.8', oid_index='23', value=500, snmp_type='GAUGE'),
+        SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.8', oid_index='23', value='500', snmp_type='GAUGE'),
         [
-            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='1', value=60000, snmp_type='GAUGE'),
-            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='2', value=61000, snmp_type='GAUGE'),
-            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='3', value=62000, snmp_type='GAUGE'),
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='1', value='60000', snmp_type='GAUGE'),
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='2', value='61000', snmp_type='GAUGE'),
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='3', value='62000', snmp_type='GAUGE'),
         ],
     ]
     methods = ['walk' if isinstance(x, list) else 'get' for x in results]
@@ -65,16 +65,16 @@ def test_expression_add():
     ]
     assert _apply_expression_to_results(results, methods, expression, output_path) == expected_result
 
-def test_snmpwalk_missing_value():
+def test_apply_expression_snmpwalk_missing_value_walk():
     results = [
         [
-            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='1', value=60000, snmp_type='GAUGE'),
-            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='2', value=61000, snmp_type='GAUGE'),
-            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='3', value=62000, snmp_type='GAUGE'),
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='1', value='60000', snmp_type='GAUGE'),
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='2', value='61000', snmp_type='GAUGE'),
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='3', value='62000', snmp_type='GAUGE'),
         ],
         [
-            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.2.2', oid_index='1', value=10, snmp_type='GAUGE'),
-            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.2.2', oid_index='2', value=10, snmp_type='GAUGE'),
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.2.2', oid_index='1', value='10', snmp_type='GAUGE'),
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.2.2', oid_index='2', value='10', snmp_type='GAUGE'),
         ],
     ]
     methods = ['walk' if isinstance(x, list) else 'get' for x in results]
@@ -85,3 +85,66 @@ def test_snmpwalk_missing_value():
         { 'p': 'snmp.test123.asdf.2', 'v': 6100.0 },
     ]
     assert _apply_expression_to_results(results, methods, expression, output_path) == expected_result
+
+def test_apply_expression_snmpwalk_missing_value_get():
+    results = [
+        SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='1', value='60000', snmp_type='GAUGE'),
+        SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.2.2', oid_index='1', value=None, snmp_type='GAUGE'),
+    ]
+    methods = ['walk' if isinstance(x, list) else 'get' for x in results]
+    expression = '$1 / $2'
+    output_path = 'snmp.test123.asdf'
+    expected_result = []
+    assert _apply_expression_to_results(results, methods, expression, output_path) == expected_result
+
+def test_convert_counters_no_counters_no_change():
+    """ If there are no counters, nothing should change """
+    now = 1234567890.123456
+    results = [
+        [
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='1', value='60000', snmp_type='GAUGE'),
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='2', value='61000', snmp_type='GAUGE'),
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.1.3', oid_index='3', value='62000', snmp_type='GAUGE'),
+        ],
+        [
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.2.2', oid_index='1', value='10', snmp_type='GAUGE'),
+            SNMPVariable(oid='1.3.6.1.4.1.2021.13.16.2.2.2', oid_index='2', value='10', snmp_type='GAUGE'),
+        ],
+    ]
+    assert _convert_counters_to_values(results, now, "ASDF/1234") == results
+
+def test_convert_counters_counter():
+    """ First expression should be empty, next ones should work """
+    now = 1234567890.123456
+
+    results_0 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.16', oid_index='1', value='1000', snmp_type='COUNTER')]
+    expected_0 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.16', oid_index='1', value=None, snmp_type='COUNTER_PER_S')]
+    assert _convert_counters_to_values(results_0, now, "ASDF/1234") == expected_0
+
+    results_1 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.16', oid_index='1', value='2000.0', snmp_type='COUNTER')]
+    expected_1 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.16', oid_index='1', value='1000.0', snmp_type='COUNTER_PER_S')]
+    assert _convert_counters_to_values(results_1, now + 1.0, "ASDF/1234") == expected_1
+
+    results_2 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.16', oid_index='1', value='2300.0', snmp_type='COUNTER')]
+    expected_2 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.16', oid_index='1', value='100.0', snmp_type='COUNTER_PER_S')]
+    assert _convert_counters_to_values(results_2, now + 1.0 + 3.0, "ASDF/1234") == expected_2
+
+def test_convert_counters_overflow():
+    """ First expression should be empty, next ones should work """
+    now = 1234567890.123456
+
+    results_0 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.17', oid_index='1', value='123000.0', snmp_type='COUNTER')]
+    expected_0 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.17', oid_index='1', value=None, snmp_type='COUNTER_PER_S')]
+    assert _convert_counters_to_values(results_0, now, "ASDF/1234") == expected_0
+
+    results_1 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.17', oid_index='1', value='234000.0', snmp_type='COUNTER')]
+    expected_1 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.17', oid_index='1', value='111000.0', snmp_type='COUNTER_PER_S')]
+    assert _convert_counters_to_values(results_1, now + 1.0, "ASDF/1234") == expected_1
+
+    results_2 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.17', oid_index='1', value='1000.0', snmp_type='COUNTER')]
+    expected_2 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.17', oid_index='1', value=None, snmp_type='COUNTER_PER_S')]
+    assert _convert_counters_to_values(results_2, now + 1.0 + 3.0, "ASDF/1234") == expected_2
+
+    results_3 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.17', oid_index='1', value='2000.0', snmp_type='COUNTER')]
+    expected_3 = [SNMPVariable(oid='.1.3.6.1.2.1.2.2.1.17', oid_index='1', value='500.0', snmp_type='COUNTER_PER_S')]
+    assert _convert_counters_to_values(results_3, now + 1.0 + 3.0 + 2.0, "ASDF/1234") == expected_3