Metrics (#51)

* add prometheus metrics

* add docs

* metrics snippet

* bump version

* fix typo

* remove redundant flag

* add doc string, add to defaults test

* add prometheus unit tests

* update docs

* add ability to bind to different interface

* fix flake8

* fix readme

* 0.3.0

* fix readme

* fix readme

README.md

@@ -97,17 +97,23 @@ docker run --rm circa10a/ouroboros --help
 - `--cleanup`, `-c` Remove the older docker image if a new one is found and updated.
   - Default is `False`.
   - Environment variable: `CLEANUP=true`
-- `--keep-tag`, `-k` Only monitor if updates are made to the tag of the image that the container was created with instead of using `latest`.
+- `--keep-tag`, `-k` Only monitor if updates are made to the tag of the image that the container was created with instead of using `latest`. This will enable [watchtower](https://github.com/v2tec/watchtower)-like functionality.
   - Default is `False`.
   - Environment variable: `KEEPTAG=true`
+- `--metrics-addr` What address for the prometheus endpoint to bind to. Runs on `127.0.0.1` by default if `--metrics-addr` is not supplied.
+  - Default is `127.0.0.1`.
+  - Environment variable: `METRICS_ADDR=127.0.0.1`
+- `--metrics-port` What port to run prometheus endpoint on. Running on port `8000` by default if `--metrics-port` is not supplied.
+  - Default is `8000`.
+  - Environment variable: `METRICS_PORT=8000`
 
 ### Private Registries
 
 If your running containers' docker images are stored in a secure registry that requires a username and password, simply run ouroboros with 2 environment variables(`REPO_USER` and `REPO_PASS`).
 
 ```bash
 docker run -d --name ouroboros \
-  -v REPO_USER=myUser -e REPO_PASS=myPassword \
+  -e REPO_USER=myUser -e REPO_PASS=myPassword \
   -v /var/run/docker.sock:/var/run/docker.sock \
   circa10a/ouroboros
 ```
@@ -205,6 +211,55 @@ docker run -d --name ouroboros \
   circa10a/ouroboros --cleanup
 ```
 
+### Prometheus metrics
+
+Ouroboros keeps track of containers being updated and how many are being monitored. Said metrics are exported using [prometheus](https://prometheus.io/). Metrics are collected by ouroboros with or without this flag, it is up to you if you would like to expose the port or not. You can also bind the http server to a different interface for systems using multiple networks. `--metrics-port` and `--metrics-addr` can run independently of each other without issue.
+
+#### Port
+
+> Default is `8000`
+
+```bash
+docker run -d --name ouroboros \
+  -p 5000:5000 \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  circa10a/ouroboros --metrics-port 5000
+```
+
+You should then be able to see the metrics at http://localhost:5000/
+
+#### Bind Address
+
+Ouroboros allows you to bind the exporter to a different interface using the `--metrics-addr` argument.
+
+> Default is `127.0.0.1`
+
+```bash
+docker run -d --name ouroboros \
+  -p 8000:8000 \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  circa10a/ouroboros --metrics-addr 10.0.0.1
+```
+
+Then access via http://10.0.0.1:8000/
+
+**Example text from endpoint:**
+
+```
+# HELP containers_updated_total Count of containers updated
+# TYPE containers_updated_total counter
+containers_updated_total{container="all"} 2.0
+containers_updated_total{container="alpine"} 1.0
+containers_updated_total{container="busybox"} 1.0
+# TYPE containers_updated_created gauge
+containers_updated_created{container="all"} 1542152615.625264
+containers_updated_created{container="alpine"} 1542152615.6252713
+containers_updated_created{container="busybox"} 1542152627.7476819
+# HELP containers_being_monitored Count of containers being monitored
+# TYPE containers_being_monitored gauge
+containers_being_monitored 2.0
+```
+
 ## Execute Tests
 
 > Script will install dependencies from `requirements-dev.txt`

dev-environment.yml

@@ -3,6 +3,7 @@ dependencies:
   - pip:
       - docker
       - schedule
+      - prometheus_client
       - pytest
       - pytest-cov
       - pytest-mock

environment.yml

@@ -3,3 +3,4 @@ dependencies:
   - pip:
       - docker
       - schedule
+      - prometheus_client

ouroboros/cli.py

@@ -35,11 +35,10 @@ def checkURI(uri):
     return re.match(regex, uri)
 
 
-def get_interval_env():
-    """Attempt to convert INTERVAL environment variable to int"""
-    int_env = environ.get('INTERVAL')
+def get_int_env_var(env_var):
+    """Attempt to convert environment variable to int"""
     try:
-        return int(int_env)
+        return int(env_var)
     except (ValueError, TypeError):
         return False
 
@@ -53,7 +52,7 @@ def parse(sysargs):
     parser.add_argument('-u', '--url', default=defaults.LOCAL_UNIX_SOCKET,
                         help='Url for tcp host (defaults to "unix://var/run/docker.sock")')
 
-    parser.add_argument('-i', '--interval', type=int, default=get_interval_env() or defaults.INTERVAL, dest='interval',
+    parser.add_argument('-i', '--interval', type=int, default=get_int_env_var(env_var=environ.get('INTERVAL')) or defaults.INTERVAL, dest='interval',
                         help='Interval in seconds between checking for updates (defaults to 300s)')
 
     parser.add_argument('-m', '--monitor', nargs='+', default=environ.get('MONITOR') or [], dest='monitor',
@@ -74,6 +73,12 @@ def parse(sysargs):
 
     parser.add_argument('-k', '--keep-tag', default=environ.get('KEEPTAG') or False, dest='keep_tag',
                         help='Check for image updates of the same tag instead of pulling latest', action='store_true')
+
+    parser.add_argument('--metrics-addr', default=environ.get('METRICS_ADDR') or defaults.METRICS_ADDR, dest='metrics_addr',
+                        help='Bind address to run Prometheus exporter on')
+
+    parser.add_argument('--metrics-port', type=int, default=get_int_env_var(env_var=environ.get('METRICS_PORT')) or defaults.METRICS_PORT, dest='metrics_port',
+                        help='Port to run Prometheus exporter on')
     args = parser.parse_args(sysargs)
 
     if not args.url:

ouroboros/defaults.py

@@ -5,3 +5,5 @@
 RUNONCE = False
 CLEANUP = False
 KEEPTAG = False
+METRICS_ADDR = '127.0.0.1'
+METRICS_PORT = 8000

ouroboros/main.py

@@ -3,6 +3,7 @@
 import docker
 from ouroboros import container
 from ouroboros import image
+from ouroboros import metrics
 
 
 def main(args, api_client):
@@ -12,7 +13,10 @@ def main(args, api_client):
         log.info('No containers are running')
     else:
         updated_count = 0
-        for running_container in container.to_monitor(monitor=args.monitor, ignore=args.ignore, api_client=api_client):
+        monitored_containers = container.to_monitor(monitor=args.monitor, ignore=args.ignore, api_client=api_client)
+        metrics.monitored_containers(num=len(monitored_containers))
+        for running_container in monitored_containers:
+            container_name = f'{container.get_name(container_object=running_container)}'
             current_image = api_client.inspect_image(running_container['Config']['Image'])
 
             try:
@@ -40,6 +44,10 @@ def main(args, api_client):
                 if args.cleanup:
                     image.remove(old_image=current_image, api_client=api_client)
                 updated_count += 1
+
+                metrics.container_updates(label='all')
+                metrics.container_updates(label=container_name)
+
         log.info(f'{updated_count} container(s) updated')
         if args.run_once:
             exit(0)

ouroboros/metrics.py

@@ -0,0 +1,17 @@
+from prometheus_client import Counter, Gauge
+
+updated_containers_counter = Counter(
+    'containers_updated', 'Count of containers updated', ['container'])
+
+monitored_containers_gauge = Gauge(
+    'containers_being_monitored', 'Count of containers being monitored', [])
+
+
+def container_updates(label):
+    """Increment container update count based on label"""
+    updated_containers_counter.labels(container=label).inc()
+
+
+def monitored_containers(num):
+    """Set number of containers being monitoring with a gauge"""
+    monitored_containers_gauge.set(num)

ouroboros/ouroboros

@@ -4,12 +4,14 @@ if __name__ == "__main__":
     import sys
     import ouroboros.cli as cli
     from ouroboros.logger import set_logger
+    from prometheus_client import start_http_server
     import docker
     import schedule
     import logging
     from ouroboros.main import main
     import time
     args = cli.parse(sys.argv[1:])
+    start_http_server(args.metrics_port, addr=args.metrics_addr)
     api_client = docker.APIClient(base_url=args.url)
     logging.basicConfig(**set_logger(args.loglevel))
     schedule.every(args.interval).seconds.do(

requirements-dev.txt

@@ -1,5 +1,6 @@
 docker
 schedule
+prometheus_client
 pytest
 pytest-cov
 pytest-mock

requirements.txt

@@ -1,2 +1,3 @@
 docker
-schedule
+schedule
+prometheus_client

setup.py

@@ -13,7 +13,7 @@ def read_reqs(requirements):
 
 setup(
     name='ouroboros-cli',
-    version='0.2.3',
+    version='0.3.0',
     description='Automatically update running docker containers',
     long_description=readme(),
     long_description_content_type='text/markdown',

tests/unit/_metrics_test.py

@@ -0,0 +1,17 @@
+import pytest
+from prometheus_client import REGISTRY
+import ouroboros.metrics as metrics
+
+
+def test_container_updates():
+    test_label = 'test'
+    metrics.container_updates(label=test_label)
+    increment = REGISTRY.get_sample_value('containers_updated_total', labels={'container': test_label})
+    assert increment == 1.0
+
+
+def test_monitored_containers():
+    test_count = 5.0
+    metrics.monitored_containers(num=test_count)
+    num_monitored = REGISTRY.get_sample_value('containers_being_monitored')
+    assert num_monitored == test_count

tests/unit/cli_test.py

@@ -1,3 +1,4 @@
+from os import environ
 import pytest
 import ouroboros.cli as cli
 import ouroboros.defaults as defaults
@@ -22,16 +23,15 @@ def test_url_args(mocker, url_args, url_result):
     args = cli.parse(url_args)
     assert args.url == url_result
 
-# Interval
-
 
+# Interval
 @pytest.mark.parametrize('interval_env, interval_env_result', [
     ({'INTERVAL': 't'}, False),
     ({'INTERVAL': '10'}, 10),
 ])
-def test_get_interval_env(mocker, interval_env, interval_env_result):
+def get_int_env_var(mocker, interval_env, interval_env_result):
     mocker.patch.dict('os.environ', interval_env)
-    assert cli.get_interval_env() == interval_env_result
+    assert cli.get_int_env_var(env_var=environ.get('INTERVAL')) == interval_env_result
 
 
 def test_interval_arg_invalid_value(mocker):
@@ -45,9 +45,8 @@ def test_interval_arg_valid_value(mocker):
     mocker.patch('ouroboros.cli')
     assert cli.parse(['--interval', 0])
 
-# Monitor
-
 
+# Monitor
 @pytest.mark.parametrize('monitor_args, monitor_result', [
     (['-m', 'test1', 'test2', 'test3'], ['test1', 'test2', 'test3']),
     (['--monitor', 'test1', 'test2', 'test3'], ['test1', 'test2', 'test3']),
@@ -105,6 +104,7 @@ def test_loglevel_env_var(mocker, loglevel_env_var, loglevel_env_var_result):
     assert args.loglevel == loglevel_env_var_result
 
 
+# Run once
 @pytest.mark.parametrize('runonce_args, runonce_result', [
     (['-r', ], True),
     (['--runonce', ], True)
@@ -126,6 +126,7 @@ def test_runonce_env_var(mocker, runonce_env_var, runonce_env_var_result):
     assert args.run_once == runonce_env_var_result
 
 
+# Cleanup
 @pytest.mark.parametrize('cleanup_args, cleanup_result', [
     (['-c', ], True),
     (['--cleanup', ], True)
@@ -147,6 +148,7 @@ def test_cleanup_env_var(mocker, cleanup_env_var, cleanup_env_var_result):
     assert args.cleanup == cleanup_env_var_result
 
 
+# Keeptag
 @pytest.mark.parametrize('keeptag_args, keeptag_result', [
     (['-k', ], True),
     (['--keep-tag', ], True)
@@ -165,4 +167,50 @@ def test_keeptag_env_var(mocker, keeptag_env_var, keeptag_env_var_result):
     mocker.patch.dict('os.environ', keeptag_env_var)
     mocker.patch('ouroboros.cli')
     args = cli.parse([])
-    assert args.keep_tag == keeptag_env_var_result
+    assert args.keep_tag == keeptag_env_var_result
+
+
+# METRICS_ADDR
+@pytest.mark.parametrize('metrics_addr_args, metrics_addr_result', [
+    (['--metrics-addr', '127.0.0.0'], '127.0.0.0')
+])
+def test_metrics_addr_args(mocker, metrics_addr_args, metrics_addr_result):
+    mocker.patch('ouroboros.cli')
+    args = cli.parse(metrics_addr_args)
+    assert args.metrics_addr == metrics_addr_result
+
+
+@pytest.mark.parametrize('metrics_addr_env_var, metrics_addr_env_var_result', [
+    ({'METRICS_ADDR': '127.0.0.0'}, '127.0.0.0'),
+])
+def test_metrics_addr_env_var(mocker, metrics_addr_env_var, metrics_addr_env_var_result):
+    mocker.patch.dict('os.environ', metrics_addr_env_var)
+    mocker.patch('ouroboros.cli')
+    args = cli.parse([])
+    assert args.metrics_addr == metrics_addr_env_var_result
+
+
+# METRICS_PORT
+@pytest.mark.parametrize('metrics_port_args, metrics_port_result', [
+    (['--metrics-port', '8001'], 8001)
+])
+def test_metrics_port_args(mocker, metrics_port_args, metrics_port_result):
+    mocker.patch('ouroboros.cli')
+    args = cli.parse(metrics_port_args)
+    assert args.metrics_port == metrics_port_result
+
+
+@pytest.mark.parametrize('metrics_port_env_var, metrics_port_env_varresult', [
+    ({'METRICS_PORT': 'test'}, False),
+    ({'METRICS_PORT': '8001'}, 8001),
+])
+def get_metrics_port_int_env_var(mocker, metrics_port_env_var, metrics_port_env_var_result):
+    mocker.patch.dict('os.environ', metrics_port_env_var)
+    assert cli.get_int_env_var(env_var=environ.get('METRICS_PORT')) == metrics_port_env_var_result
+
+
+def test_metrics_port_arg_invalid_value(mocker):
+    mocker.patch('ouroboros.cli')
+    with pytest.raises(SystemExit) as pytest_wrapped_e:
+        cli.parse(['--metrics-port', 'test'])
+        assert pytest_wrapped_e.type == SystemExit

tests/unit/defaults_test.py

@@ -7,8 +7,9 @@
     (defaults.MONITOR, []),
     (defaults.LOGLEVEL, 'info'),
     (defaults.RUNONCE, False),
-    (defaults.CLEANUP, False)
+    (defaults.CLEANUP, False),
+    (defaults.METRICS_ADDR, '127.0.0.1'),
+    (defaults.METRICS_PORT, 8000)
 ])
-
 def test_defaults(default, result):
     assert default == result

tests/unit/logger_test.py

@@ -8,9 +8,9 @@
     ('error', 40),
     ('critical', 50),
 ])
-
 def test_logger_levels(level_string, level_code):
     assert log.set_logger(level_string)['level'] == level_code
 
+
 def test_logger_invalid_level():
-    assert log.set_logger('wrong')['level'] == 20
+    assert log.set_logger('wrong')['level'] == 20