Add logging

docker-compose.yml

@@ -3,6 +3,8 @@ version: "3.7"
 volumes:
   telraam_grafana_storage: {}
   prometheus_data: {}
+  loki_data: {}
+  promtail_data: {}
 
 services:
   timesync:
@@ -11,7 +13,7 @@ services:
       dockerfile: Dockerfile
     network_mode: host
   prometheus:
-    image: prom/prometheus:v2.35.0
+    image: prom/prometheus:v2.42.0
     volumes:
       - ./prometheus/:/etc/prometheus
       - prometheus_data:/prometheus
@@ -47,3 +49,19 @@ services:
       - ./grafana/provisioning/:/etc/grafana/provisioning/
     env_file:
       - ./grafana/config.monitoring
+  loki:
+    image: grafana/loki
+    command: "-config.file /etc/loki/config.yaml"
+    network_mode: host
+    volumes:
+      - ./loki:/etc/loki
+      - loki_data:/loki
+  promtail:
+    image: grafana/promtail
+    network_mode: host
+    command: "-config.file /etc/promtail/config.yaml"
+    volumes:
+      - ./promtail:/etc/promtail
+      - promtail_data:/promtail
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/log:/var/log:ro

grafana/provisioning/datasources/datasource.yml

@@ -48,6 +48,11 @@ datasources:
     version: 1
     # <bool> allow users to edit datasources from the UI.
     editable: true
+  - name: Loki
+    type: loki
+    access: proxy
+    orgId: 1
+    url: http://localhost:3100
   - name: 'ManualCount API'
     type: 'marcusolsson-json-datasource'
     url: 'http://172.12.50.21:3000'

loki/config.yaml

@@ -0,0 +1,39 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+ingester:
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+  wal:
+    dir: /loki/wal
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: boltdb
+    object_store: filesystem
+    schema: v11
+    index:
+      prefix: index_
+      period: 168h
+
+storage_config:
+  boltdb:
+    directory: /loki/index
+
+  filesystem:
+    directory: /loki/chunks
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h

promtail/config.yaml

@@ -0,0 +1,52 @@
+server:
+  http_listen_address: 0.0.0.0
+  http_listen_port: 9080
+  grpc_listen_port: 9096
+
+positions:
+  filename: /promtail/positions.yaml
+
+clients:
+  - url: http://localhost:3100/loki/api/v1/push
+
+scrape_configs:
+- job_name: system
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: varlogs
+      __path__: /var/log/*log
+- job_name: containers
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: containerlogs
+      __path__: /var/lib/docker/containers/*/*log
+  pipeline_stages:
+  - json:
+      expressions:
+        stream: stream
+        attrs: attrs
+        tag: attrs.tag
+  - regex:
+      expression: (?P<image_name>(?:[^|]*[^|])).(?P<container_name>(?:[^|]*[^|])).(?P<image_id>(?:[^|]*[^|])).(?P<container_id>(?:[^|]*[^|]))
+      source: "tag"
+  - labels:
+      tag:
+      stream:
+      image_name:
+      container_name:
+      image_id:
+      container_id:
+- job_name: journal
+  journal:
+    json: false
+    max_age: 12h
+    path: /var/log/journal
+    labels:
+      job: systemd-journal
+  relabel_configs:
+    - source_labels: ['__journal_systemd_unit']
+      target_label: 'unit'