From b1b3f24409cc5147a6a9a9685dba24b2bdcfed9b Mon Sep 17 00:00:00 2001 From: Francis Begyn Date: Wed, 22 Feb 2023 17:57:51 +0100 Subject: [PATCH 1/2] inital attempt to add logging to monitoring --- docker-compose.yml | 20 ++++++++- .../provisioning/datasources/datasource.yml | 5 +++ loki/config.yaml | 39 +++++++++++++++++ promtail/config.yaml | 43 +++++++++++++++++++ 4 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 loki/config.yaml create mode 100644 promtail/config.yaml diff --git a/docker-compose.yml b/docker-compose.yml index 368bdc1..94f4184 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,8 @@ version: "3.7" volumes: telraam_grafana_storage: {} prometheus_data: {} + loki_data: {} + promtail_data: {} services: timesync: @@ -11,7 +13,7 @@ services: dockerfile: Dockerfile network_mode: host prometheus: - image: prom/prometheus:v2.35.0 + image: prom/prometheus:v2.42.0 volumes: - ./prometheus/:/etc/prometheus - prometheus_data:/prometheus @@ -47,3 +49,19 @@ services: - ./grafana/provisioning/:/etc/grafana/provisioning/ env_file: - ./grafana/config.monitoring + loki: + image: grafana/loki + command: "-config.file /etc/loki/config.yaml" + network_mode: host + volumes: + - ./loki:/etc/loki + - loki_data:/loki + promtail: + image: grafana/promtail + network_mode: host + command: "-config.file /etc/promtail/config.yaml" + volumes: + - ./promtail:/etc/promtail + - promtail_data:/promtail + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/log:/var/log:ro diff --git a/grafana/provisioning/datasources/datasource.yml b/grafana/provisioning/datasources/datasource.yml index e0d9536..42c460a 100644 --- a/grafana/provisioning/datasources/datasource.yml +++ b/grafana/provisioning/datasources/datasource.yml @@ -48,6 +48,11 @@ datasources: version: 1 # allow users to edit datasources from the UI. editable: true + - name: Loki + type: loki + access: proxy + orgId: 1 + url: http://localhost:3100 - name: 'ManualCount API' type: 'marcusolsson-json-datasource' url: 'http://172.12.50.21:3000' diff --git a/loki/config.yaml b/loki/config.yaml new file mode 100644 index 0000000..410e686 --- /dev/null +++ b/loki/config.yaml @@ -0,0 +1,39 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +ingester: + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 5m + chunk_retain_period: 30s + wal: + dir: /loki/wal + +schema_config: + configs: + - from: 2020-05-15 + store: boltdb + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 168h + +storage_config: + boltdb: + directory: /loki/index + + filesystem: + directory: /loki/chunks + +limits_config: + enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 168h diff --git a/promtail/config.yaml b/promtail/config.yaml new file mode 100644 index 0000000..4bca90b --- /dev/null +++ b/promtail/config.yaml @@ -0,0 +1,43 @@ +server: + http_listen_address: 0.0.0.0 + http_listen_port: 9080 + grpc_listen_port: 9096 + +positions: + filename: /promtail/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: +- job_name: system + static_configs: + - targets: + - localhost + labels: + job: varlogs + __path__: /var/log/*log +- job_name: containers + static_configs: + - targets: + - localhost + labels: + job: containerlogs + __path__: /var/lib/docker/containers/*/*log + # --log-opt tag="{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}" + pipeline_stages: + - json: + expressions: + stream: stream + attrs: attrs + tag: attrs.tag + - regex: + expression: (?P(?:[^|]*[^|])).(?P(?:[^|]*[^|])).(?P(?:[^|]*[^|])).(?P(?:[^|]*[^|])) + source: "tag" + - labels: + tag: + stream: + image_name: + container_name: + image_id: + container_id: From ddf741d5193282b3efc22e7068b06c486dd5000a Mon Sep 17 00:00:00 2001 From: Francis Begyn Date: Wed, 22 Feb 2023 20:31:11 +0100 Subject: [PATCH 2/2] finetune the promtail settings to also parse journald --- promtail/config.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/promtail/config.yaml b/promtail/config.yaml index 4bca90b..e45fd6c 100644 --- a/promtail/config.yaml +++ b/promtail/config.yaml @@ -7,7 +7,7 @@ positions: filename: /promtail/positions.yaml clients: - - url: http://loki:3100/loki/api/v1/push + - url: http://localhost:3100/loki/api/v1/push scrape_configs: - job_name: system @@ -24,7 +24,6 @@ scrape_configs: labels: job: containerlogs __path__: /var/lib/docker/containers/*/*log - # --log-opt tag="{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}" pipeline_stages: - json: expressions: @@ -41,3 +40,13 @@ scrape_configs: container_name: image_id: container_id: +- job_name: journal + journal: + json: false + max_age: 12h + path: /var/log/journal + labels: + job: systemd-journal + relabel_configs: + - source_labels: ['__journal_systemd_unit'] + target_label: 'unit'