Merge pull request #4 from flownative/feature/promtail-3

Promtail 3 and custom labels

Dockerfile

@@ -1,12 +1,12 @@
-FROM grafana/promtail:2.6.1 as promtail
-FROM europe-docker.pkg.dev/flownative/docker/base:buster
-MAINTAINER Robert Lemke <[email protected]>
+FROM grafana/promtail:3.2.0 AS promtail
+FROM europe-docker.pkg.dev/flownative/docker/base:bookworm
+LABEL org.opencontainers.image.authors="Robert Lemke <[email protected]>"
 
 # -----------------------------------------------------------------------------
 # Promtail
 # Latest versions: https://github.com/grafana/loki/releases / https://hub.docker.com/r/grafana/promtail/tags
 
-ENV PROMTAIL_VERSION=2.6.1
+ENV PROMTAIL_VERSION=3.2.0
 
 ENV FLOWNATIVE_LIB_PATH=/opt/flownative/lib \
     PROMTAIL_BASE_PATH=/opt/flownative/promtail \

README.md

@@ -4,10 +4,13 @@
 
 # Flownative Promtail Image
 
-A Docker image providing [Promtail](https://grafana.com/docs/loki/latest/clients/promtail/) for [Beach](https://www.flownative.com/beach),
-[Local Beach](https://www.flownative.com/localbeach) and other purposes. Compared to the
-official Promtail image, this one provides an opinionated configuration which can be
-controlled through environment variables.
+A Docker image
+providing [Promtail](https://grafana.com/docs/loki/latest/clients/promtail/)
+for [Beach](https://www.flownative.com/beach),
+[Local Beach](https://www.flownative.com/localbeach) and other purposes.
+Compared to the
+official Promtail image, this one provides an opinionated configuration which
+can be controlled through environment variables.
 
 ## Sending logs from Local Beach
 
@@ -21,13 +24,13 @@ controlled through environment variables.
       - ./Data/Logs:/application/Data/Logs:delegated
     environment:
       - PROMTAIL_CLIENT_URL=http://loki_loki_1:3100/loki/api/v1/push
-      - PROMTAIL_LABEL_JOB=localbeach-flow
+      - PROMTAIL_LABELS=eyJqb2IiOiAibXktam9iIiwgInNldmVyaXR5IjogImZpbmUifQ==
       - PROMTAIL_LABEL_HOST=localbeach-${BEACH_PROJECT_NAME}
 ```
 
 ## Testing with docker run
 
-Create a directory "test" on your local machine. Within that directory 
+Create a directory "test" on your local machine. Within that directory
 create a file "test.log".
 
 Then run Promtail with `docker run`:
@@ -38,26 +41,24 @@ docker run -v $(pwd)/logs:/logs \
   -e PROMTAIL_CLIENT_URL=https://logs-prod-us-central1.grafana.net/loki/api/v1/push \
   -e PROMTAIL_BASIC_AUTH_USERNAME=loki-username \
   -e PROMTAIL_BASIC_AUTH_PASSWORD=loki-password \
-  -e PROMTAIL_LABEL_JOB=test-job \ 
+  -e PROMTAIL_LABELS=eyJqb2IiOiAibXktam9iIiwgInNldmVyaXR5IjogImZpbmUifQ== \ 
   flownative/promtail
 ```
 
-In a separate terminal, add test content to the log file, for example with 
+In a separate terminal, add test content to the log file, for example with
 `echo "some test" >> logs/test.log`.
 
 ### Environment variables
 
-| Variable Name                | Type   | Default                                  | Description                                                         |
-|------------------------------|--------|------------------------------------------|---------------------------------------------------------------------|
-| PROMTAIL_CLIENT_URL          | string | http://loki_loki_1:3100/loki/api/v1/push | URL pointing to the Loki push endpoint                              |
-| PROMTAIL_LABEL_HOST          | string | $(hostname)                              | Value of the label "host" which is added to all log entries         |
-| PROMTAIL_LABEL_JOB           | string | promtail                                 | Value of the label "job" which is added to all log entries          |
-| PROMTAIL_LABEL_ORGANIZATION  | string |                                          | Value of the label "organization" which is added to all log entries |
-| PROMTAIL_LABEL_PROJECT       | string |                                          | Value of the label "project" which is added to all log entries      |
-| PROMTAIL_CLIENT_TENANT_ID    | string |                                          | An optional tenant id to sent as the `X-Scope-OrgID`-header         |
-| PROMTAIL_BASIC_AUTH_USERNAME | string |                                          | Username to use for basic auth, if required by Loki                 |
-| PROMTAIL_BASIC_AUTH_PASSWORD | string |                                          | Password to use for basic auth, if required by Loki                 |
-| PROMTAIL_SCRAPE_PATH         | string | /application/Data/Logs/*.log             | Path leading to log files to be scraped; supports glob syntax       |
+| Variable Name                | Type   | Default                                  | Description                                                                                                            |
+|------------------------------|--------|------------------------------------------|------------------------------------------------------------------------------------------------------------------------|
+| PROMTAIL_CLIENT_URL          | string | http://loki_loki_1:3100/loki/api/v1/push | URL pointing to the Loki push endpoint                                                                                 |
+| PROMTAIL_LABEL_HOST          | string | $(hostname)                              | Value of the label "host" which is added to all log entries                                                            |
+| PROMTAIL_LABELS_BASE64       | string |                                          | Additional labels to set. Must be a BASE64-encoded JSONs string. JSON example: `{"job": "my-job", "severity": "fine"}` |
+| PROMTAIL_CLIENT_TENANT_ID    | string |                                          | An optional tenant id to sent as the `X-Scope-OrgID`-header                                                            |
+| PROMTAIL_BASIC_AUTH_USERNAME | string |                                          | Username to use for basic auth, if required by Loki                                                                    |
+| PROMTAIL_BASIC_AUTH_PASSWORD | string |                                          | Password to use for basic auth, if required by Loki                                                                    |
+| PROMTAIL_SCRAPE_PATH         | string | /application/Data/Logs/*.log             | Path leading to log files to be scraped; supports glob syntax                                                          |
 
 ## Security aspects
 

root-files/build.sh

@@ -1,9 +1,13 @@
 #!/bin/bash
 
+. "${FLOWNATIVE_LIB_PATH}/packages.sh"
+
 set -o errexit
 set -o nounset
 set -o pipefail
 
+install_packages jq
+
 useradd --home-dir "${PROMTAIL_BASE_PATH}" --no-create-home --no-user-group --uid 1000 promtail
 groupadd --gid 1000 promtail
 
@@ -25,3 +29,7 @@ chown -R promtail:promtail \
 
 # We don't need logrotate in this image:
 rm -f /opt/flownative/supervisor/etc/conf.d/logrotate.conf
+
+rm -rf \
+    /var/cache/* \
+    /var/log/*

root-files/opt/flownative/lib/promtail.sh

@@ -23,21 +23,46 @@ promtail_env() {
 export PROMTAIL_BASE_PATH="${PROMTAIL_BASE_PATH}"
 export PROMTAIL_CONF_PATH="${PROMTAIL_BASE_PATH}/etc"
 export PROMTAIL_TMP_PATH="${PROMTAIL_BASE_PATH}/tmp"
-
 export PROMTAIL_CLIENT_URL="${PROMTAIL_CLIENT_URL:-http://loki:3100/loki/api/v1/push}"
-export PROMTAIL_LABEL_JOB="${PROMTAIL_LABEL_JOB:-promtail}"
-export PROMTAIL_LABEL_HOST="${PROMTAIL_LABEL_HOST:-$(hostname)}"
-export PROMTAIL_LABEL_ORGANIZATION="${PROMTAIL_LABEL_ORGANIZATION:-}"
-export PROMTAIL_LABEL_PROJECT="${PROMTAIL_LABEL_PROJECT:-}"
 export PROMTAIL_CLIENT_TENANT_ID="${PROMTAIL_CLIENT_TENANT_ID:-}"
 export PROMTAIL_BASIC_AUTH_USERNAME="${PROMTAIL_BASIC_AUTH_USERNAME:-}"
 export PROMTAIL_BASIC_AUTH_PASSWORD="${PROMTAIL_BASIC_AUTH_PASSWORD:-}"
 export PROMTAIL_SCRAPE_PATH=${PROMTAIL_SCRAPE_PATH:-/application/Data/Logs/*.log}
-
+export PROMTAIL_LABELS_BASE64=${PROMTAIL_LABELS_BASE64:-}
+export PROMTAIL_LABELS=${PROMTAIL_LABELS:-}
+export PROMTAIL_LABEL_HOST=${PROMTAIL_LABEL_HOST:-$(hostname)}
 export PATH="${PATH}:${PROMTAIL_BASE_PATH}/bin"
 EOF
 }
 
+# ---------------------------------------------------------------------------------------
+# promtail_render_config() - Render the configuration file for Promtail
+#
+# @global PROMTAIL_* The PROMTAIL_ environment variables
+#
+promtail_render_config() {
+    default_labels='{ "job": "promtail" }'
+
+    if [ -n "$PROMTAIL_LABELS_BASE64" ]; then
+      decoded_labels=$(echo "$PROMTAIL_LABELS_BASE64" | base64 --decode)
+    else
+      decoded_labels="$default_labels"
+    fi
+
+    # Convert JSON to YAML format for labels and add proper indentation
+    labels_yaml=$(echo "$decoded_labels" | jq -r 'to_entries | map("          \(.key): \(.value|@sh)") | .[]')
+
+    # Replace placeholders in the template configuration file using awk
+    awk -v labels="$labels_yaml" '
+    {
+      if ($0 ~ /__dynamic_labels__/) {
+        print labels
+      } else {
+        print
+      }
+    }' "${PROMTAIL_BASE_PATH}/etc/config-template.yaml" > "${PROMTAIL_BASE_PATH}/etc/config.yaml"
+}
+
 # ---------------------------------------------------------------------------------------
 # promtail_initialize() - Initialize Promtail configuration
 #
@@ -49,8 +74,8 @@ promtail_initialize() {
 
     info "Will scrape logs found at ${PROMTAIL_SCRAPE_PATH}"
 
-    path=$(dirname ${PROMTAIL_SCRAPE_PATH})
-    if [ ! -d  $path ]; then
+    path=$(dirname "${PROMTAIL_SCRAPE_PATH}")
+    if [ ! -d "$path" ]; then
       warn "$path does not exist"
     fi
 
@@ -62,4 +87,10 @@ promtail_initialize() {
     if [[ $PROMTAIL_BASIC_AUTH_USERNAME != "" && $PROMTAIL_BASIC_AUTH_PASSWORD == "" ]]; then
         warn "Authentication is configured to use username ${PROMTAIL_BASIC_AUTH_USERNAME} but no password was specified!"
     fi
+
+    info "Generating configuration file"
+    promtail_render_config
+
+    info "Checking configuration syntax"
+    "${PROMTAIL_BASE_PATH}/bin/promtail" -config.file=${PROMTAIL_BASE_PATH}/etc/config.yaml -check-syntax
 }

root-files/opt/flownative/promtail/etc/config.yaml → root-files/opt/flownative/promtail/etc/config-template.yaml

@@ -1,12 +1,12 @@
+# For configuration options see: https://grafana.com/docs/loki/latest/send-data/promtail/configuration/
+
 scrape_configs:
   - job_name: default
     static_configs:
       - targets:
           - localhost
         labels:
-          job: ${PROMTAIL_LABEL_JOB}
-          organization: ${PROMTAIL_LABEL_ORGANIZATION}
-          project: ${PROMTAIL_LABEL_PROJECT}
+          __dynamic_labels__
           host: ${PROMTAIL_LABEL_HOST}
           __path__: ${PROMTAIL_SCRAPE_PATH}
 
@@ -30,6 +30,3 @@ clients:
     basic_auth:
       username: ${PROMTAIL_BASIC_AUTH_USERNAME}
       password: ${PROMTAIL_BASIC_AUTH_PASSWORD}
-
-options:
-  stream_lag_labels: filename