diff --git a/Dockerfile b/Dockerfile index fff1d1c..a7ab242 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ -FROM grafana/promtail:2.6.1 as promtail -FROM europe-docker.pkg.dev/flownative/docker/base:buster -MAINTAINER Robert Lemke +FROM grafana/promtail:3.2.0 AS promtail +FROM europe-docker.pkg.dev/flownative/docker/base:bookworm +LABEL org.opencontainers.image.authors="Robert Lemke " # ----------------------------------------------------------------------------- # Promtail # Latest versions: https://github.com/grafana/loki/releases / https://hub.docker.com/r/grafana/promtail/tags -ENV PROMTAIL_VERSION=2.6.1 +ENV PROMTAIL_VERSION=3.2.0 ENV FLOWNATIVE_LIB_PATH=/opt/flownative/lib \ PROMTAIL_BASE_PATH=/opt/flownative/promtail \ diff --git a/README.md b/README.md index 33dccaa..0309d99 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,13 @@ # Flownative Promtail Image -A Docker image providing [Promtail](https://grafana.com/docs/loki/latest/clients/promtail/) for [Beach](https://www.flownative.com/beach), -[Local Beach](https://www.flownative.com/localbeach) and other purposes. Compared to the -official Promtail image, this one provides an opinionated configuration which can be -controlled through environment variables. +A Docker image +providing [Promtail](https://grafana.com/docs/loki/latest/clients/promtail/) +for [Beach](https://www.flownative.com/beach), +[Local Beach](https://www.flownative.com/localbeach) and other purposes. +Compared to the +official Promtail image, this one provides an opinionated configuration which +can be controlled through environment variables. ## Sending logs from Local Beach @@ -21,13 +24,13 @@ controlled through environment variables. - ./Data/Logs:/application/Data/Logs:delegated environment: - PROMTAIL_CLIENT_URL=http://loki_loki_1:3100/loki/api/v1/push - - PROMTAIL_LABEL_JOB=localbeach-flow + - PROMTAIL_LABELS=eyJqb2IiOiAibXktam9iIiwgInNldmVyaXR5IjogImZpbmUifQ== - PROMTAIL_LABEL_HOST=localbeach-${BEACH_PROJECT_NAME} ``` ## Testing with docker run -Create a directory "test" on your local machine. Within that directory +Create a directory "test" on your local machine. Within that directory create a file "test.log". Then run Promtail with `docker run`: @@ -38,26 +41,24 @@ docker run -v $(pwd)/logs:/logs \ -e PROMTAIL_CLIENT_URL=https://logs-prod-us-central1.grafana.net/loki/api/v1/push \ -e PROMTAIL_BASIC_AUTH_USERNAME=loki-username \ -e PROMTAIL_BASIC_AUTH_PASSWORD=loki-password \ - -e PROMTAIL_LABEL_JOB=test-job \ + -e PROMTAIL_LABELS=eyJqb2IiOiAibXktam9iIiwgInNldmVyaXR5IjogImZpbmUifQ== \ flownative/promtail ``` -In a separate terminal, add test content to the log file, for example with +In a separate terminal, add test content to the log file, for example with `echo "some test" >> logs/test.log`. ### Environment variables -| Variable Name | Type | Default | Description | -|------------------------------|--------|------------------------------------------|---------------------------------------------------------------------| -| PROMTAIL_CLIENT_URL | string | http://loki_loki_1:3100/loki/api/v1/push | URL pointing to the Loki push endpoint | -| PROMTAIL_LABEL_HOST | string | $(hostname) | Value of the label "host" which is added to all log entries | -| PROMTAIL_LABEL_JOB | string | promtail | Value of the label "job" which is added to all log entries | -| PROMTAIL_LABEL_ORGANIZATION | string | | Value of the label "organization" which is added to all log entries | -| PROMTAIL_LABEL_PROJECT | string | | Value of the label "project" which is added to all log entries | -| PROMTAIL_CLIENT_TENANT_ID | string | | An optional tenant id to sent as the `X-Scope-OrgID`-header | -| PROMTAIL_BASIC_AUTH_USERNAME | string | | Username to use for basic auth, if required by Loki | -| PROMTAIL_BASIC_AUTH_PASSWORD | string | | Password to use for basic auth, if required by Loki | -| PROMTAIL_SCRAPE_PATH | string | /application/Data/Logs/*.log | Path leading to log files to be scraped; supports glob syntax | +| Variable Name | Type | Default | Description | +|------------------------------|--------|------------------------------------------|------------------------------------------------------------------------------------------------------------------------| +| PROMTAIL_CLIENT_URL | string | http://loki_loki_1:3100/loki/api/v1/push | URL pointing to the Loki push endpoint | +| PROMTAIL_LABEL_HOST | string | $(hostname) | Value of the label "host" which is added to all log entries | +| PROMTAIL_LABELS_BASE64 | string | | Additional labels to set. Must be a BASE64-encoded JSONs string. JSON example: `{"job": "my-job", "severity": "fine"}` | +| PROMTAIL_CLIENT_TENANT_ID | string | | An optional tenant id to sent as the `X-Scope-OrgID`-header | +| PROMTAIL_BASIC_AUTH_USERNAME | string | | Username to use for basic auth, if required by Loki | +| PROMTAIL_BASIC_AUTH_PASSWORD | string | | Password to use for basic auth, if required by Loki | +| PROMTAIL_SCRAPE_PATH | string | /application/Data/Logs/*.log | Path leading to log files to be scraped; supports glob syntax | ## Security aspects diff --git a/root-files/build.sh b/root-files/build.sh index 295f635..d0aa8ed 100755 --- a/root-files/build.sh +++ b/root-files/build.sh @@ -1,9 +1,13 @@ #!/bin/bash +. "${FLOWNATIVE_LIB_PATH}/packages.sh" + set -o errexit set -o nounset set -o pipefail +install_packages jq + useradd --home-dir "${PROMTAIL_BASE_PATH}" --no-create-home --no-user-group --uid 1000 promtail groupadd --gid 1000 promtail @@ -25,3 +29,7 @@ chown -R promtail:promtail \ # We don't need logrotate in this image: rm -f /opt/flownative/supervisor/etc/conf.d/logrotate.conf + +rm -rf \ + /var/cache/* \ + /var/log/* diff --git a/root-files/opt/flownative/lib/promtail.sh b/root-files/opt/flownative/lib/promtail.sh index 4e5f278..cad10b2 100755 --- a/root-files/opt/flownative/lib/promtail.sh +++ b/root-files/opt/flownative/lib/promtail.sh @@ -23,21 +23,46 @@ promtail_env() { export PROMTAIL_BASE_PATH="${PROMTAIL_BASE_PATH}" export PROMTAIL_CONF_PATH="${PROMTAIL_BASE_PATH}/etc" export PROMTAIL_TMP_PATH="${PROMTAIL_BASE_PATH}/tmp" - export PROMTAIL_CLIENT_URL="${PROMTAIL_CLIENT_URL:-http://loki:3100/loki/api/v1/push}" -export PROMTAIL_LABEL_JOB="${PROMTAIL_LABEL_JOB:-promtail}" -export PROMTAIL_LABEL_HOST="${PROMTAIL_LABEL_HOST:-$(hostname)}" -export PROMTAIL_LABEL_ORGANIZATION="${PROMTAIL_LABEL_ORGANIZATION:-}" -export PROMTAIL_LABEL_PROJECT="${PROMTAIL_LABEL_PROJECT:-}" export PROMTAIL_CLIENT_TENANT_ID="${PROMTAIL_CLIENT_TENANT_ID:-}" export PROMTAIL_BASIC_AUTH_USERNAME="${PROMTAIL_BASIC_AUTH_USERNAME:-}" export PROMTAIL_BASIC_AUTH_PASSWORD="${PROMTAIL_BASIC_AUTH_PASSWORD:-}" export PROMTAIL_SCRAPE_PATH=${PROMTAIL_SCRAPE_PATH:-/application/Data/Logs/*.log} - +export PROMTAIL_LABELS_BASE64=${PROMTAIL_LABELS_BASE64:-} +export PROMTAIL_LABELS=${PROMTAIL_LABELS:-} +export PROMTAIL_LABEL_HOST=${PROMTAIL_LABEL_HOST:-$(hostname)} export PATH="${PATH}:${PROMTAIL_BASE_PATH}/bin" EOF } +# --------------------------------------------------------------------------------------- +# promtail_render_config() - Render the configuration file for Promtail +# +# @global PROMTAIL_* The PROMTAIL_ environment variables +# +promtail_render_config() { + default_labels='{ "job": "promtail" }' + + if [ -n "$PROMTAIL_LABELS_BASE64" ]; then + decoded_labels=$(echo "$PROMTAIL_LABELS_BASE64" | base64 --decode) + else + decoded_labels="$default_labels" + fi + + # Convert JSON to YAML format for labels and add proper indentation + labels_yaml=$(echo "$decoded_labels" | jq -r 'to_entries | map(" \(.key): \(.value|@sh)") | .[]') + + # Replace placeholders in the template configuration file using awk + awk -v labels="$labels_yaml" ' + { + if ($0 ~ /__dynamic_labels__/) { + print labels + } else { + print + } + }' "${PROMTAIL_BASE_PATH}/etc/config-template.yaml" > "${PROMTAIL_BASE_PATH}/etc/config.yaml" +} + # --------------------------------------------------------------------------------------- # promtail_initialize() - Initialize Promtail configuration # @@ -49,8 +74,8 @@ promtail_initialize() { info "Will scrape logs found at ${PROMTAIL_SCRAPE_PATH}" - path=$(dirname ${PROMTAIL_SCRAPE_PATH}) - if [ ! -d $path ]; then + path=$(dirname "${PROMTAIL_SCRAPE_PATH}") + if [ ! -d "$path" ]; then warn "$path does not exist" fi @@ -62,4 +87,10 @@ promtail_initialize() { if [[ $PROMTAIL_BASIC_AUTH_USERNAME != "" && $PROMTAIL_BASIC_AUTH_PASSWORD == "" ]]; then warn "Authentication is configured to use username ${PROMTAIL_BASIC_AUTH_USERNAME} but no password was specified!" fi + + info "Generating configuration file" + promtail_render_config + + info "Checking configuration syntax" + "${PROMTAIL_BASE_PATH}/bin/promtail" -config.file=${PROMTAIL_BASE_PATH}/etc/config.yaml -check-syntax } diff --git a/root-files/opt/flownative/promtail/etc/config.yaml b/root-files/opt/flownative/promtail/etc/config-template.yaml similarity index 78% rename from root-files/opt/flownative/promtail/etc/config.yaml rename to root-files/opt/flownative/promtail/etc/config-template.yaml index 990cd85..636d655 100644 --- a/root-files/opt/flownative/promtail/etc/config.yaml +++ b/root-files/opt/flownative/promtail/etc/config-template.yaml @@ -1,12 +1,12 @@ +# For configuration options see: https://grafana.com/docs/loki/latest/send-data/promtail/configuration/ + scrape_configs: - job_name: default static_configs: - targets: - localhost labels: - job: ${PROMTAIL_LABEL_JOB} - organization: ${PROMTAIL_LABEL_ORGANIZATION} - project: ${PROMTAIL_LABEL_PROJECT} + __dynamic_labels__ host: ${PROMTAIL_LABEL_HOST} __path__: ${PROMTAIL_SCRAPE_PATH} @@ -30,6 +30,3 @@ clients: basic_auth: username: ${PROMTAIL_BASIC_AUTH_USERNAME} password: ${PROMTAIL_BASIC_AUTH_PASSWORD} - -options: - stream_lag_labels: filename