From 39b9ae9c7e304e10d1c2bb5386f6b3ad3b77d526 Mon Sep 17 00:00:00 2001 From: sviik Date: Tue, 7 Jan 2025 21:19:21 +0200 Subject: [PATCH 1/8] Bump Elixir/Erlang and fix warnings --- .github/workflows/ci.yml | 4 ++-- .tool-versions | 4 ++-- config/config.exs | 2 +- lib/logstash_logger_formatter.ex | 6 +++--- mix.exs | 2 +- test/logstash_logger_formatter_test.exs | 20 ++++++++++---------- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b32d25..7afac1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,8 @@ jobs: name: OTP ${{matrix.otp}} / Elixir ${{matrix.elixir}} strategy: matrix: - otp: ['23.2'] - elixir: ['1.11'] + otp: ['25.3'] + elixir: ['1.14'] steps: - uses: actions/checkout@v2 with: diff --git a/.tool-versions b/.tool-versions index ce1dc93..b0c6c87 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -elixir 1.11 -erlang 23.2 +elixir 1.14 +erlang 25.3 diff --git a/config/config.exs b/config/config.exs index b0bec42..40c4ad9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config config :logger, :logstash_formatter, diff --git a/lib/logstash_logger_formatter.ex b/lib/logstash_logger_formatter.ex index 54eaf3d..b1a49b1 100644 --- a/lib/logstash_logger_formatter.ex +++ b/lib/logstash_logger_formatter.ex @@ -20,7 +20,7 @@ defmodule LogstashLoggerFormatter do format: {LogstashLoggerFormatter, :format} """ - @config Application.get_env(:logger, :logstash_formatter, []) + @config Application.compile_env(:logger, :logstash_formatter, []) @engine Keyword.get(@config, :engine, Jason) @ts_field Keyword.get(@config, :timestamp_field, "@timestamp") @msg_field Keyword.get(@config, :message_field, "message") @@ -51,7 +51,7 @@ defmodule LogstashLoggerFormatter do event = apply(@engine, @encode_fn, [event]) - [event, '\n'] + [event, "\n"] end defp prepare_metadata(metadata) do @@ -215,7 +215,7 @@ defmodule LogstashLoggerFormatter do end defp format_timestamp({date, time}) do - to_string([@ts_formatter.format_date(date), 'T', @ts_formatter.format_time(time), '+00:00']) + to_string([@ts_formatter.format_date(date), "T", @ts_formatter.format_time(time), "+00:00"]) end defp add_level(event, level) do diff --git a/mix.exs b/mix.exs index 2a2ac42..7c4ab23 100644 --- a/mix.exs +++ b/mix.exs @@ -5,7 +5,7 @@ defmodule LogstashLoggerFormatter.Mixfile do [ app: :logstash_logger_formatter, version: "1.1.4", - elixir: "~> 1.11", + elixir: "~> 1.14", elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, build_embedded: Mix.env() == :prod, diff --git a/test/logstash_logger_formatter_test.exs b/test/logstash_logger_formatter_test.exs index 2539e23..f0ead7a 100644 --- a/test/logstash_logger_formatter_test.exs +++ b/test/logstash_logger_formatter_test.exs @@ -27,10 +27,10 @@ defmodule LogstashLoggerFormatterTest do }\"}, {\"host\", \"localhost:4112\"}, {\"user-agent\", \"curl/7.87.0\"}], request_path: \"/my_path\", resp_body: nil, resp_cookies: %{}, resp_headers: [{\"cache-control\", \"max-age=0, private, must-revalidate\"}, {\"access-control-allow-origin\", \"*\"}, {\"access-control-expose-headers\", \"\"}, {\"access-control-allow-credentials\", \"true\"}], scheme: :http, script_name: [], secret_key_base: nil, state: :unset, status: 200}, queues: []} }}}, []}", [ - '\n', + "\n", "Initial Call: ", ":cowboy_stream_h.request_process/3", - '\n', + "\n", "Ancestors: ", "[#PID<0.2.0>, #PID<0.1.0>]" ], @@ -49,7 +49,7 @@ defmodule LogstashLoggerFormatterTest do gl: "#PID<0.2.0>", initial_call: {:cowboy_stream_h, :request_process, 3}, line: 1, - logger_formatter: %{title: 'CRASH REPORT'}, + logger_formatter: %{title: "CRASH REPORT"}, module: :proc_lib, pid: "#PID<0.2.0>", report_cb: "&:proc_lib.report_cb/2", @@ -73,7 +73,7 @@ defmodule LogstashLoggerFormatterTest do message = capture_log(fn -> - Logger.warn( + Logger.warning( "Test message", application: :otp_app, extra_pid: pid, @@ -89,7 +89,7 @@ defmodule LogstashLoggerFormatterTest do assert decoded_message["application"] == "logstash_formatter" assert decoded_message["otp_application"] == "otp_app" assert decoded_message["@timestamp"] =~ ~r[\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+00:00] - assert decoded_message["level"] == "warn" + assert decoded_message["level"] == "warning" assert decoded_message["module"] == "Elixir.#{inspect(__MODULE__)}" assert decoded_message["function"] == "#{to_string(test_name)}/1" assert decoded_message["extra_pid"] == inspect(pid) @@ -109,7 +109,7 @@ defmodule LogstashLoggerFormatterTest do message = capture_log(fn -> - Logger.warn("Test message", datetime: datetime) + Logger.warning("Test message", datetime: datetime) end) decoded_message = Jason.decode!(message) @@ -123,7 +123,7 @@ defmodule LogstashLoggerFormatterTest do message = capture_log(fn -> - Logger.warn("Test message", datetime: struct) + Logger.warning("Test message", datetime: struct) end) decoded_message = Jason.decode!(message) @@ -136,7 +136,7 @@ defmodule LogstashLoggerFormatterTest do message = capture_log(fn -> - Logger.warn("Test message", foo: function) + Logger.warning("Test message", foo: function) end) decoded_message = Jason.decode!(message) @@ -190,7 +190,7 @@ defmodule LogstashLoggerFormatterTest do test "truncates metadata" do message = capture_log(fn -> - Logger.warn( + Logger.warning( "Test message", long_list: [ "some long string in it 1", @@ -375,7 +375,7 @@ defmodule LogstashLoggerFormatterTest do message = capture_log(fn -> - Logger.warn( + Logger.warning( invalid_utf8_bytes, key1: invalid_utf8_bytes, key2: {:a, invalid_utf8_bytes} From 8e4c73fa5511c2761de4b6e4e1ca2d1e8d1817a3 Mon Sep 17 00:00:00 2001 From: sviik Date: Wed, 8 Jan 2025 09:12:01 +0200 Subject: [PATCH 2/8] Bump ex_doc Removes deprecation warnings. --- mix.lock | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mix.lock b/mix.lock index fbeed35..af69b5f 100644 --- a/mix.lock +++ b/mix.lock @@ -1,8 +1,9 @@ %{ - "earmark_parser": {:hex, :earmark_parser, "1.4.10", "6603d7a603b9c18d3d20db69921527f82ef09990885ed7525003c7fe7dc86c56", [:mix], [], "hexpm", "8e2d5370b732385db2c9b22215c3f59c84ac7dda7ed7e544d7c459496ae519c0"}, - "ex_doc": {:hex, :ex_doc, "0.22.6", "0fb1e09a3e8b69af0ae94c8b4e4df36995d8c88d5ec7dbd35617929144b62c00", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "1e0aceda15faf71f1b0983165e6e7313be628a460e22a031e32913b98edbd638"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.42", "f23d856f41919f17cd06a493923a722d87a2d684f143a1e663c04a2b93100682", [:mix], [], "hexpm", "6915b6ca369b5f7346636a2f41c6a6d78b5af419d61a611079189233358b8b8b"}, + "ex_doc": {:hex, :ex_doc, "0.36.1", "4197d034f93e0b89ec79fac56e226107824adcce8d2dd0a26f5ed3a95efc36b1", [:mix], [{:earmark_parser, "~> 1.4.42", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "d7d26a7cf965dacadcd48f9fa7b5953d7d0cfa3b44fa7a65514427da44eafd89"}, "jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"}, - "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.15.0", "98312c9f0d3730fde4049985a1105da5155bfe5c11e47bdc7406d88e01e4219b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "75ffa34ab1056b7e24844c90bfc62aaf6f3a37a15faa76b07bc5eba27e4a8b4a"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, + "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, + "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.0.1", "c7f58c120b2b5aa5fd80d540a89fdf866ed42f1f3994e4fe189abebeab610839", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "8a89a1eeccc2d798d6ea15496a6e4870b75e014d1af514b1b71fa33134f57814"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, } From f1e1fe395e24aea9d5d4235a740afce87ed8f320 Mon Sep 17 00:00:00 2001 From: sviik Date: Thu, 9 Jan 2025 09:29:40 +0200 Subject: [PATCH 3/8] Log file metadata as a string After previous updates file metadata started appearing as a charlist instead of string. This caused the logger treating it as a list, so the output was a comma-separated list of integers. Fix by converting `:file` metadata value to a string in case it's a char list. --- lib/logstash_logger_formatter.ex | 11 +++++++++++ test/logstash_logger_formatter_test.exs | 16 +++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/logstash_logger_formatter.ex b/lib/logstash_logger_formatter.ex index b1a49b1..f3edecd 100644 --- a/lib/logstash_logger_formatter.ex +++ b/lib/logstash_logger_formatter.ex @@ -56,11 +56,22 @@ defmodule LogstashLoggerFormatter do defp prepare_metadata(metadata) do metadata + |> prepare_file() |> prepare_mfa() |> prepare_crash_report_meta() |> Map.new(fn {k, v} -> {metadata_key(k), format_metadata(v)} end) end + defp prepare_file(metadata) do + Keyword.update(metadata, :file, "", fn file -> + if is_list(file) && List.ascii_printable?(file) do + List.to_string(file) + else + file + end + end) + end + defp prepare_mfa(metadata) do # Elixir versions prior to 1.10-otp-22 include `module` and `function/arity` in metadata. # Since 1.10-otp22 metadata includes a `mfa` tuple. diff --git a/test/logstash_logger_formatter_test.exs b/test/logstash_logger_formatter_test.exs index f0ead7a..0ab46ff 100644 --- a/test/logstash_logger_formatter_test.exs +++ b/test/logstash_logger_formatter_test.exs @@ -44,7 +44,7 @@ defmodule LogstashLoggerFormatterTest do domain: [:otp, :sasl], erl_level: :error, error_logger: %{tag: :error_report, type: :crash_report}, - file: "proc_lib.erl", + file: ~c"proc_lib.erl", function: "crash_report/4", gl: "#PID<0.2.0>", initial_call: {:cowboy_stream_h, :request_process, 3}, @@ -143,6 +143,20 @@ defmodule LogstashLoggerFormatterTest do assert decoded_message["foo"] == "&:application_controller.format_log/1" end + test "logs file metadata as a string" do + log_event = + Jason.decode!( + LogstashLoggerFormatter.format( + :info, + "message", + @example_timestamp, + Keyword.new(%{file: ~c"proc_lib.erl"}) + ) + ) + + assert log_event["file"] == "proc_lib.erl" + end + test "logs unhandled structs" do message = capture_log(fn -> From 66d90a953053cf817fc68c5c2936c44441493c01 Mon Sep 17 00:00:00 2001 From: sviik Date: Tue, 7 Jan 2025 21:36:47 +0200 Subject: [PATCH 4/8] Run formatter It makes example test data less readable but there's no good way around it. --- .github/workflows/ci.yml | 2 +- test/logstash_logger_formatter_test.exs | 18 +++++------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7afac1f..87a7d08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ on: jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 name: OTP ${{matrix.otp}} / Elixir ${{matrix.elixir}} strategy: matrix: diff --git a/test/logstash_logger_formatter_test.exs b/test/logstash_logger_formatter_test.exs index 0ab46ff..d7ef088 100644 --- a/test/logstash_logger_formatter_test.exs +++ b/test/logstash_logger_formatter_test.exs @@ -13,19 +13,11 @@ defmodule LogstashLoggerFormatterTest do " terminating", [ 1, - "** (exit) {{{%SomeError{ - assigns: %{conn: %Plug.Conn{adapter: {Plug.Cowboy.Conn, :...}, assigns: %{layout: false, queues: []}, body_params: %{\"foo\": \"#{ - @secret_value_example - }\"}, cookies: %Plug.Conn.Unfetched{aspect: :cookies}, halted: false, host: \"localhost\", method: \"GET\", owner: #PID<0.1369.0>, params: %{\"foo\" => \"#{ - @secret_value_example - }\"}, path_info: [\"my_path\"], path_params: %{}, port: 4112, query_params: %{\"foo\" => \"#{ - @secret_value_example - }\"}, remote_ip: {127, 0, 0, 1}, req_cookies: %Plug.Conn.Unfetched{aspect: :cookies, foo: \"#{ - @secret_value_example - }\"}, req_headers: [{\"accept\", \"*/*\"}, {\"authorization\", \"Bearer #{ - @secret_value_example - }\"}, {\"host\", \"localhost:4112\"}, {\"user-agent\", \"curl/7.87.0\"}], request_path: \"/my_path\", resp_body: nil, resp_cookies: %{}, resp_headers: [{\"cache-control\", \"max-age=0, private, must-revalidate\"}, {\"access-control-allow-origin\", \"*\"}, {\"access-control-expose-headers\", \"\"}, {\"access-control-allow-credentials\", \"true\"}], scheme: :http, script_name: [], secret_key_base: nil, state: :unset, status: 200}, queues: []} - }}}, []}", + """ + ** (exit) {{{%SomeError{ + assigns: %{conn: %Plug.Conn{adapter: {Plug.Cowboy.Conn, :...}, assigns: %{layout: false, queues: []}, body_params: %{"foo": "#{@secret_value_example}"}, cookies: %Plug.Conn.Unfetched{aspect: :cookies}, halted: false, host: "localhost", method: "GET", owner: #PID<0.1369.0>, params: %{"foo" => "#{@secret_value_example}"}, path_info: ["my_path"], path_params: %{}, port: 4112, query_params: %{"foo" => "#{@secret_value_example}"}, remote_ip: {127, 0, 0, 1}, req_cookies: %Plug.Conn.Unfetched{aspect: :cookies, foo: "#{@secret_value_example}"}, req_headers: [{"accept", "*/*"}, {"authorization", "Bearer #{@secret_value_example}"}, {"host", "localhost:4112"}, {"user-agent", "curl/7.87.0"}], request_path: "/my_path", resp_body: nil, resp_cookies: %{}, resp_headers: [{"cache-control", "max-age=0, private, must-revalidate"}, {"access-control-allow-origin", "*"}, {"access-control-expose-headers", ""}, {"access-control-allow-credentials", "true"}], scheme: :http, script_name: [], secret_key_base: nil, state: :unset, status: 200}, queues: []} + }}}, []} + """, [ "\n", "Initial Call: ", From 901fd4ba1c6a0afd83d8a32fa6787d67db4bd150 Mon Sep 17 00:00:00 2001 From: sviik Date: Tue, 7 Jan 2025 21:22:27 +0200 Subject: [PATCH 5/8] Change GH workflow name `Ruby` did not make sense. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87a7d08..b00bda6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: Ruby +name: ci on: push: From 088f8074a29dd30a0a12951bcd2478256e2094a9 Mon Sep 17 00:00:00 2001 From: sviik Date: Tue, 7 Jan 2025 21:49:49 +0200 Subject: [PATCH 6/8] Bump actions/checkout to v4 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b00bda6..59c1188 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: otp: ['25.3'] elixir: ['1.14'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: true - uses: erlef/setup-beam@v1 From 9e7695eeac2afe0286a611de47f2410e3d7da396 Mon Sep 17 00:00:00 2001 From: sviik Date: Tue, 7 Jan 2025 21:53:54 +0200 Subject: [PATCH 7/8] Bump version to 1.1.5 --- CHANGELOG.md | 4 ++++ mix.exs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f2eead..f9f8b4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 1.1.5 + +* Add support for Elixir 1.14 with OTP 25 + ## 1.1.4 - Allow crash reports to be ingested by ElasticSearch diff --git a/mix.exs b/mix.exs index 7c4ab23..e17388f 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule LogstashLoggerFormatter.Mixfile do def project do [ app: :logstash_logger_formatter, - version: "1.1.4", + version: "1.1.5", elixir: "~> 1.14", elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, From 13c2228c16dcae3aa1fe3ea2ba66b672e463f1ea Mon Sep 17 00:00:00 2001 From: sviik Date: Tue, 7 Jan 2025 21:55:16 +0200 Subject: [PATCH 8/8] Update version number in installation doc The latest version has been 1.x for a while now. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 429d155..de68146 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ list of dependencies in `mix.exs`: ```elixir def deps do [ - {:logstash_logger_formatter, "~> 0.1.0"} + {:logstash_logger_formatter, "~> 1.1.5"} ] end ```