From 3b74a7a0f9170243c563e2ca29738dcf2dc7b43f Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Sun, 23 Jun 2024 21:24:25 +0200 Subject: [PATCH 1/6] Remove obsolete database config option for test env --- config/test.exs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/config/test.exs b/config/test.exs index 20960f22f..791eaa75d 100644 --- a/config/test.exs +++ b/config/test.exs @@ -9,10 +9,6 @@ config :asciinema, Asciinema.Repo, pool: Ecto.Adapters.SQL.Sandbox, pool_size: 10 -if db_url = System.get_env("TEST_DATABASE_URL") do - System.put_env("DATABASE_URL", db_url) -end - # In test we don't send emails. config :asciinema, Asciinema.Emails.Mailer, adapter: Swoosh.Adapters.Test From 45e84c7ed5684e1361cdc79933427a5c6894eb9a Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Sun, 23 Jun 2024 21:28:16 +0200 Subject: [PATCH 2/6] Move public endpoint configuration to Application module --- config/runtime.exs | 34 ------------------ lib/asciinema/application.ex | 69 +++++++++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 35 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 26c3abf1d..9146c88f2 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -9,7 +9,6 @@ import Config env = &System.get_env/1 if env.("PHX_SERVER") do - config :asciinema, AsciinemaWeb.Endpoint, server: true config :asciinema, AsciinemaWeb.Admin.Endpoint, server: true end @@ -26,43 +25,10 @@ end if config_env() in [:prod, :dev] do if secret_key_base = env.("SECRET_KEY_BASE") do - config :asciinema, AsciinemaWeb.Endpoint, secret_key_base: secret_key_base config :asciinema, AsciinemaWeb.Admin.Endpoint, secret_key_base: secret_key_base config :asciinema, Asciinema.Accounts, secret: secret_key_base end - if port = env.("PORT") do - config :asciinema, AsciinemaWeb.Endpoint, http: [port: String.to_integer(port)] - end - - if url_scheme = env.("URL_SCHEME") do - config :asciinema, AsciinemaWeb.Endpoint, url: [scheme: url_scheme] - - case url_scheme do - "http" -> - config :asciinema, AsciinemaWeb.Endpoint, url: [port: 80] - - "https" -> - config :asciinema, AsciinemaWeb.Endpoint, url: [port: 443] - - _ -> - :ok - end - end - - if url_host = env.("URL_HOST") do - config :asciinema, AsciinemaWeb.Endpoint, url: [host: url_host] - end - - if url_path = env.("URL_PATH") do - config :asciinema, AsciinemaWeb.Endpoint, url: [path: url_path] - # this requires path prefix stripping at reverse proxy (nginx) level - end - - if url_port = env.("URL_PORT") do - config :asciinema, AsciinemaWeb.Endpoint, url: [port: String.to_integer(url_port)] - end - if env.("ADMIN_BIND_ALL") do config :asciinema, AsciinemaWeb.Admin.Endpoint, http: [ip: {0, 0, 0, 0}] end diff --git a/lib/asciinema/application.ex b/lib/asciinema/application.ex index e78cf34d0..074bbbba0 100644 --- a/lib/asciinema/application.ex +++ b/lib/asciinema/application.ex @@ -37,7 +37,7 @@ defmodule Asciinema.Application do # Start rate limiter {PlugAttack.Storage.Ets, name: AsciinemaWeb.PlugAttack.Storage, clean_period: 60_000}, # Start the public endpoint - AsciinemaWeb.Endpoint, + {AsciinemaWeb.Endpoint, endpoint_config()}, # Start the admin endpoint AsciinemaWeb.Admin.Endpoint ] @@ -59,4 +59,71 @@ defmodule Asciinema.Application do defp oban_config do Application.fetch_env!(:asciinema, Oban) end + + defp endpoint_config do + defaults = take_app_env(AsciinemaWeb.Endpoint) + + http = put_option([], :port, "PORT", &String.to_integer/1) + + url = + [] + |> put_option(:scheme, "URL_SCHEME") + |> put_option(:host, "URL_HOST") + |> put_option(:port, "URL_PORT", &String.to_integer/1) + |> put_option(:path, "URL_PATH") + + url = + case Keyword.get(url, :scheme) do + "http" -> Keyword.put_new(url, :port, 80) + "https" -> Keyword.put_new(url, :port, 443) + nil -> url + end + + overrides = + [] + |> put_option(:server, "PHX_SERVER", fn _ -> true end) + |> put_option(:secret_key_base, "SECRET_KEY_BASE") + |> Keyword.merge(http: http, url: url) + + config = deep_merge(defaults, overrides) + + if System.get_env("INSPECT_CONFIG") do + IO.inspect(config, label: "public endpoint config") + end + + config + end + + defp take_app_env(app \\ :asciinema, key) do + env = Application.get_env(app, key) + Application.delete_env(app, key) + Application.put_env(app, key, []) + + env + end + + defp put_option(opts, key, var, coerce \\ nil, default \\ :none) + + defp put_option(opts, key, var, nil, default), + do: put_option(opts, key, var, fn value -> value end, default) + + defp put_option(opts, key, var, coerce, default) do + case System.get_env(var) do + nil -> + case default do + :none -> opts + value -> Keyword.put(opts, key, value) + end + + value -> + Keyword.put(opts, key, coerce.(value)) + end + end + + def deep_merge(original, overrides) do + Keyword.merge(original, overrides, &on_conflict/3) + end + + defp on_conflict(_key, a, b) when is_list(a) and is_list(b), do: deep_merge(a, b) + defp on_conflict(_key, _a, b), do: b end From 8bd4d224ef075b0c956af53759a54d70dc1bf819 Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Sun, 23 Jun 2024 21:36:43 +0200 Subject: [PATCH 3/6] Move admin endpoint configuration to Application module --- config/runtime.exs | 25 ------------------------- lib/asciinema/application.ex | 35 ++++++++++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 9146c88f2..b5391de42 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -8,10 +8,6 @@ import Config env = &System.get_env/1 -if env.("PHX_SERVER") do - config :asciinema, AsciinemaWeb.Admin.Endpoint, server: true -end - if config_env() == :prod do database_url = System.get_env("DATABASE_URL") || @@ -25,30 +21,9 @@ end if config_env() in [:prod, :dev] do if secret_key_base = env.("SECRET_KEY_BASE") do - config :asciinema, AsciinemaWeb.Admin.Endpoint, secret_key_base: secret_key_base config :asciinema, Asciinema.Accounts, secret: secret_key_base end - if env.("ADMIN_BIND_ALL") do - config :asciinema, AsciinemaWeb.Admin.Endpoint, http: [ip: {0, 0, 0, 0}] - end - - if port = env.("ADMIN_PORT") do - config :asciinema, AsciinemaWeb.Admin.Endpoint, http: [port: String.to_integer(port)] - end - - if url_scheme = env.("ADMIN_URL_SCHEME") do - config :asciinema, AsciinemaWeb.Admin.Endpoint, url: [scheme: url_scheme] - end - - if url_host = env.("ADMIN_URL_HOST") do - config :asciinema, AsciinemaWeb.Admin.Endpoint, url: [host: url_host] - end - - if url_port = env.("ADMIN_URL_PORT") do - config :asciinema, AsciinemaWeb.Admin.Endpoint, url: [port: String.to_integer(url_port)] - end - if ip_limit = env.("IP_RATE_LIMIT") do config :asciinema, AsciinemaWeb.PlugAttack, ip_limit: String.to_integer(ip_limit), diff --git a/lib/asciinema/application.ex b/lib/asciinema/application.ex index 074bbbba0..da2a45cee 100644 --- a/lib/asciinema/application.ex +++ b/lib/asciinema/application.ex @@ -37,9 +37,9 @@ defmodule Asciinema.Application do # Start rate limiter {PlugAttack.Storage.Ets, name: AsciinemaWeb.PlugAttack.Storage, clean_period: 60_000}, # Start the public endpoint - {AsciinemaWeb.Endpoint, endpoint_config()}, + {AsciinemaWeb.Endpoint, public_endpoint_config()}, # Start the admin endpoint - AsciinemaWeb.Admin.Endpoint + {AsciinemaWeb.Admin.Endpoint, admin_endpoint_config()} ] # See https://hexdocs.pm/elixir/Supervisor.html @@ -60,7 +60,7 @@ defmodule Asciinema.Application do Application.fetch_env!(:asciinema, Oban) end - defp endpoint_config do + defp public_endpoint_config do defaults = take_app_env(AsciinemaWeb.Endpoint) http = put_option([], :port, "PORT", &String.to_integer/1) @@ -94,6 +94,35 @@ defmodule Asciinema.Application do config end + defp admin_endpoint_config do + defaults = take_app_env(AsciinemaWeb.Admin.Endpoint) + + http = + [] + |> put_option(:port, "ADMIN_PORT", &String.to_integer/1) + |> put_option(:ip, "ADMIN_BIND_ALL", fn _ -> {0, 0, 0, 0} end) + + url = + [] + |> put_option(:scheme, "ADMIN_URL_SCHEME") + |> put_option(:host, "ADMIN_URL_HOST") + |> put_option(:port, "ADMIN_URL_PORT", &String.to_integer/1) + + overrides = + [] + |> put_option(:server, "PHX_SERVER", fn _ -> true end) + |> put_option(:secret_key_base, "SECRET_KEY_BASE") + |> Keyword.merge(http: http, url: url) + + config = deep_merge(defaults, overrides) + + if System.get_env("INSPECT_CONFIG") do + IO.inspect(config, label: "admin endpoint config") + end + + config + end + defp take_app_env(app \\ :asciinema, key) do env = Application.get_env(app, key) Application.delete_env(app, key) From 14d2f92f6468a9c1750b8b8ff2b2ba47efdb8bb3 Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Sun, 23 Jun 2024 21:41:45 +0200 Subject: [PATCH 4/6] Move getting cluster config into a helper function --- lib/asciinema/application.ex | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/asciinema/application.ex b/lib/asciinema/application.ex index da2a45cee..0f4050a36 100644 --- a/lib/asciinema/application.ex +++ b/lib/asciinema/application.ex @@ -10,14 +10,12 @@ defmodule Asciinema.Application do :ok = Oban.Telemetry.attach_default_logger() :ok = Asciinema.ObanErrorReporter.configure() - topologies = Application.get_env(:libcluster, :topologies, []) - # List all child processes to be supervised children = [ # Start task supervisor {Task.Supervisor, name: Asciinema.TaskSupervisor}, # Start cluster supervisor - {Cluster.Supervisor, [topologies, [name: Asciinema.ClusterSupervisor]]}, + {Cluster.Supervisor, [cluster_topologies(), [name: Asciinema.ClusterSupervisor]]}, # Start the PubSub system {Phoenix.PubSub, [name: Asciinema.PubSub, adapter: Phoenix.PubSub.PG2]}, # Start live stream viewer tracker @@ -56,9 +54,9 @@ defmodule Asciinema.Application do :ok end - defp oban_config do - Application.fetch_env!(:asciinema, Oban) - end + defp cluster_topologies, do: Application.get_env(:libcluster, :topologies, []) + + defp oban_config, do: Application.fetch_env!(:asciinema, Oban) defp public_endpoint_config do defaults = take_app_env(AsciinemaWeb.Endpoint) From 2e57d978b6b19bb3824bf77ed20d6637e6706604 Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Sun, 23 Jun 2024 22:44:48 +0200 Subject: [PATCH 5/6] Move Oban's remaining runtime config to Application module --- config/runtime.exs | 4 ---- lib/asciinema/application.ex | 26 +++++++++++++++++++++++--- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index b5391de42..ce857abf6 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -145,10 +145,6 @@ if config_env() in [:prod, :dev] do config :asciinema, :unclaimed_recording_ttl, ttls end - if String.downcase("#{env.("CRON")}") in ["0", "false", "no"] do - config :asciinema, Oban, plugins: [{Oban.Plugins.Cron, crontab: []}] - end - if env.("SIGN_UP_DISABLED") in ["1", "true"] do config :asciinema, Asciinema.Accounts, sign_up_enabled?: false end diff --git a/lib/asciinema/application.ex b/lib/asciinema/application.ex index 0f4050a36..37a6004de 100644 --- a/lib/asciinema/application.ex +++ b/lib/asciinema/application.ex @@ -56,7 +56,22 @@ defmodule Asciinema.Application do defp cluster_topologies, do: Application.get_env(:libcluster, :topologies, []) - defp oban_config, do: Application.fetch_env!(:asciinema, Oban) + defp oban_config do + defaults = Application.fetch_env!(:asciinema, Oban) + + config = + if String.downcase("#{System.get_env("CRON")}") in ["0", "false"] do + deep_merge(defaults, plugins: [{Oban.Plugins.Cron, crontab: []}]) + else + defaults + end + + if System.get_env("INSPECT_CONFIG") do + IO.inspect(config, label: "oban config") + end + + config + end defp public_endpoint_config do defaults = take_app_env(AsciinemaWeb.Endpoint) @@ -151,6 +166,11 @@ defmodule Asciinema.Application do Keyword.merge(original, overrides, &on_conflict/3) end - defp on_conflict(_key, a, b) when is_list(a) and is_list(b), do: deep_merge(a, b) - defp on_conflict(_key, _a, b), do: b + defp on_conflict(_key, a, b) do + if Keyword.keyword?(a) and Keyword.keyword?(b) do + deep_merge(a, b) + else + b + end + end end From c8702193572a77bfe7a301d6df7e1593c8a33020 Mon Sep 17 00:00:00 2001 From: Marcin Kulik Date: Thu, 27 Jun 2024 09:21:45 +0200 Subject: [PATCH 6/6] wip --- config/dev.exs | 2 +- config/runtime.exs | 19 ------------------- lib/asciinema/application.ex | 5 ++++- lib/asciinema/repo.ex | 20 ++++++++++++++++++++ 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index 7a1c56d9c..4bd349c58 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -8,7 +8,7 @@ config :asciinema, Asciinema.Repo, database: "asciinema_development", stacktrace: true, show_sensitive_data_on_connection_error: true, - pool_size: 10 + pool_size: 11 secret_key_base = "60BnXnzGGwwiZj91YA9XYKF9BCiM7lQ/1um8VXcWWLSdUp9OcPZV6YnQv7eFTYSY" diff --git a/config/runtime.exs b/config/runtime.exs index ce857abf6..0a342289f 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -8,17 +8,6 @@ import Config env = &System.get_env/1 -if config_env() == :prod do - database_url = - System.get_env("DATABASE_URL") || - raise """ - environment variable DATABASE_URL is missing. - For example: ecto://USER:PASS@HOST/DATABASE - """ - - config :asciinema, Asciinema.Repo, url: database_url -end - if config_env() in [:prod, :dev] do if secret_key_base = env.("SECRET_KEY_BASE") do config :asciinema, Asciinema.Accounts, secret: secret_key_base @@ -74,14 +63,6 @@ if config_env() in [:prod, :dev] do end end - if db_pool_size = env.("DB_POOL_SIZE") do - config :asciinema, Asciinema.Repo, pool_size: String.to_integer(db_pool_size) - end - - if env.("ECTO_IPV6") in ~w(true 1) do - config :asciinema, Asciinema.Repo, socket_options: [:inet6] - end - if smtp_host = env.("SMTP_HOST") do config :asciinema, Asciinema.Emails.Mailer, adapter: Swoosh.Adapters.SMTP, diff --git a/lib/asciinema/application.ex b/lib/asciinema/application.ex index 37a6004de..4dd93d1ec 100644 --- a/lib/asciinema/application.ex +++ b/lib/asciinema/application.ex @@ -9,6 +9,9 @@ defmodule Asciinema.Application do def start(_type, _args) do :ok = Oban.Telemetry.attach_default_logger() :ok = Asciinema.ObanErrorReporter.configure() + # rr = take_app_env(Asciinema.Repo) + rr = Application.fetch_env!(:asciinema, Asciinema.Repo) + IO.inspect(rr, label: "rr") # List all child processes to be supervised children = [ @@ -23,7 +26,7 @@ defmodule Asciinema.Application do # Start telemetry reporters Asciinema.Telemetry, # Start the Ecto repository - Asciinema.Repo, + {Asciinema.Repo, rr}, # Start PNG generator poolboy pool :poolboy.child_spec(:worker, Asciinema.PngGenerator.Rsvg.poolboy_config(), []), # Start Oban diff --git a/lib/asciinema/repo.ex b/lib/asciinema/repo.ex index c1bc9b35d..594778969 100644 --- a/lib/asciinema/repo.ex +++ b/lib/asciinema/repo.ex @@ -5,6 +5,26 @@ defmodule Asciinema.Repo do use Scrivener, page_size: 10 + def init(_type, config) do + config = Keyword.put(config, :url, System.get_env("DATABASE_URL")) + + config = + if pool_size = System.get_env("DB_POOL_SIZE") do + Keyword.put(config, :pool_size, String.to_integer(pool_size)) + else + config + end + + config = + if System.get_env("ECTO_IPV6") in ~w(true 1) do + Keyword.put(config, :socket_options, [:inet6]) + else + config + end + + {:ok, config} + end + def transact(fun, opts \\ []) do transaction( fn ->