PA message table migration and postgres configuration (#321)

* Added modules needed to keep live alerts in memory.

* Added new processes to supervision tree.

* Added store test module.

* Made fetcher more testable.

* Added Fetcher test module.

* Added moduledocs.

* Credo.

* Consolidated logic so fetching and storing happens in the same module.

* Removed unnecessary pattern match.

* Changed reduce to map.

* Fixed AuthManager so users can have more than one role.

* Changed attribute name to be more specific.

* Added new role for PA Message workflow.

* Renamed a role so it better represents how it's used.

* Added a plug for verifying permission.

* Added scope with an auth check for PA message creation.

* Added role to fake Keycloak config.

* Added new role to metadata.

* Renamed modules and files to better describe functionality.

* Changed references of admin to emergency admin.

* Simplified how roles are mapped to access levels.

* Removed explicit readonly permission. Can be implied.

* Added roles list to metadata to cut down on function calls.

* Fixed tests now that roles are in assigns.

* Added test for new plug.

* Use Plug instead of manual adding assigns.

* Added a doc for the endpoint RTS will use to retrieve active messages.

* add migration and dependencies

* Add migration and database configuration

* Add documentation for postgres

* Add migrations process

* Update config/test.exs

Co-authored-by: Christian Maddox <[email protected]>

* remove async migrations method

* Add postgres service to CI

* formatting

* fetch cert

* move cert file to priv

* fix typo

---------

Co-authored-by: cmaddox5 <[email protected]>

.envrc.template

@@ -15,3 +15,9 @@
 # export SCREENS_URL=
 # export SIGNS_UI_URL=
 # export ALERTS_UI_URL=
+
+## Postgres configuration: username, password, and hostname
+## * Your local Postgres server should go here
+# export DATABASE_USER=
+# export DATABASE_PASSWORD=
+# export DATABASE_HOSTNAME=

.github/workflows/ci.yml

@@ -28,8 +28,22 @@ jobs:
     name: Build and test
     runs-on: ubuntu-latest
     env:
+      DATABASE_PASSWORD: postgres
+      DATABASE_USER: postgres
+      DATABASE_NAME: screenplay_test
+      DATABASE_HOST: localhost
       GUARDIAN_SECRET_KEY: test_auth_secret
       SECRET_KEY_BASE: local_secret_key_base_at_least_64_bytes_________________________________
+    services:
+      postgres:
+        image: postgres
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_PASSWORD: ${{env.DATABASE_PASSWORD}}
+          POSTGRES_USER: ${{env.DATABASE_USER}}
+          POSTGRES_DB: ${{env.DATABASE_NAME}}
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     needs: asdf
     steps:
       - uses: actions/checkout@v2

Dockerfile

@@ -30,8 +30,14 @@ FROM elixir-builder as app-builder
 
 ENV LANG="C.UTF-8" MIX_ENV="prod"
 
+RUN apk add --no-cache --update curl
+
 WORKDIR /root
 
+RUN curl https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \
+    -o aws-cert-bundle.pem
+RUN echo "51b107da46717aed974d97464b63f7357b220fe8737969db1492d1cae74b3947  aws-cert-bundle.pem" | sha256sum -c -
+
 # add frontend assets compiled in node container, required by phx.digest
 COPY --from=assets-builder /root/priv/static ./priv/static
 
@@ -57,5 +63,7 @@ COPY --from=app-builder /root/priv/static ./priv/static
 # add application artifact comipled in app build container
 COPY --from=app-builder /root/_build/prod/rel/screenplay .
 
+COPY --from=app-builder --chown=screenplay:screenplay /root/aws-cert-bundle.pem ./priv/aws-cert-bundle.pem
+
 # run the application
 CMD ["bin/screenplay", "start"]

README.md

@@ -2,6 +2,10 @@
 
 This tool enables PIOs to upload urgent messages to the Outfront signs in and outside stations.
 
+## Prerequisites
+
+Screenplay requires Postgres. If you don't already have Postgres installed, and you're on a Mac, [Postgres.app](https://postgresapp.com/downloads.html) is an easy way to get started. However, any Postgres instance to which you can connect and in which you have sufficient privileges should work.
+
 ## Development
 
 To start your Phoenix server:

config/config.exs

@@ -7,6 +7,17 @@
 # General application configuration
 import Config
 
+config :screenplay, Screenplay.Repo,
+  database: "screenplay_dev",
+  username: System.get_env("DATABASE_USER", ""),
+  password: System.get_env("DATABASE_PASSWORD", ""),
+  hostname: System.get_env("DATABASE_HOST", "localhost"),
+  port: System.get_env("DATABASE_PORT", "5432") |> String.to_integer(),
+  show_sensitive_data_on_connection_error: true,
+  backoff_min: 5_000
+
+config :screenplay, ecto_repos: [Screenplay.Repo]
+
 # Configures the endpoint
 config :screenplay, ScreenplayWeb.Endpoint,
   url: [host: "localhost"],

config/prod.exs

@@ -31,6 +31,12 @@ config :ueberauth, Ueberauth,
       {Ueberauth.Strategy.Oidcc, userinfo: true, uid_field: "email", scopes: ~w(openid email)}
   ]
 
+config :screenplay, Screenplay.Repo,
+  database: "screenplay",
+  ssl: true,
+  show_sensitive_data_on_connection_error: false,
+  configure: {Screenplay.Repo, :add_prod_credentials, []}
+
 # ## SSL Support
 #
 # To get SSL working, you will need to add the `https` key

config/runtime.exs

@@ -64,3 +64,5 @@ scheduler_jobs =
     else: []
 
 config :screenplay, Screenplay.Scheduler, jobs: scheduler_jobs
+
+config :screenplay, Screenplay.Repo, pool_size: 10

config/test.exs

@@ -28,5 +28,10 @@ config :ueberauth_oidcc,
     ]
   ]
 
+config :screenplay, Screenplay.Repo,
+  adapter: Ecto.Adapters.Postgres,
+  database: "screenplay_test",
+  pool: Ecto.Adapters.SQL.Sandbox
+
 # Print only warnings and errors during test
 config :logger, level: :warning

docs/tech_specs/pa_message_api.md

@@ -0,0 +1,54 @@
+# List Active Messages
+
+Lists all PA messages that are currently eligible to play.
+
+**URL** : `/api/active_pa_messages`
+
+**Method** : `GET`
+
+**Parameters**: None
+
+**API key required** : YES
+
+## Success Responses
+
+**Code** : `200 OK`
+
+**Response**:
+
+```json
+[
+  {
+    "id": 1,
+    "sign_ids": ["sign_1", "sign2"],
+    "priority": 0,
+    "interval_in_minutes": 4,
+    "visual_text": "This message will be played.",
+    "audio_text": "This message will be played."
+  },
+  {
+    "id": 2,
+    "sign_ids": ["sign_3", "sign4"],
+    "priority": 0,
+    "interval_in_minutes": 3,
+    "visual_text": "This message will be played.",
+    "audio_text": "This message will be played."
+  },
+  {
+    "id": 3,
+    "sign_ids": ["sign_1"],
+    "priority": 0,
+    "interval_in_minutes": 2,
+    "visual_text": "This message will be played.",
+    "audio_text": "This message will be played."
+  }
+]
+```
+
+## Failure Responses
+
+**Code** : `403 Forbidden`
+
+**Response**:
+
+`Invalid API key`

lib/screenplay/application.ex

@@ -16,7 +16,9 @@ defmodule Screenplay.Application do
         ScreenplayWeb.Endpoint,
         Screenplay.OutfrontTakeoverTool.Alerts.State,
         Screenplay.OutfrontTakeoverTool.Alerts.Reminders,
-        Screenplay.Scheduler
+        Screenplay.Scheduler,
+        Screenplay.Repo,
+        Screenplay.Migrate
       ] ++
         if Application.get_env(:screenplay, :start_alerts_cache) do
           [Screenplay.Alerts.Cache]

lib/screenplay/migrate.ex

@@ -0,0 +1,32 @@
+defmodule Screenplay.Migrate do
+  @moduledoc """
+  GenServer which runs on startup to run Ecto migrations. All migrations
+  stored in the "migrations" directory are run during init. Migrations stored
+  in the "async_migrations" directory will be run after the regular migrations
+  complete and will only log a warning on failure.
+  """
+  use GenServer, restart: :transient
+  require Logger
+
+  def start_link(opts) do
+    GenServer.start_link(__MODULE__, opts)
+  end
+
+  @impl GenServer
+  def init(opts) do
+    Logger.info("#{__MODULE__} synchronous migrations starting")
+    Keyword.get(opts, :sync_migrate_fn, &default_migrate_fn/1).("migrations")
+
+    Logger.info("#{__MODULE__} synchronous migrations finished")
+    {:ok, opts}
+  end
+
+  defp default_migrate_fn(migration_directory) do
+    Ecto.Migrator.run(
+      Screenplay.Repo,
+      Ecto.Migrator.migrations_path(Screenplay.Repo, migration_directory),
+      :up,
+      all: true
+    )
+  end
+end

lib/screenplay/repo.ex

@@ -0,0 +1,43 @@
+defmodule Screenplay.Repo do
+  require Logger
+
+  use Ecto.Repo,
+    otp_app: :screenplay,
+    adapter: Ecto.Adapters.Postgres
+
+  def add_prod_credentials(config, auth_token_fn \\ &ExAws.RDS.generate_db_auth_token/4) do
+    host = System.get_env("DATABASE_HOST")
+    port = String.to_integer(System.get_env("DATABASE_PORT", "5432"))
+    user = System.get_env("DATABASE_USER")
+
+    token =
+      auth_token_fn.(
+        host,
+        user,
+        port,
+        %{}
+      )
+
+    if is_nil(token) do
+      Logger.info("#{__MODULE__} add_prod_credentials token_is_nil")
+    else
+      hash_string = Base.encode16(:crypto.hash(:sha3_256, token))
+
+      Logger.info("#{__MODULE__} add_prod_credentials token_hash=#{hash_string}")
+    end
+
+    Keyword.merge(config,
+      hostname: host,
+      username: user,
+      port: port,
+      password: token,
+      ssl_opts: [
+        cacertfile: "priv/aws-cert-bundle.pem",
+        verify: :verify_peer,
+        server_name_indication: String.to_charlist(host),
+        verify_fun:
+          {&:ssl_verify_hostname.verify_fun/3, [check_hostname: String.to_charlist(host)]}
+      ]
+    )
+  end
+end

mix.exs

@@ -53,11 +53,14 @@ defmodule Screenplay.MixProject do
       {:sftp_client, "~> 2.0"},
       {:ex_aws, "~> 2.5"},
       {:ex_aws_s3, "~> 2.5"},
+      {:ex_aws_rds, "~> 2.0.2"},
       {:httpoison, "~> 2.2.1"},
       {:lcov_ex, "~> 0.2", only: [:dev, :test], runtime: false},
       {:sobelow, "~> 0.8", only: :dev},
       {:sentry, "~> 10.3"},
       {:stream_data, "~> 0.5", only: :test},
+      {:ecto_sql, "~> 3.0"},
+      {:postgrex, ">= 0.0.0"},
       {:quantum, "~> 3.0"}
     ]
   end

mix.lock

@@ -7,9 +7,14 @@
   "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"},
   "credo": {:hex, :credo, "1.7.5", "643213503b1c766ec0496d828c90c424471ea54da77c8a168c725686377b9545", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "f799e9b5cd1891577d8c773d245668aa74a2fcd15eb277f51a0131690ebfb3fd"},
   "crontab": {:hex, :crontab, "1.1.13", "3bad04f050b9f7f1c237809e42223999c150656a6b2afbbfef597d56df2144c5", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "d67441bec989640e3afb94e123f45a2bc42d76e02988c9613885dc3d01cf7085"},
+  "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"},
+  "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
   "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"},
+  "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"},
+  "ecto_sql": {:hex, :ecto_sql, "3.11.1", "e9abf28ae27ef3916b43545f9578b4750956ccea444853606472089e7d169470", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ce14063ab3514424276e7e360108ad6c2308f6d88164a076aac8a387e1fea634"},
   "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
   "ex_aws": {:hex, :ex_aws, "2.5.1", "7418917974ea42e9e84b25e88b9f3d21a861d5f953ad453e212f48e593d8d39f", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1b95431f70c446fa1871f0eb9b183043c5a625f75f9948a42d25f43ae2eff12b"},
+  "ex_aws_rds": {:hex, :ex_aws_rds, "2.0.2", "38dd8e83d57cf4b7286c4f6f5c978f700c40c207ffcdd6ca5d738e5eba933f9a", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}], "hexpm", "9e5b5cc168077874cbd0d29ba65d01caf1877e705fb5cecacf0667dd19bfa75c"},
   "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.3", "422468e5c3e1a4da5298e66c3468b465cfd354b842e512cb1f6fbbe4e2f5bdaf", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "4f09dd372cc386550e484808c5ac5027766c8d0cd8271ccc578b82ee6ef4f3b8"},
   "expo": {:hex, :expo, "0.5.1", "249e826a897cac48f591deba863b26c16682b43711dd15ee86b92f25eafd96d9", [:mix], [], "hexpm", "68a4233b0658a3d12ee00d27d37d856b1ba48607e7ce20fd376958d0ba6ce92b"},
   "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"},
@@ -40,6 +45,7 @@
   "plug": {:hex, :plug, "1.15.3", "712976f504418f6dff0a3e554c40d705a9bcf89a7ccef92fc6a5ef8f16a30a97", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cc4365a3c010a56af402e0809208873d113e9c38c401cabd88027ef4f5c01fd2"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.7.1", "87677ffe3b765bc96a89be7960f81703223fe2e21efa42c125fcd0127dd9d6b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "02dbd5f9ab571b864ae39418db7811618506256f6d13b4a45037e5fe78dc5de3"},
   "plug_crypto": {:hex, :plug_crypto, "2.0.0", "77515cc10af06645abbfb5e6ad7a3e9714f805ae118fa1a70205f80d2d70fe73", [:mix], [], "hexpm", "53695bae57cc4e54566d993eb01074e4d894b65a3766f1c43e2c61a1b0f45ea9"},
+  "postgrex": {:hex, :postgrex, "0.17.5", "0483d054938a8dc069b21bdd636bf56c487404c241ce6c319c1f43588246b281", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "50b8b11afbb2c4095a3ba675b4f055c416d0f3d7de6633a595fc131a828a67eb"},
   "quantum": {:hex, :quantum, "3.5.3", "ee38838a07761663468145f489ad93e16a79440bebd7c0f90dc1ec9850776d99", [:mix], [{:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.14 or ~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_registry, "~> 0.2", [hex: :telemetry_registry, repo: "hexpm", optional: false]}], "hexpm", "500fd3fa77dcd723ed9f766d4a175b684919ff7b6b8cfd9d7d0564d58eba8734"},
   "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
   "sentry": {:hex, :sentry, "10.3.0", "4b7543dfea5e59f3be6db28a032427884d55fbc828173b23115064e75dcb1eed", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_ownership, "~> 0.3.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "c1c08ba57f0634b7fda92adb0818ea0677e043e2d28ea4464351a0e4e8e142e5"},

priv/repo/migrations/20240422170503_create_pa_message_table.exs

@@ -0,0 +1,25 @@
+defmodule Screenplay.Repo.Migrations.CreatePaMessageTable do
+  use Ecto.Migration
+
+  def change do
+    create table("pa_message") do
+      add :alert_id, :string
+      add :start_time, :utc_datetime
+      add :end_time, :utc_datetime
+      add :days_of_week, {:array, :string}
+      add :sign_ids, {:array, :string}, null: false
+      add :priority, :integer, null: false
+      add :interval_in_minutes, :integer, null: false
+      add :visual_text, :text, null: false
+      add :audio_text, :text, null: false
+      add :paused, :boolean
+      add :saved, :boolean
+      add :message_type, :string
+
+      timestamps(type: :utc_datetime)
+    end
+
+    create index("pa_message", [:start_time, :end_time])
+    create index("pa_message", ["(to_tsvector('english', visual_text))"], using: "GIN")
+  end
+end