Pull out media server and handling into separate container

.dockerignore

@@ -4,3 +4,4 @@ _build
 deps
 .elixir_ls
 priv
+native/philomena/target

config/runtime.exs

@@ -27,6 +27,7 @@ config :philomena,
   image_url_root: System.fetch_env!("IMAGE_URL_ROOT"),
   badge_url_root: System.fetch_env!("BADGE_URL_ROOT"),
   mailer_address: System.fetch_env!("MAILER_ADDRESS"),
+  mediaproc_addr: System.fetch_env!("MEDIAPROC_ADDR"),
   tag_file_root: System.fetch_env!("TAG_FILE_ROOT"),
   site_domains: System.fetch_env!("SITE_DOMAINS"),
   tag_url_root: System.fetch_env!("TAG_URL_ROOT"),

docker-compose.yml

@@ -30,6 +30,7 @@ services:
       - IMAGE_URL_ROOT=/img
       - BADGE_URL_ROOT=/badge-img
       - TAG_URL_ROOT=/tag-img
+      - MEDIAPROC_ADDR=mediaproc:1500
       - OPENSEARCH_URL=http://opensearch:9200
       - REDIS_HOST=valkey
       - DATABASE_URL=ecto://postgres:postgres@postgres/philomena_dev
@@ -52,6 +53,7 @@ services:
       - app_deps_data:/srv/philomena/deps
       - app_native_data:/srv/philomena/priv/native
     depends_on:
+      - mediaproc
       - postgres
       - opensearch
       - valkey
@@ -89,6 +91,18 @@ services:
       - .:/srv/philomena
     attach: false
 
+  mediaproc:
+    build:
+      context: .
+      dockerfile: ./docker/mediaproc/Dockerfile
+    attach: false
+    deploy:
+      resources:
+        limits:
+          cpus: '4'
+          memory: 8gb
+          pids: 8192
+
   web:
     build:
       context: .

docker/app/Dockerfile

@@ -1,28 +1,12 @@
 FROM elixir:1.18.1-alpine
 
-ADD https://api.github.com/repos/philomena-dev/FFmpeg/git/refs/heads/release/6.1 /tmp/ffmpeg_version.json
-RUN (echo "https://github.com/philomena-dev/prebuilt-ffmpeg/raw/master"; cat /etc/apk/repositories) > /tmp/repositories \
-    && cp /tmp/repositories /etc/apk/repositories \
-    && apk update --allow-untrusted \
-    && apk add inotify-tools build-base git ffmpeg ffmpeg-dev npm nodejs file-dev libjpeg-turbo-dev libpng-dev gifsicle optipng libjpeg-turbo-utils librsvg rsvg-convert imagemagick postgresql16-client wget rust cargo --allow-untrusted \
+RUN apk add inotify-tools build-base git npm nodejs postgresql16-client wget rust cargo \
     && mix local.hex --force \
     && mix local.rebar --force
 
-ADD https://api.github.com/repos/philomena-dev/cli_intensities/git/refs/heads/master /tmp/cli_intensities_version.json
-RUN git clone --depth 1 https://github.com/philomena-dev/cli_intensities /tmp/cli_intensities \
-    && cd /tmp/cli_intensities \
-    && make -j$(nproc) install
-
-ADD https://api.github.com/repos/philomena-dev/mediatools/git/refs/heads/master /tmp/mediatools_version.json
-RUN git clone --depth 1 https://github.com/philomena-dev/mediatools /tmp/mediatools \
-    && ln -s /usr/lib/librsvg-2.so.2 /usr/lib/librsvg-2.so \
-    && cd /tmp/mediatools \
-    && make -j$(nproc) install
-
 COPY docker/app/run-development /usr/local/bin/run-development
 COPY docker/app/run-test /usr/local/bin/run-test
-COPY docker/app/safe-rsvg-convert /usr/local/bin/safe-rsvg-convert
 COPY docker/app/purge-cache /usr/local/bin/purge-cache
 ENV PATH=$PATH:/root/.cargo/bin
 EXPOSE 5173
-CMD run-development
+CMD ["/usr/local/bin/run-development"]

docker/mediaproc/Dockerfile

@@ -0,0 +1,75 @@
+FROM rust:1.83-slim
+
+RUN apt update \
+    && apt install -y build-essential git libmagic-dev libturbojpeg0-dev libpng-dev \
+       gifsicle optipng libjpeg-turbo-progs librsvg2-bin librsvg2-dev file imagemagick \
+       libx264-dev libx265-dev libvpx-dev libdav1d-dev libaom-dev libopus-dev \
+       libmp3lame-dev libvorbis-dev libwebp-dev libjxl-dev yasm wget
+
+ADD https://api.github.com/repos/philomena-dev/FFmpeg/git/refs/heads/release/7.1 /tmp/ffmpeg_version.json
+ADD https://api.github.com/repos/philomena-dev/cli_intensities/git/refs/heads/master /tmp/cli_intensities_version.json
+ADD https://api.github.com/repos/philomena-dev/mediatools/git/refs/heads/master /tmp/mediatools_version.json
+
+RUN wget -qO /tmp/FFmpeg.tar.gz https://github.com/philomena-dev/FFmpeg/archive/refs/heads/release/7.1.tar.gz \
+    && wget -qO /tmp/cli_intensities.tar.gz https://github.com/philomena-dev/cli_intensities/archive/refs/heads/master.tar.gz \
+    && wget -qO /tmp/mediatools.tar.gz https://github.com/philomena-dev/mediatools/archive/refs/heads/master.tar.gz
+
+RUN cd /tmp \
+    && tar -xf FFmpeg.tar.gz \
+    && tar -xf cli_intensities.tar.gz \
+    && tar -xf mediatools.tar.gz \
+    && cd /tmp/FFmpeg-release-7.1 \
+    && ./configure \
+        --prefix=/usr \
+        --disable-everything \
+        --disable-stripping \
+        --disable-static \
+        --disable-ffplay \
+        --disable-doc \
+        --disable-htmlpages \
+        --disable-manpages \
+        --disable-podpages \
+        --disable-txtpages \
+        --disable-protocols \
+        --enable-shared \
+        --enable-pic \
+        --enable-pthreads \
+        --enable-gpl \
+        --enable-avfilter \
+        --enable-bsf=extract_extradata \
+        --enable-decoder=aac,apng,av1,gif,h264,hevc,jpeg2000,jpegxl,libaom-av1,libdav1d,libvorbis,libvpx_vp8,libvpx_vp9,mp3,mjpeg,opus,png,vorbis,vp8,vp9,webvtt \
+        --enable-demuxer=apng,gif,image2,image_gif_pipe,image_jpeg_pipe,image_png_pipe,image_webp_pipe,matroska,mjpeg,mjpeg_2000,mov,webm \
+        --enable-encoder=aac,apng,gif,jpegxl,libmp3lame,libaom-av1,libvorbis,libopus,libvpx_vp8,libvpx_vp9,libx265,libx264,opus,mjpeg,png,vorbis,webvtt \
+        --enable-filter=concat,palettegen,paletteuse,scale,setpts,setsar,settb,split,trim \
+        --enable-libaom \
+        --enable-libjxl \
+        --enable-libdav1d \
+        --enable-libopus \
+        --enable-libmp3lame \
+        --enable-libvpx \
+        --enable-libvorbis \
+        --enable-libx264 \
+        --enable-libx265 \
+        --enable-libwebp \
+        --enable-muxer=apng,image2,gif,matroska,mp4,webp,webm \
+        --enable-parser=aac,gif,h264,hevc,jpeg2000,jpegxl,mjpeg,opus,png,vorbis,vp8,vp9,webp \
+        --enable-protocol=concat,data,file,subfile \
+    && make -j$(nproc) install \
+    && cd /tmp/cli_intensities-master \
+    && make -j$(nproc) install \
+    && cd /tmp/mediatools-master \
+    && make -j$(nproc) install
+
+COPY native/philomena /tmp/philomena
+COPY docker/mediaproc/safe-rsvg-convert /usr/bin/safe-rsvg-convert
+
+RUN cd /tmp/philomena \
+    && cargo build --release -p mediaproc_server \
+    && cp target/release/mediaproc_server /usr/bin/mediaproc_server
+
+# Set up unprivileged user account
+RUN useradd -ms /bin/bash mediaproc
+USER mediaproc
+WORKDIR /home/mediaproc
+ENV RUST_LOG=trace
+CMD ["/usr/bin/mediaproc_server", "0.0.0.0:1500"]

lib/philomena/native.ex

@@ -12,6 +12,10 @@ defmodule Philomena.Native do
   @spec camo_image_url(String.t()) :: String.t()
   def camo_image_url(_uri), do: :erlang.nif_error(:nif_not_loaded)
 
+  @spec async_process_command(String.t(), String.t(), [String.t()]) :: :ok
+  def async_process_command(_server_addr, _program, _arguments),
+    do: :erlang.nif_error(:nif_not_loaded)
+
   @spec zip_open_writer(Path.t()) :: {:ok, reference()} | {:error, atom()}
   def zip_open_writer(_path), do: :erlang.nif_error(:nif_not_loaded)
 

lib/philomena_media/analyzers/gif.ex

@@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Gif do
 
   alias PhilomenaMedia.Analyzers.Analyzer
   alias PhilomenaMedia.Analyzers.Result
+  alias PhilomenaMedia.Remote
 
   @behaviour Analyzer
 
@@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Gif do
   end
 
   defp stats(file) do
-    case System.cmd("mediastat", [file]) do
+    case Remote.cmd("mediastat", [file]) do
       {output, 0} ->
         [_size, frames, width, height, num, den] =
           output

lib/philomena_media/analyzers/jpeg.ex

@@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Jpeg do
 
   alias PhilomenaMedia.Analyzers.Analyzer
   alias PhilomenaMedia.Analyzers.Result
+  alias PhilomenaMedia.Remote
 
   @behaviour Analyzer
 
@@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Jpeg do
   end
 
   defp stats(file) do
-    case System.cmd("mediastat", [file]) do
+    case Remote.cmd("mediastat", [file]) do
       {output, 0} ->
         [_size, _frames, width, height, num, den] =
           output

lib/philomena_media/analyzers/png.ex

@@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Png do
 
   alias PhilomenaMedia.Analyzers.Analyzer
   alias PhilomenaMedia.Analyzers.Result
+  alias PhilomenaMedia.Remote
 
   @behaviour Analyzer
 
@@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Png do
   end
 
   defp stats(file) do
-    case System.cmd("mediastat", [file]) do
+    case Remote.cmd("mediastat", [file]) do
       {output, 0} ->
         [_size, frames, width, height, num, den] =
           output

lib/philomena_media/analyzers/svg.ex

@@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Svg do
 
   alias PhilomenaMedia.Analyzers.Analyzer
   alias PhilomenaMedia.Analyzers.Result
+  alias PhilomenaMedia.Remote
 
   @behaviour Analyzer
 
@@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Svg do
   end
 
   defp stats(file) do
-    case System.cmd("svgstat", [file]) do
+    case Remote.cmd("svgstat", [file]) do
       {output, 0} ->
         [_size, _frames, width, height, _num, _den] =
           output

lib/philomena_media/analyzers/webm.ex

@@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Analyzers.Webm do
 
   alias PhilomenaMedia.Analyzers.Analyzer
   alias PhilomenaMedia.Analyzers.Result
+  alias PhilomenaMedia.Remote
 
   @behaviour Analyzer
 
@@ -20,7 +21,7 @@ defmodule PhilomenaMedia.Analyzers.Webm do
   end
 
   defp stats(file) do
-    case System.cmd("mediastat", [file]) do
+    case Remote.cmd("mediastat", [file]) do
       {output, 0} ->
         [_size, frames, width, height, num, den] =
           output

lib/philomena_media/gif_preview.ex

@@ -3,6 +3,8 @@ defmodule PhilomenaMedia.GifPreview do
   GIF preview generation for video files.
   """
 
+  alias PhilomenaMedia.Remote
+
   @type duration :: float()
   @type dimensions :: {pos_integer(), pos_integer()}
 
@@ -49,7 +51,7 @@ defmodule PhilomenaMedia.GifPreview do
       end)
 
     {_output, 0} =
-      System.cmd(
+      Remote.cmd(
         "ffmpeg",
         commands(decoder, video, gif, clamp(duration), dimensions, num_images, target_framerate)
       )

lib/philomena_media/intensities.ex

@@ -17,6 +17,8 @@ defmodule PhilomenaMedia.Intensities do
   of image dimensions, with poor precision and a poor-to-fair accuracy.
   """
 
+  alias PhilomenaMedia.Remote
+
   @type t :: %__MODULE__{
           nw: float(),
           ne: float(),
@@ -50,7 +52,7 @@ defmodule PhilomenaMedia.Intensities do
   """
   @spec file(Path.t()) :: {:ok, t()} | :error
   def file(input) do
-    System.cmd("image-intensities", [input])
+    Remote.cmd("image-intensities", [input])
     |> case do
       {output, 0} ->
         [nw, ne, sw, se] =

lib/philomena_media/mime.ex

@@ -27,7 +27,7 @@ defmodule PhilomenaMedia.Mime do
   """
   @spec file(Path.t()) :: {:ok, t()} | {:unsupported_mime, t()} | :error
   def file(path) do
-    System.cmd("file", ["-b", "--mime-type", path])
+    PhilomenaMedia.Remote.cmd("file", ["-b", "--mime-type", path])
     |> case do
       {output, 0} ->
         true_mime(String.trim(output))

lib/philomena_media/processors/gif.ex

@@ -3,6 +3,7 @@ defmodule PhilomenaMedia.Processors.Gif do
 
   alias PhilomenaMedia.Intensities
   alias PhilomenaMedia.Analyzers.Result
+  alias PhilomenaMedia.Remote
   alias PhilomenaMedia.Processors.Processor
   alias PhilomenaMedia.Processors
 
@@ -46,15 +47,15 @@ defmodule PhilomenaMedia.Processors.Gif do
   defp optimize(file) do
     optimized = Briefly.create!(extname: ".gif")
 
-    {_output, 0} = System.cmd("gifsicle", ["--careful", "-O2", file, "-o", optimized])
+    {_output, 0} = Remote.cmd("gifsicle", ["--careful", "-O2", file, "-o", optimized])
 
     optimized
   end
 
   defp preview(duration, file) do
     preview = Briefly.create!(extname: ".png")
 
-    {_output, 0} = System.cmd("mediathumb", [file, to_string(duration / 2), preview])
+    {_output, 0} = Remote.cmd("mediathumb", [file, to_string(duration / 2), preview])
 
     preview
   end
@@ -63,7 +64,7 @@ defmodule PhilomenaMedia.Processors.Gif do
     palette = Briefly.create!(extname: ".png")
 
     {_output, 0} =
-      System.cmd("ffmpeg", [
+      Remote.cmd("ffmpeg", [
         "-loglevel",
         "0",
         "-y",
@@ -88,7 +89,7 @@ defmodule PhilomenaMedia.Processors.Gif do
     filter_graph = "[0:v]#{scale_filter}[x];[x][1:v]#{palette_filter}"
 
     {_output, 0} =
-      System.cmd("ffmpeg", [
+      Remote.cmd("ffmpeg", [
         "-loglevel",
         "0",
         "-y",
@@ -109,7 +110,7 @@ defmodule PhilomenaMedia.Processors.Gif do
     mp4 = Briefly.create!(extname: ".mp4")
 
     {_output, 0} =
-      System.cmd("ffmpeg", [
+      Remote.cmd("ffmpeg", [
         "-loglevel",
         "0",
         "-y",
@@ -127,7 +128,7 @@ defmodule PhilomenaMedia.Processors.Gif do
       ])
 
     {_output, 0} =
-      System.cmd("ffmpeg", [
+      Remote.cmd("ffmpeg", [
         "-loglevel",
         "0",
         "-y",