API cursor navigation

lib/philomena_query/cursor.ex

@@ -0,0 +1,146 @@
+defmodule PhilomenaQuery.Cursor do
+  alias PhilomenaQuery.Search
+  alias Philomena.Repo
+  import Ecto.Query
+
+  @typedoc """
+  The underlying cursor type, which contains the ordered sort field values
+  of a document.
+  """
+  @type cursor :: [integer() | binary() | boolean()]
+
+  @typedoc """
+  A mapping of document IDs to cursors.
+  """
+  @type cursor_map :: %{integer() => cursor()}
+
+  @doc """
+  Execute search with optional input cursor, and return results as tuple of
+  `{results, cursors}`.
+
+  ## Example
+
+      iex> search_records(
+      ...>   %{query: ..., sort: [%{created_at: :desc}, %{id: :desc}]}
+      ...>   Image
+      ...> )
+      {%Scrivener.Page{entries: [%Image{id: 1}, ...]},
+       %{1 => [1325394000000, 1], ...}}
+
+  """
+  @spec search_records(Search.search_definition(), Search.queryable(), search_after :: term()) ::
+          {Scrivener.Page.t(), cursor_map()}
+  def search_records(search_definition, queryable, search_after) do
+    search_definition = search_after_definition(search_definition, search_after)
+    page = Search.search_records_with_hits(search_definition, queryable)
+
+    {records, cursors} =
+      Enum.map_reduce(page, %{}, fn {record, hit}, cursors ->
+        sort = Map.fetch!(hit, "sort")
+
+        {record, Map.put(cursors, record.id, sort)}
+      end)
+
+    {Map.put(page, :entries, records), cursors}
+  end
+
+  @doc """
+  Return page of records and cursors map based on sort.
+
+  ## Example
+
+      iex> paginate(Forum, [page_size: 25], ["dis", 3], asc: :name, asc: :id)
+      %{4 => ["Generals", 4]}
+
+  """
+  @spec paginate(
+          Ecto.Query.t(),
+          scrivener_opts :: any(),
+          search_after :: term(),
+          sorts :: Keyword.t()
+        ) :: {Scrivener.Page.t(), cursor_map()}
+  def paginate(query, pagination, search_after, sorts) do
+    total_entries = Repo.aggregate(query, :count)
+    pagination = Keyword.merge(pagination, options: [total_entries: total_entries])
+
+    records =
+      query
+      |> order_by(^sorts)
+      |> search_after_query(search_after, sorts)
+      |> Repo.paginate(pagination)
+
+    fields = Keyword.values(sorts)
+
+    cursors =
+      Enum.reduce(records, %{}, fn record, cursors ->
+        field_values = Enum.map(fields, &Map.fetch!(record, &1))
+        Map.put(cursors, record.id, field_values)
+      end)
+
+    {records, cursors}
+  end
+
+  @spec search_after_definition(Search.search_definition(), term()) :: Search.search_definition()
+  defp search_after_definition(search_definition, search_after) do
+    search_after
+    |> permit_search_after()
+    |> case do
+      [] ->
+        search_definition
+
+      search_after ->
+        update_in(search_definition.body, &Map.put(&1, :search_after, search_after))
+    end
+  end
+
+  @spec search_after_query(Ecto.Query.t(), term(), Keyword.t()) :: Ecto.Query.t()
+  defp search_after_query(query, search_after, sorts) do
+    search_after = permit_search_after(search_after)
+    combined = Enum.zip(sorts, search_after)
+
+    case combined do
+      [_some | _rest] = values ->
+        or_clauses = dynamic([], false)
+
+        {or_clauses, _} =
+          Enum.reduce(values, {or_clauses, []}, fn {{sd, col}, value}, {next, equal_parts} ->
+            # more specific column has next value
+            and_clauses =
+              if sd == :asc do
+                dynamic([s], field(s, ^col) > ^value)
+              else
+                dynamic([s], field(s, ^col) < ^value)
+              end
+
+            # and
+            and_clauses =
+              Enum.reduce(equal_parts, and_clauses, fn {col, value}, rest ->
+                # less specific columns are equal
+                dynamic([s], field(s, ^col) == ^value and ^rest)
+              end)
+
+            {dynamic(^next or ^and_clauses), equal_parts ++ [{col, value}]}
+          end)
+
+        where(query, ^or_clauses)
+
+      _ ->
+        query
+    end
+  end
+
+  # Validate that search_after values are only strings, numbers, and bools
+  defp permit_search_after(search_after) do
+    search_after
+    |> permit_list()
+    |> Enum.flat_map(&permit_value/1)
+  end
+
+  defp permit_list(value) when is_list(value), do: value
+  defp permit_list(_value), do: []
+
+  defp permit_value(value) when is_binary(value) or is_number(value) or is_boolean(value),
+    do: [value]
+
+  defp permit_value(_value), do: []
+end

lib/philomena_web/controllers/api/json/filter/system_filter_controller.ex

@@ -2,18 +2,21 @@ defmodule PhilomenaWeb.Api.Json.Filter.SystemFilterController do
   use PhilomenaWeb, :controller
 
   alias Philomena.Filters.Filter
-  alias Philomena.Repo
+  alias PhilomenaQuery.Cursor
   import Ecto.Query
 
-  def index(conn, _params) do
-    system_filters =
+  def index(conn, params) do
+    {system_filters, cursors} =
       Filter
       |> where(system: true)
-      |> order_by(asc: :id)
-      |> Repo.paginate(conn.assigns.scrivener)
+      |> Cursor.paginate(conn.assigns.scrivener, params["search_after"], asc: :id)
 
     conn
     |> put_view(PhilomenaWeb.Api.Json.FilterView)
-    |> render("index.json", filters: system_filters, total: system_filters.total_entries)
+    |> render("index.json",
+      cursors: cursors,
+      filters: system_filters,
+      total: system_filters.total_entries
+    )
   end
 end

lib/philomena_web/controllers/api/json/filter/user_filter_controller.ex

@@ -2,10 +2,10 @@ defmodule PhilomenaWeb.Api.Json.Filter.UserFilterController do
   use PhilomenaWeb, :controller
 
   alias Philomena.Filters.Filter
-  alias Philomena.Repo
+  alias PhilomenaQuery.Cursor
   import Ecto.Query
 
-  def index(conn, _params) do
+  def index(conn, params) do
     user = conn.assigns.current_user
 
     case user do
@@ -15,15 +15,18 @@ defmodule PhilomenaWeb.Api.Json.Filter.UserFilterController do
         |> text("")
 
       _ ->
-        user_filters =
+        {user_filters, cursors} =
           Filter
           |> where(user_id: ^user.id)
-          |> order_by(asc: :id)
-          |> Repo.paginate(conn.assigns.scrivener)
+          |> Cursor.paginate(conn.assigns.scrivener, params["search_after"], asc: :id)
 
         conn
         |> put_view(PhilomenaWeb.Api.Json.FilterView)
-        |> render("index.json", filters: user_filters, total: user_filters.total_entries)
+        |> render("index.json",
+          cursors: cursors,
+          filters: user_filters,
+          total: user_filters.total_entries
+        )
     end
   end
 end

lib/philomena_web/controllers/api/json/forum/topic/post_controller.ex

@@ -1,39 +1,46 @@
 defmodule PhilomenaWeb.Api.Json.Forum.Topic.PostController do
   use PhilomenaWeb, :controller
 
+  alias Philomena.Topics.Topic
   alias Philomena.Posts.Post
+  alias PhilomenaQuery.Cursor
   alias Philomena.Repo
   import Ecto.Query
 
+  def index(conn, %{
+        "forum_id" => forum_id,
+        "topic_id" => topic_id,
+        "search_after" => search_after
+      }) do
+    topic = Repo.one!(topic_query(topic_id, forum_id))
+
+    {posts, cursors} =
+      post_query(topic_id, forum_id)
+      |> Cursor.paginate(conn.assigns.scrivener, search_after, asc: :topic_position)
+
+    render(conn, "index.json", cursors: cursors, posts: posts, total: topic.post_count)
+  end
+
   def index(conn, %{"forum_id" => forum_id, "topic_id" => topic_id}) do
     page = conn.assigns.pagination.page_number
 
-    posts =
-      Post
-      |> join(:inner, [p], _ in assoc(p, :topic))
-      |> join(:inner, [_p, t], _ in assoc(t, :forum))
-      |> where(destroyed_content: false)
-      |> where([_p, t], t.hidden_from_users == false and t.slug == ^topic_id)
-      |> where([_p, _t, f], f.access_level == "normal" and f.short_name == ^forum_id)
-      |> where([p], p.topic_position >= ^(25 * (page - 1)) and p.topic_position < ^(25 * page))
-      |> order_by(asc: :topic_position)
-      |> preload([:user, :topic])
-      |> preload([_p, t, _f], topic: t)
-      |> Repo.all()
-
-    render(conn, "index.json", posts: posts, total: hd(posts).topic.post_count)
+    topic = Repo.one!(topic_query(topic_id, forum_id))
+
+    {posts, cursors} =
+      post_query(topic_id, forum_id)
+      |> where(
+        [posts: p],
+        p.topic_position >= ^(25 * (page - 1)) and p.topic_position < ^(25 * page)
+      )
+      |> Cursor.paginate([page_size: 25], [], asc: :topic_position)
+
+    render(conn, "index.json", cursors: cursors, posts: posts, total: topic.post_count)
   end
 
   def show(conn, %{"forum_id" => forum_id, "topic_id" => topic_id, "id" => post_id}) do
     post =
-      Post
-      |> join(:inner, [p], _ in assoc(p, :topic))
-      |> join(:inner, [_p, t], _ in assoc(t, :forum))
+      post_query(forum_id, topic_id)
       |> where(id: ^post_id)
-      |> where(destroyed_content: false)
-      |> where([_p, t], t.hidden_from_users == false and t.slug == ^topic_id)
-      |> where([_p, _t, f], f.access_level == "normal" and f.short_name == ^forum_id)
-      |> preload([:user, :topic])
       |> Repo.one()
 
     cond do
@@ -46,4 +53,27 @@ defmodule PhilomenaWeb.Api.Json.Forum.Topic.PostController do
         render(conn, "show.json", post: post)
     end
   end
+
+  defp topic_query(topic_id, forum_id) do
+    Topic
+    |> from(as: :topic)
+    |> join(:inner, [topic: t], _ in assoc(t, :forum), as: :forum)
+    |> topic_conditions(topic_id, forum_id)
+  end
+
+  defp post_query(topic_id, forum_id) do
+    Post
+    |> from(as: :posts)
+    |> join(:inner, [posts: p], _ in assoc(p, :topic), as: :topic)
+    |> join(:inner, [topic: t], _ in assoc(t, :forum), as: :forum)
+    |> topic_conditions(topic_id, forum_id)
+    |> where([posts: p], p.destroyed_content == false)
+    |> preload([:user])
+  end
+
+  defp topic_conditions(queryable, topic_id, forum_id) do
+    queryable
+    |> where([topic: t], t.hidden_from_users == false and t.slug == ^topic_id)
+    |> where([forum: f], f.access_level == "normal" and f.short_name == ^forum_id)
+  end
 end

lib/philomena_web/controllers/api/json/forum/topic_controller.ex

@@ -2,20 +2,24 @@ defmodule PhilomenaWeb.Api.Json.Forum.TopicController do
   use PhilomenaWeb, :controller
 
   alias Philomena.Topics.Topic
+  alias PhilomenaQuery.Cursor
   alias Philomena.Repo
   import Ecto.Query
 
-  def index(conn, %{"forum_id" => id}) do
-    topics =
+  def index(conn, %{"forum_id" => id} = params) do
+    {topics, cursors} =
       Topic
       |> join(:inner, [t], _ in assoc(t, :forum))
       |> where(hidden_from_users: false)
       |> where([_t, f], f.access_level == "normal" and f.short_name == ^id)
-      |> order_by(desc: :sticky, desc: :last_replied_to_at)
       |> preload([:user])
-      |> Repo.paginate(conn.assigns.scrivener)
+      |> Cursor.paginate(conn.assigns.scrivener, params["search_after"],
+        desc: :sticky,
+        desc: :last_replied_to_at,
+        desc: :slug
+      )
 
-    render(conn, "index.json", topics: topics, total: topics.total_entries)
+    render(conn, "index.json", cursors: cursors, topics: topics, total: topics.total_entries)
   end
 
   def show(conn, %{"forum_id" => forum_id, "id" => id}) do
@@ -25,7 +29,6 @@ defmodule PhilomenaWeb.Api.Json.Forum.TopicController do
       |> where(slug: ^id)
       |> where(hidden_from_users: false)
       |> where([_t, f], f.access_level == "normal" and f.short_name == ^forum_id)
-      |> order_by(desc: :sticky, desc: :last_replied_to_at)
       |> preload([:user])
       |> Repo.one()
 

lib/philomena_web/controllers/api/json/forum_controller.ex

@@ -1,18 +1,21 @@
 defmodule PhilomenaWeb.Api.Json.ForumController do
   use PhilomenaWeb, :controller
 
+  alias PhilomenaQuery.Cursor
   alias Philomena.Forums.Forum
   alias Philomena.Repo
   import Ecto.Query
 
-  def index(conn, _params) do
-    forums =
+  def index(conn, params) do
+    {forums, cursors} =
       Forum
       |> where(access_level: "normal")
-      |> order_by(asc: :name)
-      |> Repo.paginate(conn.assigns.scrivener)
+      |> Cursor.paginate(conn.assigns.scrivener, params["search_after"],
+        asc: :name,
+        asc: :short_name
+      )
 
-    render(conn, forums: forums, total: forums.total_entries)
+    render(conn, cursors: cursors, forums: forums, total: forums.total_entries)
   end
 
   def show(conn, %{"id" => id}) do