Local dev environment

packages-rs/drainpipe/Dockerfile

@@ -12,6 +12,5 @@ FROM alpine:3.14
 COPY --from=builder /usr/src/unravel/target/release/drainpipe /
 
 ENV DATABASE_URL="/drainpipedata/drainpipe.db"
-ENV FRONTPAGE_CONSUMER_URL="https://frontpage.fyi/api/receive_hook"
 
 ENTRYPOINT ["/drainpipe"]

packages-rs/drainpipe/fly.toml

@@ -22,3 +22,6 @@ processes = ['app']
 
 [[vm]]
 size = 'shared-cpu-1x'
+
+[env]
+FRONTPAGE_CONSUMER_URL = "https://frontpage.fyi/api/receive_hook"

packages-rs/drainpipe/migrations/2024-06-12-232142_create_db/up.sql

@@ -7,6 +7,3 @@ CREATE TABLE IF NOT EXISTS dead_letter_queue(
   seq bigint NOT NULL,
   msg text NOT NULL
 );
-
-INSERT INTO drainpipe(seq)
-  VALUES (622867028);

packages-rs/drainpipe/src/main.rs

@@ -10,7 +10,7 @@ use diesel::{
 };
 use futures::{StreamExt as _, TryFutureExt};
 use serde::Serialize;
-use std::{path::PathBuf, thread, time::Duration};
+use std::{path::PathBuf, process::ExitCode, time::Duration};
 use tokio_tungstenite::tungstenite::{client::IntoClientRequest, protocol::Message};
 
 mod db;
@@ -147,7 +147,7 @@ struct Context {
 }
 
 #[tokio::main]
-async fn main() {
+async fn main() -> ExitCode {
     // Load environment variables from .env.local and .env when ran with cargo run
     if let Some(manifest_dir) = std::env::var("CARGO_MANIFEST_DIR").ok() {
         let env_path: PathBuf = [&manifest_dir, ".env.local"].iter().collect();
@@ -156,6 +156,7 @@ async fn main() {
         dotenv_flow::from_filename(env_path).ok();
     }
 
+    let relay_url = std::env::var("RELAY_URL").unwrap_or("wss://bsky.network".into());
     let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL not set");
     let conn = db::db_connect(&database_url).expect("Failed to connect to db");
     let mut ctx = Context {
@@ -179,28 +180,36 @@ async fn main() {
         });
     }
 
-    loop {
-        let cursor = db::get_seq(&mut ctx.db_connection).expect("Failed to get sequence");
+    for attempt_number in 0..5 {
+        tokio::time::sleep(Duration::from_secs(attempt_number)).await; // Exponential backoff
+
         let connect_result = {
+            let query_string = match db::get_seq(&mut ctx.db_connection) {
+                Ok(cursor) => format!("?cursor={}", cursor),
+                Err(_) => {
+                    eprintln!("Failed to get sequence number. Starting from the beginning.");
+                    "".into()
+                }
+            };
             let mut ws_request = format!(
-                "wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos?cursor={}",
-                cursor
+                "{}/xrpc/com.atproto.sync.subscribeRepos{}",
+                relay_url, query_string
             )
             .into_client_request()
             .unwrap();
+
             ws_request.headers_mut().insert(
                 "User-Agent",
                 reqwest::header::HeaderValue::from_static(
                     "drainpipe/@frontpage.fyi (@tom-sherman.com)",
                 ),
             );
 
-            tokio_tungstenite::connect_async(format!(
-                "wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos?cursor={}",
-                cursor
-            ))
-            .await
+            println!("Connecting to {}", ws_request.uri());
+
+            tokio_tungstenite::connect_async(ws_request).await
         };
+
         match connect_result {
             Ok((mut socket, _response)) => loop {
                 match socket.next().await {
@@ -243,11 +252,15 @@ async fn main() {
             },
             Err(error) => {
                 eprintln!(
-                    "Error connecting to bgs.bsky-sandbox.dev. Waiting to reconnect: {error:?}"
+                    "Error connecting to {}. Waiting to reconnect: {error:?}",
+                    relay_url
                 );
-                thread::sleep(Duration::from_millis(500));
+                tokio::time::sleep(Duration::from_millis(500)).await;
                 continue;
             }
         }
     }
+
+    eprintln!("Max retries exceeded. Exiting.");
+    ExitCode::FAILURE
 }

packages/frontpage/app/(app)/post/[postAuthor]/[postRkey]/[commentAuthor]/[commentRkey]/og-image/route.tsx

@@ -12,6 +12,7 @@ import { CommentPageParams, getCommentPageData } from "../_lib/page-data";
 import { getBlueskyProfile } from "@/lib/data/user";
 import { shouldHideComment } from "@/lib/data/db/comment";
 import { notFound } from "next/navigation";
+import { getVerifiedHandle } from "@/lib/data/atproto/identity";
 
 export const dynamic = "force-static";
 export const revalidate = 60 * 60; // 1 hour
@@ -25,7 +26,10 @@ export async function GET(
     notFound();
   }
 
-  const { avatar, handle } = await getBlueskyProfile(comment.authorDid);
+  const [handle, profile] = await Promise.all([
+    getVerifiedHandle(comment.authorDid),
+    getBlueskyProfile(comment.authorDid),
+  ]);
 
   return frontpageOgImageResponse(
     <OgWrapper
@@ -61,14 +65,25 @@ export async function GET(
         </OgBox>
       </OgBox>
       <OgBottomBar>
-        <img
-          src={avatar}
-          width={48}
-          height={48}
-          style={{
-            borderRadius: "100%",
-          }}
-        />
+        {profile ? (
+          <img
+            src={profile.avatar}
+            width={48}
+            height={48}
+            style={{
+              borderRadius: "100%",
+            }}
+          />
+        ) : (
+          <div
+            style={{
+              width: 48,
+              height: 48,
+              borderRadius: "100%",
+              backgroundColor: "#00000073",
+            }}
+          />
+        )}
         <OgBox style={{ alignItems: "center", gap: 4 }}>
           <OgVoteIcon />
           <OgBox>

packages/frontpage/app/(app)/post/[postAuthor]/[postRkey]/og-image/route.tsx

@@ -22,7 +22,7 @@ export const revalidate = 60 * 60; // 1 hour
 
 export async function GET(_req: Request, { params }: { params: Params }) {
   const { post } = await getPostPageData(params);
-  const { avatar } = await getBlueskyProfile(post.authorDid);
+  const profile = await getBlueskyProfile(post.authorDid);
 
   return frontpageOgImageResponse(
     <OgWrapper
@@ -48,14 +48,25 @@ export async function GET(_req: Request, { params }: { params: Params }) {
         {post.title}
       </OgBox>
       <OgBottomBar>
-        <img
-          src={avatar}
-          width={48}
-          height={48}
-          style={{
-            borderRadius: "100%",
-          }}
-        />
+        {profile ? (
+          <img
+            src={profile.avatar}
+            width={48}
+            height={48}
+            style={{
+              borderRadius: "100%",
+            }}
+          />
+        ) : (
+          <div
+            style={{
+              width: 48,
+              height: 48,
+              borderRadius: "100%",
+              backgroundColor: "#00000073",
+            }}
+          />
+        )}
         <OgBox style={{ alignItems: "center", gap: 4 }}>
           <OgVoteIcon />
           <OgBox>

packages/frontpage/drizzle.config.ts

@@ -5,10 +5,16 @@ import { defineConfig } from "drizzle-kit";
 loadEnvConfig(process.cwd());
 
 const URL = process.env.TURSO_CONNECTION_URL!;
-const AUTH_TOKEN = process.env.TURSO_AUTH_TOKEN!;
+const AUTH_TOKEN = process.env.TURSO_AUTH_TOKEN;
 
-if (!URL || !AUTH_TOKEN) {
-  throw new Error("TURSO_CONNECTION_URL and TURSO_AUTH_TOKEN must be set");
+if (!URL) {
+  throw new Error("TURSO_CONNECTION_URL must be set");
+}
+
+console.log("Connecting to", URL);
+
+if (URL.endsWith(".turso.io") && !AUTH_TOKEN) {
+  throw new Error("TURSO_AUTH_TOKEN must be set when connecting to turso.io");
 }
 
 export default defineConfig({

packages/frontpage/lib/components/user-avatar.tsx

@@ -45,12 +45,15 @@ async function AvatarImage({
   did,
   size: sizeVariant = "small",
 }: UserAvatarProps) {
-  const { avatar } = await getBlueskyProfile(did);
+  const profile = await getBlueskyProfile(did);
+  if (!profile) {
+    return <AvatarFallback size={sizeVariant} />;
+  }
   const size = userAvatarSizes[sizeVariant];
   return (
     // eslint-disable-next-line @next/next/no-img-element
     <img
-      src={avatar}
+      src={profile.avatar}
       alt=""
       width={size}
       height={size}

packages/frontpage/lib/data/atproto/did.ts

@@ -16,7 +16,8 @@ export function parseDid(s: string): DID | null {
 }
 
 export const getDidDoc = cache(async (did: DID) => {
-  const response = await fetch(`https://plc.directory/${did}`, {
+  const url = process.env.PLC_DIRECTORY_URL ?? "https://plc.directory";
+  const response = await fetch(`${url}/${did}`, {
     next: {
       // TODO: Also revalidate this when we receive an identity change event
       // That would allow us to extend the revalidation time to 1 day

packages/frontpage/lib/data/user.ts

@@ -55,6 +55,9 @@ const ProfileResponse = z.object({
   handle: z.string(),
 });
 
+/**
+ * Returns null if the profile is not found (happens only really in local development)
+ */
 export const getBlueskyProfile = cache(async (did: DID) => {
   const json = await fetch(
     `https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor=${did}`,
@@ -65,7 +68,7 @@ export const getBlueskyProfile = cache(async (did: DID) => {
     },
   ).then((res) => res.json());
 
-  return ProfileResponse.parse(json);
+  return ProfileResponse.safeParse(json)?.data ?? null;
 });
 
 export const getTotalSubmissions = cache(async (did: DID) => {

packages/frontpage/local-infra/.gitignore

@@ -0,0 +1,3 @@
+pds/
+plc/
+drainpipe/

packages/frontpage/local-infra/Caddyfile

@@ -0,0 +1,50 @@
+{
+	storage file_system /data/
+	debug
+	pki {
+		ca unravel {
+			name "Unravel"
+		}
+	}
+}
+
+acme.dev.unravel.fyi {
+	tls {
+		issuer internal {
+			ca unravel
+		}
+	}
+	acme_server {
+		ca unravel
+	}
+}
+
+plc.dev.unravel.fyi {
+	tls {
+		issuer internal {
+			ca unravel
+		}
+	}
+
+	reverse_proxy http://plc:8080
+}
+
+turso.dev.unravel.fyi {
+	tls {
+		issuer internal {
+			ca unravel
+		}
+	}
+
+	reverse_proxy http://turso_dev:8080
+}
+
+*.pds.dev.unravel.fyi, pds.dev.unravel.fyi {
+	tls {
+		issuer internal {
+			ca unravel
+		}
+	}
+
+	reverse_proxy http://pds:3000
+}

packages/frontpage/local-infra/README.md

@@ -0,0 +1,34 @@
+# Frontpage Dev Environment
+
+Docker compose file that runs the required peices of infrastructure for frontpage locally.
+
+> ![NOTE]
+> Does not include the frontpage service itself, you should run that with `pnpm run dev`
+
+## What's inside
+
+- ATProto [PLC server](https://github.com/did-method-plc/did-method-plc) (http://localhost:4000 & https://plc.dev.unravel.fyi)
+- ATProto [PDS](https://github.com/bluesky-social/pds) (http://localhost:4001 & https://pds.dev.unravel.fyi)
+- [Drainpipe](../../../packages-rs/drainpipe/README.md) (pushes data from the PDS to the Frontpage Next.js app)
+- Turso sqlite server (http://localhost:4002 && https://turso.dev.unravel.fyi)
+- [Caddy](https://caddyserver.com/) reverse proxy (it provides the above services over HTTPS)
+- [`cloudflared`](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/do-more-with-tunnels/trycloudflare/) (a public https tunnel to the local Frontpage Next.js app)
+
+## Setup
+
+- `docker-compose up`
+- Install the Unravel CA root certificate in your system's trust store. You can find it in the `frontpage-local-infra_caddy_data` volume at `/data/caddy/pki/authorities/unravel/root.crt`
+- Grab the auto generated `cloudflared` tunnel URL from the logs of the `cloudflared` container
+- Create a test account with `./scripts/create-account.sh <email> <handle>`
+- Go about your business
+
+> ![IMPORTANT]
+> When running Node.js based apps make sure you're setting the `NODE_OPTIONS` environment variable to `--use-openssl-ca` to tell Node.js to use the system's trust store. The scripts inside of Frontpage's `package.json` already do this for you.
+
+## Troubleshooting
+
+### `docker-compose up` fails with `failed to solve: error from sender: open ~/unravel/packages/frontpage/local-infra/plc/db: permission denied`
+
+Delete the ./plc directory and try again.
+
+TODO: This can probably be fixed by using named volumes instead of bind mounts.