Merge pull request #11 from farcasterxyz/horsefacts/classify-embeds

feat: classify embeds

web/package.json

@@ -14,20 +14,23 @@
     "@farcaster/auth-kit": "^0.0.39",
     "@farcaster/core": "^0.13.4",
     "date-fns": "^3.3.1",
-    "linkifyjs": "^4.1.3",
+    "linkify-it": "^5.0.0",
     "next": "14.1.0",
     "next-auth": "^4.24.5",
     "react": "^18",
     "react-dom": "^18"
   },
   "devDependencies": {
+    "@types/linkify-it": "^3.0.5",
     "@types/node": "^20",
     "@types/pg": "^8.10.9",
     "@types/react": "^18",
     "@types/react-dom": "^18",
     "autoprefixer": "^10.0.1",
     "eslint": "^8",
     "eslint-config-next": "14.1.0",
+    "eslint-plugin-simple-import-sort": "^10.0.0",
+    "eslint-plugin-unused-imports": "^3.0.0",
     "kysely": "^0.27.2",
     "kysely-codegen": "^0.11.0",
     "pg": "^8.11.3",

web/src/lib/services/casts.ts

@@ -1,6 +1,7 @@
 import { sql } from 'kysely';
 
 import { db } from '../database/db';
+import { processEmbeds } from './embeds';
 
 export function formatHash(hash: Buffer) {
   return hash.toString('hex');
@@ -51,10 +52,11 @@ export async function getCasts({
     .execute();
 
   return casts.map((row) => {
-    const { fid, pfp_url, display_name, bio, username, ...rest } = row;
+    const { fid, pfp_url, display_name, bio, username, embeds, ...rest } = row;
     return {
       ...rest,
       hash: formatHash(row.hash),
+      embeds: processEmbeds(row.embeds),
       user: {
         fid,
         pfp_url,

web/src/lib/services/embeds.ts

@@ -0,0 +1,82 @@
+import Linkify from 'linkify-it';
+
+const imageExtensions = ['.gif', '.jpeg', '.jpg', '.png', '.svg', '.webp'];
+const videoExtensions = ['.mp4', '.avi', '.mov', '.mkv', '.wmv', '.flv', '.webm', '.m4v', '.ogg'];
+
+export function extractUrlsFromText(text: string): string[] {
+    const li = new Linkify();
+    li.set({ fuzzyEmail: false });
+    li.add('mailto:', null);
+    li.add('@',null);
+
+    const matches = li.match(text) || [];
+
+    return matches.map((match) => match.url);
+};
+
+export function hasExtension(url: string, extensions: string[]): boolean {
+    const parsedUrl = new URL(url.toLowerCase());
+
+    return extensions.some((extension) => parsedUrl.pathname.endsWith(extension) || parsedUrl.search.endsWith(extension));
+  };
+
+export function hasImageExtension(url: string): boolean {
+    return hasExtension(url, imageExtensions);
+}
+
+export function hasVideoExtension(url: string): boolean {
+    return hasExtension(url, videoExtensions);
+}
+
+interface UrlEmbed {
+    url: string;
+}
+
+interface CastEmbed {
+    castId: {
+        fid: number;
+        hash: string;
+    }
+}
+
+type Embed = UrlEmbed | CastEmbed;
+
+export function formatCastEmbed(embed: CastEmbed) {
+    return `${embed.castId.hash}`;
+}
+
+export function processEmbeds(embeds: Embed[]) {
+    let urls: string[] = [];
+    let images: string[] = [];
+    let videos: string[] = [];
+    let unknowns: string[] = [];
+    let casts: string[] = [];
+
+    for (const embed of embeds) {
+        if ('castId' in embed) {
+            casts.push(formatCastEmbed(embed));
+            continue;
+        }
+
+        const { url } = embed;
+        const match = extractUrlsFromText(url);
+        if (match.length !== 1) {
+            unknowns.push(url);
+            continue;
+        }
+
+        if (hasImageExtension(url)) {
+            images.push(url);
+            continue;
+        }
+
+        if (hasVideoExtension(url)) {
+            videos.push(url);
+            continue;
+        }
+
+        urls.push(url);
+    }
+
+    return { urls, images, videos, casts, unknowns };
+}

web/src/lib/services/feed.ts

@@ -1,8 +1,9 @@
-import { Cast, Embed } from '@shared/types/models';
+import { Cast } from '@shared/types/models';
 import { sql } from 'kysely';
 
 import { db } from '../database/db';
 import { formatHash } from './casts';
+import { processEmbeds } from './embeds';
 
 export async function getFeed({
   fid,
@@ -58,7 +59,7 @@ export async function getFeed({
     const cast: Cast = {
       ...rest,
       hash: formatHash(row.hash),
-      embeds: embeds as Embed[],
+      embeds: processEmbeds(row.embeds),
       user: {
         fid,
         pfp_url: pfp_url as string,