From d80b7127a11bfa75b905fc185fda5f642fec7c7c Mon Sep 17 00:00:00 2001
From: Shivaditya Shivganesh <sshivaditya@gmail.com>
Date: Tue, 7 Jan 2025 02:33:36 +0530
Subject: [PATCH] fix: handle the failures where the issue.title is missing

---
 functions/issue-scraper.ts | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/functions/issue-scraper.ts b/functions/issue-scraper.ts
index c536434..b8d620b 100644
--- a/functions/issue-scraper.ts
+++ b/functions/issue-scraper.ts
@@ -270,19 +270,30 @@ async function issueScraper(username: string, supabase: SupabaseClient, voyageAp
       throw new Error("Username is required");
     }
 
+    let storageFailed = []
+
     const octokit = new Octokit(token ? { auth: token } : {});
     const voyageClient = new VoyageAIClient({ apiKey: voyageApiKey });
 
-    const issues = await fetchUserIssuesBatch(octokit, username, timestamp);
+    let issues = await fetchUserIssuesBatch(octokit, username, timestamp);
 
     const uniqueAuthors = Array.from(new Set(issues.map((issue) => issue.author?.login).filter((login): login is string => !!login)));
 
     const authorIdMap = await batchFetchAuthorIds(octokit, uniqueAuthors);
 
-    const markdowns = issues.map((issue) => {
+    // Filter the issues to include only those with an valid title
+    issues = issues.filter((issue) => {
       if (!issue.title) {
-        throw new Error(`Issue ${issue.id} is missing a title`);
+        storageFailed.push({
+          id: issue.id,
+          reason: "Issue does not have a title",
+        })
+        return false;
       }
+    })
+
+
+    const markdowns = issues.map((issue) => {
       return `${issue.body || ""} ${issue.title}`;
     });
     const plainTexts = markdowns.map(markdownToPlainText);
@@ -338,13 +349,14 @@ async function issueScraper(username: string, supabase: SupabaseClient, voyageAp
         success: true,
         stats: {
           storageSuccessful: upsertData.length,
-          storageFailed: 0,
+          storageFailed: storageFailed.length,
         },
         issues: upsertData.map((issue) => ({
           id: issue.id,
           markdown: issue.markdown,
           plaintext: issue.plaintext,
         })),
+        storageFailed: storageFailed,
       },
       null,
       2