Skip to content

Commit

Permalink
fix: handle the failures where the issue.title is missing
Browse files Browse the repository at this point in the history
  • Loading branch information
sshivaditya committed Jan 6, 2025
1 parent 3ed6c54 commit d80b712
Showing 1 changed file with 16 additions and 4 deletions.
20 changes: 16 additions & 4 deletions functions/issue-scraper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -270,19 +270,30 @@ async function issueScraper(username: string, supabase: SupabaseClient, voyageAp
throw new Error("Username is required");
}

let storageFailed = []

const octokit = new Octokit(token ? { auth: token } : {});
const voyageClient = new VoyageAIClient({ apiKey: voyageApiKey });

const issues = await fetchUserIssuesBatch(octokit, username, timestamp);
let issues = await fetchUserIssuesBatch(octokit, username, timestamp);

const uniqueAuthors = Array.from(new Set(issues.map((issue) => issue.author?.login).filter((login): login is string => !!login)));

const authorIdMap = await batchFetchAuthorIds(octokit, uniqueAuthors);

const markdowns = issues.map((issue) => {
// Filter the issues to include only those with an valid title
issues = issues.filter((issue) => {
if (!issue.title) {
throw new Error(`Issue ${issue.id} is missing a title`);
storageFailed.push({
id: issue.id,
reason: "Issue does not have a title",
})
return false;
}
})


const markdowns = issues.map((issue) => {
return `${issue.body || ""} ${issue.title}`;
});
const plainTexts = markdowns.map(markdownToPlainText);
Expand Down Expand Up @@ -338,13 +349,14 @@ async function issueScraper(username: string, supabase: SupabaseClient, voyageAp
success: true,
stats: {
storageSuccessful: upsertData.length,
storageFailed: 0,
storageFailed: storageFailed.length,
},
issues: upsertData.map((issue) => ({
id: issue.id,
markdown: issue.markdown,
plaintext: issue.plaintext,
})),
storageFailed: storageFailed,
},
null,
2
Expand Down

0 comments on commit d80b712

Please sign in to comment.