From ca066a086bc11a655c6cab389e7a19f04abf30e2 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 27 Nov 2024 19:37:05 -0500 Subject: [PATCH] fix: comment resolution --- eval-results.md | 8 ++ src/adapters/openai/helpers/completions.ts | 17 ++- src/adapters/supabase/helpers/weights.ts | 52 --------- src/handlers/ask-llm.ts | 110 ++++++++++++----- src/handlers/comments.ts | 25 +++- src/handlers/metrics/rouge-score.ts | 102 ---------------- src/handlers/rlhf/completions-scorer.ts | 54 +++------ src/handlers/rlhf/phrase-scorer.ts | 41 ++++--- src/helpers/issue-fetching.ts | 13 ++- src/helpers/trigram-weights.ts | 82 +++++++++++++ src/helpers/weights.ts | 130 +++++++++++++++++++++ src/types/github-types.ts | 43 +++++-- src/types/llm.ts | 10 +- 13 files changed, 426 insertions(+), 261 deletions(-) create mode 100644 eval-results.md delete mode 100644 src/adapters/supabase/helpers/weights.ts delete mode 100644 src/handlers/metrics/rouge-score.ts create mode 100644 src/helpers/trigram-weights.ts create mode 100644 src/helpers/weights.ts diff --git a/eval-results.md b/eval-results.md new file mode 100644 index 0000000..78602d3 --- /dev/null +++ b/eval-results.md @@ -0,0 +1,8 @@ +## Evaluation Results + +| Metric | Current | vs Previous | Status | +|--------|---------|-------------|---------| +| Levenshtein | 0.3582 | ↑ 0.0043 | ✅ | +| Context Precision | 1.0000 | ↓ 0.0000 | ✅ | +| Duration | 38.57s | ↑ 16.27s | ⚠️ | +| Cost | $0.246485 | ↓ 0.0000 | ✅ | \ No newline at end of file diff --git a/src/adapters/openai/helpers/completions.ts b/src/adapters/openai/helpers/completions.ts index 8da79fd..92a92ce 100644 --- a/src/adapters/openai/helpers/completions.ts +++ b/src/adapters/openai/helpers/completions.ts @@ -1,7 +1,7 @@ import OpenAI from "openai"; import { Context } from "../../../types"; import { SuperOpenAi } from "./openai"; -import { CompletionsModelHelper, ModelApplications } from "../../../types/llm"; +import { CompletionsModelHelper, ModelApplications, StreamlinedComment } from "../../../types/llm"; import { encode } from "gpt-tokenizer"; import { logger } from "../../../helpers/errors"; import { createWeightTable } from "../../../handlers/rlhf/completions-scorer"; @@ -15,6 +15,7 @@ export interface CompletionsType { total: number; }; } + export const defaultCompletionsType: CompletionsType = { answer: "", groundTruths: [], @@ -24,6 +25,7 @@ export const defaultCompletionsType: CompletionsType = { total: 0, }, }; + export class Completions extends SuperOpenAi { protected context: Context; @@ -84,9 +86,13 @@ export class Completions extends SuperOpenAi { const sysMsg = [ "You Must obey the following ground truths: ", JSON.stringify(groundTruths) + "\n", - "You are tasked with assisting as a GitHub bot by generating responses based on provided chat history supported by the phrases ", + "You are tasked with assisting as a GitHub bot by generating responses based on provided chat history and weighted context. The context is weighted based on:", + "1. User Reactions: Positive reactions (👍, ❤️, 🎉, 🚀) increase weight, negative reactions (👎, 😕) decrease weight", + "2. Edit History: Comments that have been refined through edits have higher weight", + "3. Similarity to Current Query: Content more similar to the current question has higher weight", + "\nWeighted Context Table:", weightPrompt + "\n", - "and similar responses, focusing on using available knowledge within the provided corpus, which may contain code, documentation, or incomplete information. Your role is to interpret and use this knowledge effectively to answer user questions.\n\n# Steps\n\n1. **Understand Context**: Review the chat history and any similar provided responses to understand the context.\n2. **Extract Relevant Information**: Identify key pieces of information, even if they are incomplete, from the available corpus.\n3. **Apply Knowledge**: Use the extracted information and relevant documentation to construct an informed response.\n4. **Draft Response**: Compile the gathered insights into a coherent and concise response, ensuring it's clear and directly addresses the user's query.\n5. **Review and Refine**: Check for accuracy and completeness, filling any gaps with logical assumptions where necessary.\n\n# Output Format\n\n- Concise and coherent responses in paragraphs that directly address the user's question.\n- Incorporate inline code snippets or references from the documentation if relevant.\n\n# Examples\n\n**Example 1**\n\n*Input:*\n- Chat History: \"What was the original reason for moving the LP tokens?\"\n- Corpus Excerpts: \"It isn't clear to me if we redid the staking yet and if we should migrate. If so, perhaps we should make a new issue instead. We should investigate whether the missing LP tokens issue from the MasterChefV2.1 contract is critical to the decision of migrating or not.\"\n\n*Output:*\n\"It was due to missing LP tokens issue from the MasterChefV2.1 Contract.\n\n# Notes\n\n- Ensure the response is crafted from the corpus provided, without introducing information outside of what's available or relevant to the query.\n- Consider edge cases where the corpus might lack explicit answers, and justify responses with logical reasoning based on the existing information.", + "Your role is to interpret this weighted knowledge effectively to answer user questions, giving more consideration to higher-weighted content.\n\n# Steps\n\n1. **Understand Context**: Review the chat history and weighted responses, prioritizing higher-weighted content.\n2. **Extract Relevant Information**: Focus on information from highly-weighted sources, which represent community-validated content.\n3. **Apply Knowledge**: Use the extracted information, considering both content relevance and community feedback.\n4. **Draft Response**: Compile insights into a coherent response, emphasizing information from highly-weighted sources.\n5. **Review and Refine**: Ensure accuracy and alignment with the weighted context.\n\n# Output Format\n\n- Concise and coherent responses that directly address the user's question.\n- Prioritize information from highly-weighted sources.\n- Include code snippets or references when relevant.\n\n# Notes\n\n- Higher weights indicate stronger community validation through reactions and refinements.\n- Consider both the content and its weight when forming responses.\n- Balance between different sources based on their weights.", `Your name is: ${botName}`, "\n", "Main Context (Provide additional precedence in terms of information): ", @@ -149,9 +155,10 @@ export class Completions extends SuperOpenAi { localContext: string[], groundTruths: string[], botName: string, - maxTokens: number + maxTokens: number, + weightedComments: StreamlinedComment[] = [] ): Promise { - const weightPrompt = await createWeightTable(this.context); + const weightPrompt = await createWeightTable(weightedComments); return await this.createCompletion(query, model, additionalContext, localContext, groundTruths, botName, maxTokens, weightPrompt); } diff --git a/src/adapters/supabase/helpers/weights.ts b/src/adapters/supabase/helpers/weights.ts deleted file mode 100644 index c5d2919..0000000 --- a/src/adapters/supabase/helpers/weights.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { SupabaseClient } from "@supabase/supabase-js"; -import { Context } from "../../../types/context"; -import { SuperSupabase } from "./supabase"; - -export interface WeightTableResult { - weight: number; - phrase: string; - comment_node_id: string; -} - -export class Weights extends SuperSupabase { - constructor(supabase: SupabaseClient, context: Context) { - super(supabase, context); - } - - /// Get the weight for the given phrase - async getWeight(phrase: string): Promise { - /// Use trigram search to get the weight for the given phrase - /// Weight vectors would be valid within a org/repo context. Issue ? - const { data, error } = await this.supabase.rpc("get_weight", { - input_phrase: phrase, - }); - if (error) { - this.context.logger.error(error.message || "Error getting weight for phrase"); - throw new Error(`Error getting weight for phrase: ${phrase}`); - } - return data || 0; - } - - /// Set the weight for the given phrase - async setWeight(phrase: string, weight: number, commentNodeId: string) { - /// Set the weight for the given phrase - const { error } = await this.supabase.rpc("set_weight", { - inputphrase: phrase, - weight: weight, - commentnodeid: commentNodeId, - }); - if (error) { - this.context.logger.error(error.message || "Error setting weight for phrase"); - throw new Error(`Error setting weight for phrase: ${phrase}`); - } - } - - ///Dump the weight table - async getAllWeights(): Promise { - const { data, error } = await this.supabase.from<"weights", WeightTableResult>("weights").select("*"); - if (error) { - throw new Error("Error getting weights"); - } - return data as WeightTableResult[]; - } -} diff --git a/src/handlers/ask-llm.ts b/src/handlers/ask-llm.ts index 2b0e42c..8cb429e 100644 --- a/src/handlers/ask-llm.ts +++ b/src/handlers/ask-llm.ts @@ -1,60 +1,87 @@ +/// Implementation of the LLM question answering system +/// This module handles asking questions to the LLM using context from issues, +/// comments, and repository information. It now uses a weighted comment system +/// based on reactions and edit history instead of Supabase similarity search. + import { Context } from "../types"; import { CompletionsType } from "../adapters/openai/helpers/completions"; -import { CommentSimilaritySearchResult } from "../adapters/supabase/helpers/comment"; -import { IssueSimilaritySearchResult } from "../adapters/supabase/helpers/issues"; import { recursivelyFetchLinkedIssues } from "../helpers/issue-fetching"; import { formatChatHistory } from "../helpers/format-chat-history"; import { fetchRepoDependencies, fetchRepoLanguageStats } from "./ground-truths/chat-bot"; import { findGroundTruths } from "./ground-truths/find-ground-truths"; import { bubbleUpErrorComment, logger } from "../helpers/errors"; +import { calculateTextScore } from "../helpers/trigram-weights"; +import { StreamlinedComment } from "../types/llm"; + +/// Find most relevant comments based on weights and similarity to question +/// Uses the new weighted comments system that combines reactions and edit history +/// to determine relevance along with textual similarity to the question +async function findRelevantComments(question: string, comments: StreamlinedComment[], threshold: number, maxResults: number = 5): Promise { + /// Sort comments by their weight and similarity to question + const scoredComments = comments + .filter((c) => c.body) + .map((comment) => ({ + comment, + /// Combine the comment's weight from reactions/edits with its similarity score + score: (comment.weight || 0) + calculateTextScore(question, [comment]), + })) + .sort((a, b) => b.score - a.score); + + /// Take top results above threshold + return scoredComments + .filter((c) => c.score >= threshold) + .slice(0, maxResults) + .map((c) => c.comment.body || "") + .filter(Boolean); +} export async function askQuestion(context: Context, question: string) { if (!question) { throw logger.error("No question provided"); } - // using any links in comments or issue/pr bodies to fetch more context + + /// Using any links in comments or issue/pr bodies to fetch more context const { specAndBodies, streamlinedComments } = await recursivelyFetchLinkedIssues({ context, owner: context.payload.repository.owner.login, repo: context.payload.repository.name, }); - // build a nicely structure system message containing a streamlined chat history - // includes the current issue, any linked issues, and any linked PRs + + /// Get all comments as a flat array for processing + const allComments = Object.values(streamlinedComments).flat(); + + /// Find relevant comments based on weights and question similarity + /// This replaces the previous Supabase similarity search with our new weighted system + const relevantComments = await findRelevantComments(question, allComments, context.config.similarityThreshold); + + /// Build a nicely structured system message containing a streamlined chat history + /// Includes the current issue, any linked issues, and any linked PRs const formattedChat = await formatChatHistory(context, streamlinedComments, specAndBodies); logger.info(`${formattedChat.join("")}`); - return await askLlm(context, question, formattedChat); + + return await askLlm(context, question, formattedChat, relevantComments, allComments); } -export async function askLlm(context: Context, question: string, formattedChat: string[]): Promise { +export async function askLlm( + context: Context, + question: string, + formattedChat: string[], + relevantComments: string[], + weightedComments: StreamlinedComment[] +): Promise { const { env: { UBIQUITY_OS_APP_NAME }, - config: { model, similarityThreshold, maxTokens }, + config: { model, maxTokens }, adapters: { - supabase: { comment, issue }, - voyage: { reranker }, openai: { completions }, }, } = context; try { - // using db functions to find similar comments and issues - const [similarComments, similarIssues] = await Promise.all([ - comment.findSimilarComments(question, 1 - similarityThreshold, ""), - issue.findSimilarIssues(question, 1 - similarityThreshold, ""), - ]); - - // combine the similar comments and issues into a single array - const similarText = [ - ...(similarComments?.map((comment: CommentSimilaritySearchResult) => comment.comment_plaintext) || []), - ...(similarIssues?.map((issue: IssueSimilaritySearchResult) => issue.issue_plaintext) || []), - ]; - - // filter out any empty strings + /// Filter out any empty strings from the chat history formattedChat = formattedChat.filter((text) => text); - // rerank the similar text using voyageai - const rerankedText = similarText.length > 0 ? await reranker.reRankResults(similarText, question) : []; - // gather structural data about the payload repository + /// Gather structural data about the payload repository const [languages, { dependencies, devDependencies }] = await Promise.all([fetchRepoLanguageStats(context), fetchRepoDependencies(context)]); let groundTruths: string[] = []; @@ -72,11 +99,36 @@ export async function askLlm(context: Context, question: string, formattedChat: } if (groundTruths.length === 3) { - return await completions.createCompletionWithHF(10, question, model, rerankedText, formattedChat, groundTruths, UBIQUITY_OS_APP_NAME, maxTokens); + return await completions.createCompletionWithHF( + 10, + question, + model, + relevantComments, + formattedChat, + groundTruths, + UBIQUITY_OS_APP_NAME, + maxTokens, + weightedComments + ); } - groundTruths = await findGroundTruths(context, "chat-bot", { languages, dependencies, devDependencies }); - return await completions.createCompletionWithHF(10, question, model, rerankedText, formattedChat, groundTruths, UBIQUITY_OS_APP_NAME, maxTokens); + groundTruths = await findGroundTruths(context, "chat-bot", { + languages, + dependencies, + devDependencies, + }); + + return await completions.createCompletionWithHF( + 10, + question, + model, + relevantComments, + formattedChat, + groundTruths, + UBIQUITY_OS_APP_NAME, + maxTokens, + weightedComments + ); } catch (error) { throw bubbleUpErrorComment(context, error, false); } diff --git a/src/handlers/comments.ts b/src/handlers/comments.ts index 8d1418e..26ed1a2 100644 --- a/src/handlers/comments.ts +++ b/src/handlers/comments.ts @@ -2,6 +2,8 @@ import { logger } from "../helpers/errors"; import { splitKey } from "../helpers/issue"; import { LinkedIssues, SimplifiedComment } from "../types/github-types"; import { StreamlinedComment } from "../types/llm"; +import { processCommentsWithWeights } from "../helpers/weights"; +import { Context } from "../types/context"; /** * Get all streamlined comments from linked issues @@ -15,7 +17,7 @@ export async function getAllStreamlinedComments(linkedIssues: LinkedIssues[]) { const linkedIssueComments = issue.comments || []; if (linkedIssueComments.length === 0) continue; - const linkedStreamlinedComments = streamlineComments(linkedIssueComments); + const linkedStreamlinedComments = await streamlineComments(linkedIssueComments, issue.context); if (!linkedStreamlinedComments) continue; for (const [key, value] of Object.entries(linkedStreamlinedComments)) { @@ -71,13 +73,16 @@ export function createKey(issueUrl: string, issue?: number) { } /** - * Streamline comments by filtering out bot comments and organizing them by issue key + * Streamline comments by filtering out bot comments, organizing them by issue key, + * and calculating weights based on reactions and edits * @param comments - The comments to streamline + * @param context - The context object containing octokit client * @returns The streamlined comments grouped by issue key */ -export function streamlineComments(comments: SimplifiedComment[]) { +export async function streamlineComments(comments: SimplifiedComment[], context: Context) { const streamlined: Record = {}; + // First pass: organize comments by key for (const comment of comments) { const { user, issueUrl: url, body } = comment; if (user?.type === "Bot") continue; @@ -88,14 +93,24 @@ export function streamlineComments(comments: SimplifiedComment[]) { if (user && body) { streamlined[key].push({ - user: user.login, + user, body, - id: parseInt(comment.id, 10), + id: comment.id, org: owner, repo, issueUrl: url, }); } } + + // Second pass: process weights for each group of comments + for (const [key, groupComments] of Object.entries(streamlined)) { + const weightedComments = await processCommentsWithWeights(context, groupComments); + streamlined[key] = weightedComments.map((comment) => ({ + ...comment, + id: comment.id.toString(), + })); + } + return streamlined; } diff --git a/src/handlers/metrics/rouge-score.ts b/src/handlers/metrics/rouge-score.ts deleted file mode 100644 index 2327329..0000000 --- a/src/handlers/metrics/rouge-score.ts +++ /dev/null @@ -1,102 +0,0 @@ -//// Implmentation of ROUGE (https://en.wikipedia.org/wiki/ROUGE_(metric)) - -import { MetricResult, MetricsResult } from "../../types/metrics"; - -interface RougeScore { - precision: number; - recall: number; - fScore: number; -} - -export type RougeInput = { - candidate: string; - reference: string; -}; - -/// Calculate the ROUGE-N score between two strings. -function calculateRougeN(candidate: string, reference: string, n: number): RougeScore { - const candidateNgrams = getNgrams(candidate, n); - const referenceNgrams = getNgrams(reference, n); - - const overlap = candidateNgrams.filter((ngram) => referenceNgrams.includes(ngram)).length; - - const precision = overlap / candidateNgrams.length; - const recall = overlap / referenceNgrams.length; - const fScore = (2 * precision * recall) / (precision + recall) || 0; - - return { precision, recall, fScore }; -} - -/// Get all n-grams of a string. -function getNgrams(text: string, n: number): string[] { - const words = text.toLowerCase().split(/\s+/); - const ngrams: string[] = []; - - for (let i = 0; i <= words.length - n; i++) { - ngrams.push(words.slice(i, i + n).join(" ")); - } - - return ngrams; -} - -/// Calculate the ROUGE metrics for a given input. -export function calculateRougeMetrics(input: RougeInput): MetricsResult { - const { candidate, reference } = input; - const rouge1 = calculateRougeN(candidate, reference, 1); - const rouge2 = calculateRougeN(candidate, reference, 2); - const rougeL = calculateLongestCommonSubsequence(candidate, reference); - - const results: Record> = { - "rouge1.fScore": { - input, - metric: { value: rouge1.fScore, threshold: 0.5, strategy: "greater" }, - }, - "rouge2.fScore": { - input, - metric: { value: rouge2.fScore, threshold: 0.3, strategy: "greater" }, - }, - "rougeL.fScore": { - input, - metric: { value: rougeL.fScore, threshold: 0.4, strategy: "greater" }, - }, - }; - - const isPassed = Object.values(results).every((result) => - result.metric.strategy === "greater" ? result.metric.value >= result.metric.threshold : result.metric.value <= result.metric.threshold - ); - - return { passed: isPassed, results }; -} - -/// Calculate the ROUGE-L score between two strings. -function calculateLongestCommonSubsequence(candidate: string, reference: string): RougeScore { - const candidateWords = candidate.toLowerCase().split(/\s+/); - const referenceWords = reference.toLowerCase().split(/\s+/); - - const lcsLength = getLongestCommonSubsequenceLength(candidateWords, referenceWords); - - const precision = lcsLength / candidateWords.length; - const recall = lcsLength / referenceWords.length; - const fScore = (2 * precision * recall) / (precision + recall) || 0; - - return { precision, recall, fScore }; -} - -/// Returns the length of the longest common subsequence between two arrays. -function getLongestCommonSubsequenceLength(arr1: string[], arr2: string[]): number { - const dp: number[][] = Array(arr1.length + 1) - .fill(0) - .map(() => Array(arr2.length + 1).fill(0)); - - for (let i = 1; i <= arr1.length; i++) { - for (let j = 1; j <= arr2.length; j++) { - if (arr1[i - 1] === arr2[j - 1]) { - dp[i][j] = dp[i - 1][j - 1] + 1; - } else { - dp[i][j] = Math.max(dp[i - 1][j], dp[i][j - 1]); - } - } - } - - return dp[arr1.length][arr2.length]; -} diff --git a/src/handlers/rlhf/completions-scorer.ts b/src/handlers/rlhf/completions-scorer.ts index 1b944af..7fef870 100644 --- a/src/handlers/rlhf/completions-scorer.ts +++ b/src/handlers/rlhf/completions-scorer.ts @@ -1,54 +1,36 @@ -/// Implementation of the socring module for the completions. - import { CompletionsType } from "../../adapters/openai/helpers/completions"; -import { WeightTableResult } from "../../adapters/supabase/helpers/weights"; import { Context } from "../../types/context"; -import { splitIntoTrigrams } from "./phrase-scorer"; +import { StreamlinedComment } from "../../types/llm"; +import { calculateTextScore, calculateTrigramWeights } from "../../helpers/trigram-weights"; -/// Given a Phrase Check if overall weight of the phrase is -export async function calculateCompletionScore(completion: CompletionsType, context: Context): Promise { - let score = 0; +/** + * Calculate a score for a completion based on weighted comments + */ +export async function calculateCompletionScore(completion: CompletionsType, context: Context, weightedComments: StreamlinedComment[]): Promise { const { answer } = completion; - const trigrams = splitIntoTrigrams(answer); - if (trigrams.length === 0) { - throw new Error("No trigrams found in the completion"); - } else { - const { - adapters: { supabase }, - } = context; - for (const trigram of trigrams) { - const weight = await supabase.weights.getWeight(trigram); - score += weight; - } - } - return score; + return calculateTextScore(answer, weightedComments); } -/// Create a structured representation of the phrase weight table -export async function createWeightTable(context: Context) { - const { - adapters: { supabase }, - } = context; - const weights = await supabase.weights.getAllWeights(); - if (!weights) { - throw new Error("Error getting weights"); - } - /// Create a structured representation of the weights +/** + * Create a structured representation of the trigram weights + */ +export async function createWeightTable(weightedComments: StreamlinedComment[]) { + const weights = calculateTrigramWeights(weightedComments); const table = formatWeightTable(weights); return weightTableToString(table); } -function formatWeightTable(data: WeightTableResult[]): string[][] { - const table = [["Phrase Word Table"], ["word", "score", "nature"], ["----------------------------------------"]]; +function formatWeightTable(weights: Map): string[][] { + const table = [["Trigram Weight Table"], ["trigram", "score", "nature"], ["----------------------------------------"]]; - for (const weight of data) { + for (const [trigram, weight] of weights.entries()) { let nature = "NEUTRAL"; - if (weight.weight > 0) { + if (weight > 0) { nature = "POSITIVE"; - } else if (weight.weight < 0) { + } else if (weight < 0) { nature = "NEGATIVE"; } - table.push([weight.phrase, weight.weight.toString(), nature]); + table.push([trigram, weight.toFixed(2), nature]); } return table; } diff --git a/src/handlers/rlhf/phrase-scorer.ts b/src/handlers/rlhf/phrase-scorer.ts index 6f1299e..77e2742 100644 --- a/src/handlers/rlhf/phrase-scorer.ts +++ b/src/handlers/rlhf/phrase-scorer.ts @@ -8,25 +8,32 @@ import { Context } from "../../types/context"; import { Phrase } from "../../types/rlhf"; - -export async function updatePhraseScoreEdit(phrase: Phrase, context: Context, scoringMultiplier: number, isAddition: boolean) { - const { - adapters: { supabase }, - payload: { comment }, - } = context; - /// Get the Comment Node Id for the trigram - const commentNodeId = comment.node_id; - - /// Update the weight for the trigram - const weight = await supabase.weights.getWeight(phrase.text); - context.logger.info(`Weight for trigram ${phrase.text} is ${weight}`); - if (isAddition) { - await supabase.weights.setWeight(phrase.text, weight + scoringMultiplier, commentNodeId); - } else { - await supabase.weights.setWeight(phrase.text, weight - scoringMultiplier, commentNodeId); - } +import { calculateTextScore } from "../../helpers/trigram-weights"; +import { StreamlinedComment } from "../../types/llm"; + +/// Update scores for phrases based on edits and reactions +/// Now uses the new weights system based on reactions and edits history +export async function updatePhraseScoreEdit( + phrase: Phrase, + context: Context, + weightedComments: StreamlinedComment[], + scoringMultiplier: number, + isAddition: boolean +) { + /// Calculate current score using the weighted comments system + const currentScore = calculateTextScore(phrase.text, weightedComments); + + /// Update the weight based on the edit action + const newScore = isAddition ? currentScore + scoringMultiplier : currentScore - scoringMultiplier; + + context.logger.info(`Updated score for phrase "${phrase.text}": ${currentScore} -> ${newScore}`); + + return newScore; } +/// Split text into trigrams for scoring +/// This function creates both word-level and character-level trigrams +/// to capture both semantic and character patterns export function splitIntoTrigrams(phrase: string): string[] { // Normalize the text: lowercase and remove special characters const normalized = phrase.toLowerCase().replace(/[^a-z0-9\s]/g, ""); diff --git a/src/helpers/issue-fetching.ts b/src/helpers/issue-fetching.ts index 58b7b37..0d90990 100644 --- a/src/helpers/issue-fetching.ts +++ b/src/helpers/issue-fetching.ts @@ -37,7 +37,17 @@ export async function fetchLinkedIssues(params: FetchParams) { const issueKey = createKey(issue.html_url); const [owner, repo, issueNumber] = splitKey(issueKey); - const linkedIssues: LinkedIssues[] = [{ body: issue.body, comments, issueNumber: parseInt(issueNumber), owner, repo, url: issue.html_url }]; + const linkedIssues: LinkedIssues[] = [ + { + body: issue.body, + comments, + issueNumber: parseInt(issueNumber), + owner, + repo, + url: issue.html_url, + context: params.context, + }, + ]; const specAndBodies: Record = {}; const seen = new Set([issueKey]); @@ -74,6 +84,7 @@ export async function fetchLinkedIssues(params: FetchParams) { specAndBodies[linkedKey] = fetchedIssue?.body; linkedIssue.body = fetchedIssue?.body; linkedIssue.comments = fetchedComments; + linkedIssue.context = params.context; linkedIssues.push(linkedIssue); } } diff --git a/src/helpers/trigram-weights.ts b/src/helpers/trigram-weights.ts new file mode 100644 index 0000000..e7504ff --- /dev/null +++ b/src/helpers/trigram-weights.ts @@ -0,0 +1,82 @@ +import { StreamlinedComment } from "../types/llm"; +import { splitIntoTrigrams } from "../handlers/rlhf/phrase-scorer"; + +// Cache for trigram weights to avoid recalculating +const trigramWeightsCache = new Map(); + +/** + * Calculate weights for trigrams based on comment reactions and edits + * @param comments - Array of weighted comments to analyze + * @returns Map of trigram to weight + */ +export function calculateTrigramWeights(comments: StreamlinedComment[]): Map { + const trigramWeights = new Map(); + + for (const comment of comments) { + if (!comment.body) continue; + + // Get base weight from comment's reactions and edits + const commentWeight = comment.weight || 0; + + // Split comment into trigrams + const trigrams = splitIntoTrigrams(comment.body); + + // Distribute comment weight across its trigrams + const weightPerTrigram = commentWeight / trigrams.length; + + for (const trigram of trigrams) { + const currentWeight = trigramWeights.get(trigram) || 0; + trigramWeights.set(trigram, currentWeight + weightPerTrigram); + } + } + + return trigramWeights; +} + +/** + * Get weight for a specific trigram from the weighted comments + * @param trigram - The trigram to get weight for + * @param comments - Array of weighted comments to analyze + * @returns Weight of the trigram + */ +export function getTrigramWeight(trigram: string, comments: StreamlinedComment[]): number { + // Check cache first + const cachedWeight = trigramWeightsCache.get(trigram); + if (cachedWeight !== undefined) { + return cachedWeight; + } + + // Calculate weights if not in cache + const weights = calculateTrigramWeights(comments); + + // Cache all weights + for (const [t, w] of weights.entries()) { + trigramWeightsCache.set(t, w); + } + + return weights.get(trigram) || 0; +} + +/** + * Calculate score for a piece of text based on its trigrams + * @param text - Text to score + * @param comments - Array of weighted comments to analyze + * @returns Score for the text + */ +export function calculateTextScore(text: string, comments: StreamlinedComment[]): number { + const trigrams = splitIntoTrigrams(text); + let score = 0; + + for (const trigram of trigrams) { + score += getTrigramWeight(trigram, comments); + } + + return score; +} + +/** + * Clear the trigram weights cache + */ +export function clearTrigramWeightsCache(): void { + trigramWeightsCache.clear(); +} diff --git a/src/helpers/weights.ts b/src/helpers/weights.ts new file mode 100644 index 0000000..b7fd3ff --- /dev/null +++ b/src/helpers/weights.ts @@ -0,0 +1,130 @@ +import { Context } from "../types/context"; +import { Reaction, CommentEdit, WeightedComment, SimplifiedComment } from "../types/github-types"; + +// Reaction weights configuration +const REACTION_WEIGHTS = { + "+1": 1, + heart: 1, + hooray: 1, + rocket: 1, + "-1": -1, + confused: -0.5, + eyes: 0.5, + laugh: 0.25, +}; + +// Calculate weight based on reactions +function calculateReactionWeight(reactions: Reaction[]): number { + return reactions.reduce((total, reaction) => { + return total + (REACTION_WEIGHTS[reaction.content] || 0); + }, 0); +} + +// Calculate weight based on edit history +function calculateEditWeight(edits: CommentEdit[]): number { + // More edits could indicate refinement and improvement + // We use a logarithmic scale to prevent excessive weight from many edits + return edits.length > 0 ? Math.log2(edits.length + 1) : 0; +} + +// Fetch reactions for a comment +async function fetchReactions(context: Context, owner: string, repo: string, commentId: string): Promise { + try { + const { data } = await context.octokit.reactions.listForIssueComment({ + owner, + repo, + comment_id: parseInt(commentId), + }); + return data as Reaction[]; + } catch (error) { + context.logger.error("Error fetching reactions", { + error: error instanceof Error ? error : new Error("Unknown error occurred"), + owner, + repo, + commentId, + }); + return []; + } +} + +interface StrictGraphQlCommentEdit { + createdAt: string; + updatedAt: string; + editedBody: string; +} + +interface StrictGraphQlResponse { + node?: { + userContentEdits?: { + nodes?: StrictGraphQlCommentEdit[]; + }; + }; +} + +// Fetch edit history for a comment using GraphQL +async function fetchCommentEdits(context: Context, commentId: string): Promise { + try { + const query = ` + query($nodeId: ID!) { + node(id: $nodeId) { + ... on IssueComment { + userContentEdits(first: 100) { + nodes { + createdAt + updatedAt + editedBody + } + } + } + } + } + `; + + const result = await context.octokit.graphql(query, { + nodeId: commentId, + }); + + const edits = result.node?.userContentEdits?.nodes || []; + return edits.map((edit) => ({ + created_at: edit.createdAt, + updated_at: edit.updatedAt, + body: edit.editedBody, + })); + } catch (error) { + context.logger.error("Error fetching comment edits", { + error: error instanceof Error ? error : new Error("Unknown error occurred"), + commentId, + }); + return []; + } +} + +// Calculate final weight for a comment based on reactions and edits +function calculateCommentWeight(reactions: Reaction[], edits: CommentEdit[]): number { + const reactionWeight = calculateReactionWeight(reactions); + const editWeight = calculateEditWeight(edits); + + // Combine weights - reactions have more impact than edits + return reactionWeight + editWeight * 0.5; +} + +// Main function to process comments and calculate weights +export async function processCommentsWithWeights(context: Context, comments: SimplifiedComment[]): Promise { + const weightedComments: WeightedComment[] = []; + + for (const comment of comments) { + const reactions = await fetchReactions(context, comment.org, comment.repo, comment.id); + const edits = await fetchCommentEdits(context, comment.id); + const weight = calculateCommentWeight(reactions, edits); + + weightedComments.push({ + ...comment, + weight, + reactions, + edits, + }); + } + + // Sort by weight in descending order + return weightedComments.sort((a, b) => b.weight - a.weight); +} diff --git a/src/types/github-types.ts b/src/types/github-types.ts index b5692de..8766219 100644 --- a/src/types/github-types.ts +++ b/src/types/github-types.ts @@ -6,6 +6,35 @@ export type IssueComments = RestEndpointMethodTypes["issues"]["listComments"]["r export type ReviewComments = RestEndpointMethodTypes["pulls"]["listReviewComments"]["response"]["data"][0]; export type User = RestEndpointMethodTypes["users"]["getByUsername"]["response"]["data"]; +export type Reaction = { + id: number; + node_id: string; + user: Partial; + content: "+1" | "-1" | "laugh" | "confused" | "heart" | "hooray" | "rocket" | "eyes"; + created_at: string; +}; + +export type CommentEdit = { + created_at: string; + updated_at: string; + body: string; +}; + +export type SimplifiedComment = { + user: Partial | null; + body?: string | null; + id: string; + org: string; + repo: string; + issueUrl: string; +}; + +export type WeightedComment = SimplifiedComment & { + weight: number; + reactions: Reaction[]; + edits: CommentEdit[]; +}; + export type FetchParams = { context: Context; issueNum?: number; @@ -19,20 +48,12 @@ export type LinkedIssues = { owner: string; url: string; comments?: SimplifiedComment[] | null | undefined; - body: string | undefined | null; -}; - -export type SimplifiedComment = { - user: Partial | null; - body: string | undefined | null; - id: string; - org: string; - repo: string; - issueUrl: string; + body?: string | null; + context: Context; }; export type FetchedCodes = { - body: string | undefined; + body?: string; user: Partial | null; issueUrl: string; id: string; diff --git a/src/types/llm.ts b/src/types/llm.ts index 7d5bedf..347bc36 100644 --- a/src/types/llm.ts +++ b/src/types/llm.ts @@ -1,4 +1,5 @@ import { GROUND_TRUTHS_SYSTEM_MESSAGES } from "../handlers/ground-truths/prompts"; +import { Reaction, CommentEdit, User } from "./github-types"; export type ModelApplications = "code-review" | "chat-bot"; @@ -33,9 +34,9 @@ export type GroundTruthsSystemMessageTemplate = { }; export type StreamlinedComment = { - id: number; - user?: string; - body?: string; + id: string; + user: Partial | null; + body?: string | null; org: string; repo: string; issueUrl: string; @@ -43,6 +44,9 @@ export type StreamlinedComment = { html: string; text: string; }; + weight?: number; + reactions?: Reaction[]; + edits?: CommentEdit[]; }; export type StreamlinedComments = {