Merge development into main

.cspell.json

@@ -39,7 +39,10 @@
     "dylib",
     "mobileprovision",
     "icns",
-    "hono"
+    "hono",
+    "braintrust",
+    "autoevals",
+    "SEPERATOR"
   ],
   "dictionaries": ["typescript", "node", "software-terms"],
   "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"],

.github/knip.ts

@@ -6,7 +6,7 @@ const config: KnipConfig = {
   ignore: ["src/types/config.ts", "**/__mocks__/**", "**/__fixtures__/**"],
   ignoreExportsUsedInFile: true,
   // eslint can also be safely ignored as per the docs: https://knip.dev/guides/handling-issues#eslint--jest
-  ignoreDependencies: ["eslint-config-prettier", "eslint-plugin-prettier", "ts-node", "hono", "cross-env"],
+  ignoreDependencies: ["eslint-config-prettier", "eslint-plugin-prettier", "hono", "ts-node"],
   eslint: true,
 };
 

.github/workflows/evals-testing.yml

@@ -0,0 +1,47 @@
+name: Run Braintrust Evals
+
+on:
+  workflow_run:
+    workflows: ["Knip"]
+    types:
+      - completed
+
+permissions: write-all
+
+jobs:
+  eval:
+    name: Run evals
+    runs-on: ubuntu-latest
+    permissions: write-all
+
+    steps:
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20.10.0"
+
+      - uses: actions/checkout@master
+        with:
+          fetch-depth: 0
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+
+      - name: Install toolchain
+        run: bun install --frozen-lockfile
+
+      - name: Run Evals
+        id: evals
+        run: bun eval
+        env:
+          SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+          SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
+          VOYAGEAI_API_KEY: ${{ secrets.VOYAGEAI_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+          UBIQUITY_OS_APP_NAME: ${{ secrets.UBIQUITY_OS_APP_NAME }}
+          BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Add Evals Report to Github Comment
+        if: always()
+        run: echo "$(cat eval-results.md)" >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -15,4 +15,5 @@ junit.xml
 cypress/screenshots
 script.ts
 .wrangler
-test-dashboard.md
+test-dashboard.md
+eval-results.md

CHANGELOG.md

@@ -0,0 +1,71 @@
+# Changelog
+
+## 1.0.0 (2024-11-29)
+
+### Features
+
+- add commnt with diff styles ([12cbcc4](https://github.com/ubiquity-os-marketplace/command-ask/commit/12cbcc4ed767d6b69ce0f823d9e84dfa9c5c5e7c))
+- added instruction to the embedding ([9583286](https://github.com/ubiquity-os-marketplace/command-ask/commit/958328673c11aac72d5c59b5d7ca8fab9d78ad35))
+- adds OPENROUTER_API_KEY ([2811fa0](https://github.com/ubiquity-os-marketplace/command-ask/commit/2811fa0d5526fb210002f0cbdb9f5b3ad9e5dc4b))
+- basic chat rag works ([0f82015](https://github.com/ubiquity-os-marketplace/command-ask/commit/0f8201500efa88dd16424341bbd9e935877da823))
+- chat ready ([bd790b5](https://github.com/ubiquity-os-marketplace/command-ask/commit/bd790b5f5c4e58eecaa656896aeabe2ce2667471))
+- comments handler ([2b86ab2](https://github.com/ubiquity-os-marketplace/command-ask/commit/2b86ab2fa7720fbc9b76ddd2c921845e98b0295e))
+- deeper linked context fetching ([c7b6605](https://github.com/ubiquity-os-marketplace/command-ask/commit/c7b6605ba25fa0438a473905717b2ea314e67856))
+- dynamic ground truths ([9075b90](https://github.com/ubiquity-os-marketplace/command-ask/commit/9075b90008dc1354e21189ffdbf80713653214a2))
+- esm tests ([a7e92a5](https://github.com/ubiquity-os-marketplace/command-ask/commit/a7e92a5ac9d422a50b36cc7b4df0dfec102eb6bd))
+- fetch repo langs and deps ([b5bbb2b](https://github.com/ubiquity-os-marketplace/command-ask/commit/b5bbb2b0175494c79c4f4294830a1451fde40317))
+- findGroundTruths ([0567c5c](https://github.com/ubiquity-os-marketplace/command-ask/commit/0567c5cd0dc7ff2019aaa32ec1447f26cd560ec2))
+- issue related functions ([7643b3f](https://github.com/ubiquity-os-marketplace/command-ask/commit/7643b3fb9da52feb0c57b0b3b7722f3fcbb003fe))
+- issue utils ([55d5b2a](https://github.com/ubiquity-os-marketplace/command-ask/commit/55d5b2a263d1d91c154e47297cedced4c104481b))
+- manifest commands object ([e7fb9b2](https://github.com/ubiquity-os-marketplace/command-ask/commit/e7fb9b22707d9d1f4c6b36459e688179d6dc2e07))
+- SDK and command interface ([51f812a](https://github.com/ubiquity-os-marketplace/command-ask/commit/51f812afaa0b48ac573a27d3cf59299297818bcc))
+- simple openai chat fn ([b9c1ab5](https://github.com/ubiquity-os-marketplace/command-ask/commit/b9c1ab57ed7c5e47d3fc1eb5fb6baa77a9321b12))
+- type-safe proxy callbacks ([bef6e33](https://github.com/ubiquity-os-marketplace/command-ask/commit/bef6e33a7ba7162950c8c914de43fa80776ffed6))
+- ubiquibot-logger ([1d51869](https://github.com/ubiquity-os-marketplace/command-ask/commit/1d51869054031f014139214e0a59479853ff02c6))
+- ubiquity_os_app_slug config item ([d9d440a](https://github.com/ubiquity-os-marketplace/command-ask/commit/d9d440aad4126a2cecc4c8dbc829e641a0ddaef8))
+- updated compute.yml ([bbfb34f](https://github.com/ubiquity-os-marketplace/command-ask/commit/bbfb34fbacfb72c439e8a1460eda1581672a16e3))
+- workflow command ([8af40cd](https://github.com/ubiquity-os-marketplace/command-ask/commit/8af40cda20f08293d3d5e9fa7d2d24bbb4676f47))
+
+### Bug Fixes
+
+- add environment ([d3ce3eb](https://github.com/ubiquity-os-marketplace/command-ask/commit/d3ce3eb202666ad7c838da1696e5f3314a244384))
+- added plugin action workflow ([bfd968c](https://github.com/ubiquity-os-marketplace/command-ask/commit/bfd968c765d3d5333227ff6e1b1734c7a2001254))
+- adds sorting diff size and then adds diffs size wise add ([312bede](https://github.com/ubiquity-os-marketplace/command-ask/commit/312bede0f3a7c6e59617cece4c41ceaf080f6234))
+- changed to '.ts' in configuration.yml ([6e155e9](https://github.com/ubiquity-os-marketplace/command-ask/commit/6e155e98a08d4974300a6063957fdec535215d55))
+- ci passing locally and cleanup ([d1b4514](https://github.com/ubiquity-os-marketplace/command-ask/commit/d1b4514e4eca8321c8873aa9a1b6bf5b1b879cef))
+- comments and issues ([ae5f552](https://github.com/ubiquity-os-marketplace/command-ask/commit/ae5f5522ebc9968da57a27737bd61ca91f953680))
+- compute.yml ([6660546](https://github.com/ubiquity-os-marketplace/command-ask/commit/66605468fcbbfe5a04325ccfbd2eae4b4dffe7f4))
+- conflicts ([9efaf53](https://github.com/ubiquity-os-marketplace/command-ask/commit/9efaf5342c27ac3ed0c58a5457471c8e658d9178))
+- cspell ([fa67948](https://github.com/ubiquity-os-marketplace/command-ask/commit/fa679485a36be1545492e88f0d85e2b22d4ee69c))
+- cspell, eslint ([6063f36](https://github.com/ubiquity-os-marketplace/command-ask/commit/6063f36e4bb785185265a504faf4ff64efff4957))
+- deploy ([53ee2bb](https://github.com/ubiquity-os-marketplace/command-ask/commit/53ee2bbcb12ee1b3e9bf39c62c6a00f79b144576))
+- formatting ([a8b746f](https://github.com/ubiquity-os-marketplace/command-ask/commit/a8b746f1bfba8ac87fec059b3fe30319631e105a))
+- ignore all bot comments ([6b0333b](https://github.com/ubiquity-os-marketplace/command-ask/commit/6b0333b1bfb8d9d1c41eae8ad46e3b219447da85))
+- ignore file path and diffs ([5d19ba6](https://github.com/ubiquity-os-marketplace/command-ask/commit/5d19ba6781ddbae3dad26706ef0580b59b8ab973))
+- improved context issue filtering ([bfac23d](https://github.com/ubiquity-os-marketplace/command-ask/commit/bfac23dece9c966247c97dc2d05fbcae4ef2421a))
+- inverted the scale on similarity threshold ([1edbd21](https://github.com/ubiquity-os-marketplace/command-ask/commit/1edbd217430ad490e5122707914aaf08d88647ab))
+- knip ([e0b252a](https://github.com/ubiquity-os-marketplace/command-ask/commit/e0b252a211998ca10ffd42d3f31e47b503f53aeb))
+- knip ([d9cd798](https://github.com/ubiquity-os-marketplace/command-ask/commit/d9cd7985a6116e0eb75e15e4f617df8cc2ec806e))
+- knip ([6c7f136](https://github.com/ubiquity-os-marketplace/command-ask/commit/6c7f136043b2de6abe901d15dbd41ef93bcae229))
+- logger error ([6fdf794](https://github.com/ubiquity-os-marketplace/command-ask/commit/6fdf794d6a409434302ca79b5bc22ac06021d7e0))
+- manifest ([cca9dec](https://github.com/ubiquity-os-marketplace/command-ask/commit/cca9decc336b9c88cc71efd0a7231e6d122c6aa7))
+- package.json missing issue ([30f876c](https://github.com/ubiquity-os-marketplace/command-ask/commit/30f876ca3925ef15e9e094af38bb44f64de302af))
+- project setup and supabase setup ([e133616](https://github.com/ubiquity-os-marketplace/command-ask/commit/e1336160e5a7b495215d7bfc20227d7656d6382a))
+- remove file ignores ([195e8e8](https://github.com/ubiquity-os-marketplace/command-ask/commit/195e8e818d7215fee62d1c9f734c5a47be2d696a))
+- remove unused code ([674f3cf](https://github.com/ubiquity-os-marketplace/command-ask/commit/674f3cff1100d206956d0bd8f8dacb85f8a9db5d))
+- removed context optimizer ([4e928df](https://github.com/ubiquity-os-marketplace/command-ask/commit/4e928df4cc85b1aee0a7fa1acd9af946d9ff28f1))
+- removed jest commas ([b417cd5](https://github.com/ubiquity-os-marketplace/command-ask/commit/b417cd5780189453cdf00da15ce790592ba04257))
+- setup bun ([7b58b81](https://github.com/ubiquity-os-marketplace/command-ask/commit/7b58b813049b5d66a13ec7fd37b6b73f84541251))
+- tests ([3d908bd](https://github.com/ubiquity-os-marketplace/command-ask/commit/3d908bd45ace706c0b5e23ffb094963e2b4a3b5d))
+- tests ([c6873b1](https://github.com/ubiquity-os-marketplace/command-ask/commit/c6873b18b9a169b63c390ee444fdaebc91e31d3f))
+- tests ([baec08a](https://github.com/ubiquity-os-marketplace/command-ask/commit/baec08a95be7349a69ef6b043cbb4fd6b3b80943))
+- tests ([dd4c334](https://github.com/ubiquity-os-marketplace/command-ask/commit/dd4c33432a27914049cb76b30d3f6220591337ba))
+- tests ([51454d4](https://github.com/ubiquity-os-marketplace/command-ask/commit/51454d4338b07647b2bb5184f57e824a6a20f6a1))
+- tests and knip ([e37f585](https://github.com/ubiquity-os-marketplace/command-ask/commit/e37f585488e673bf88a03cb919bedb58fb5cb52b))
+- type rename and add tsx ([0f00dd4](https://github.com/ubiquity-os-marketplace/command-ask/commit/0f00dd4d62fbff74ca315ab4c97e975a30213f0a))
+- update wrangler.toml ([63099fd](https://github.com/ubiquity-os-marketplace/command-ask/commit/63099fd98e40b963f3b0576415999d1676b6f53e))
+- updated action ([fd0bda0](https://github.com/ubiquity-os-marketplace/command-ask/commit/fd0bda0197da24cd9deb4f767ffa8a0ef285a983))
+- updated compute.yml ([4d9e674](https://github.com/ubiquity-os-marketplace/command-ask/commit/4d9e674db05a8dd4b526adc556e98b2ecb2fe56e))
+- updated compute.yml ([a0c5ea5](https://github.com/ubiquity-os-marketplace/command-ask/commit/a0c5ea580ed7054f5b6a955886a319bf2afdcdad))
+- updated compute.yml ([aa309dd](https://github.com/ubiquity-os-marketplace/command-ask/commit/aa309dd8d9cbc79bc7d17410a0a402dc3e58c7a3))
+- updated logic for fetching linked pr ([c4b43d7](https://github.com/ubiquity-os-marketplace/command-ask/commit/c4b43d7ae9e109e807bdc7f41c1aae989f260b6b))

evals/data/eval-gold-responses.json

@@ -0,0 +1,25 @@
+{
+  "issueResponses": [
+    {
+      "scenario": "manifest.name mismatch ",
+      "issue": {
+        "body": "Manifests need to be updated so the name matches the intended name, which is the name of the repo it lives in.\n\nAny mismatch in manifest.name and the plugin repo, and we will not be able to install those plugins. The config will look like this:\n\nThis is because the worker URL contains the repo name, and we use that to match against manifest.name.",
+        "number": 27,
+        "html_url": "https://github.com/ubiquity-os/ubiquity-os-plugin-installer/issues/27/",
+        "question": "/ask could you please provide a summary of the issue ?"
+      },
+      "expectedResponse": "The manifest.name should match the name of the repo it lives in. This is because the worker URL contains the repo name, and we use that to match against manifest.name.",
+      "sender": {
+        "login": "sshivaditya2019",
+        "type": "User"
+      },
+      "repository": {
+        "name": "ubiquity-os-plugin-installer",
+        "owner": {
+          "login": "ubiquity-os",
+          "type": "Organization"
+        }
+      }
+    }
+  ]
+}

evals/handlers/setup-context.ts

@@ -0,0 +1,114 @@
+import { SupabaseClient } from "@supabase/supabase-js";
+import { createAdapters } from "../../src/adapters";
+import { CommentSimilaritySearchResult } from "../../src/adapters/supabase/helpers/comment";
+import { IssueSimilaritySearchResult } from "../../src/adapters/supabase/helpers/issues";
+import { fetchRepoLanguageStats, fetchRepoDependencies } from "../../src/handlers/ground-truths/chat-bot";
+import { findGroundTruths } from "../../src/handlers/ground-truths/find-ground-truths";
+import { logger } from "../../src/helpers/errors";
+import { formatChatHistory } from "../../src/helpers/format-chat-history";
+import { recursivelyFetchLinkedIssues } from "../../src/helpers/issue-fetching";
+import { Context } from "../../src/types";
+import { VoyageAIClient } from "voyageai";
+import OpenAI from "openai";
+
+const SEPERATOR = "######################################################\n";
+
+export interface FetchContext {
+  rerankedText: string[];
+  formattedChat: string[];
+  groundTruths: string[];
+}
+
+export interface EvalClients {
+  supabase: SupabaseClient;
+  voyage: VoyageAIClient;
+  openai: OpenAI;
+}
+
+export const initAdapters = (context: Context, clients: EvalClients): Context => {
+  const adapters = createAdapters(clients.supabase, clients.voyage, clients.openai, context);
+  context.adapters = adapters;
+
+  // Update adapter contexts
+  Object.values(adapters).forEach((adapterGroup) => {
+    Object.values(adapterGroup).forEach((adapter) => {
+      if (adapter && typeof adapter === "object" && "context" in adapter) {
+        adapter.context = context;
+      }
+    });
+  });
+  return context;
+};
+
+export async function fetchContext(context: Context, question: string): Promise<FetchContext> {
+  const {
+    config: { similarityThreshold },
+    adapters: {
+      supabase: { comment, issue },
+      voyage: { reranker },
+    },
+  } = context;
+  const { specAndBodies, streamlinedComments } = await recursivelyFetchLinkedIssues({
+    context,
+    owner: context.payload.repository.owner.login,
+    repo: context.payload.repository.name,
+  });
+  let formattedChat = await formatChatHistory(context, streamlinedComments, specAndBodies);
+  logger.info(`${formattedChat.join("")}`);
+  // using db functions to find similar comments and issues
+  const [similarComments, similarIssues] = await Promise.all([
+    comment.findSimilarComments(question, 1 - similarityThreshold, ""),
+    issue.findSimilarIssues(question, 1 - similarityThreshold, ""),
+  ]);
+  // combine the similar comments and issues into a single array
+  const similarText = [
+    ...(similarComments?.map((comment: CommentSimilaritySearchResult) => comment.comment_plaintext) || []),
+    ...(similarIssues?.map((issue: IssueSimilaritySearchResult) => issue.issue_plaintext) || []),
+  ];
+  // filter out any empty strings
+  formattedChat = formattedChat.filter((text) => text);
+  // rerank the similar text using voyageai
+  const rerankedText = similarText.length > 0 ? await reranker.reRankResults(similarText, question) : [];
+  // gather structural data about the payload repository
+  const [languages, { dependencies, devDependencies }] = await Promise.all([fetchRepoLanguageStats(context), fetchRepoDependencies(context)]);
+  let groundTruths: string[] = [];
+  if (!languages.length) {
+    groundTruths.push("No languages found in the repository");
+  }
+  if (!Reflect.ownKeys(dependencies).length) {
+    groundTruths.push("No dependencies found in the repository");
+  }
+  if (!Reflect.ownKeys(devDependencies).length) {
+    groundTruths.push("No devDependencies found in the repository");
+  }
+  if (groundTruths.length > 3) {
+    groundTruths = await findGroundTruths(context, "chat-bot", { languages, dependencies, devDependencies });
+  }
+  return {
+    rerankedText,
+    formattedChat,
+    groundTruths,
+  };
+}
+
+export function formattedHistory(fetchContext: FetchContext): string {
+  //Iterate through the formatted chat history and add it to the final formatted chat
+  let formattedChat = "#################### Chat History ####################\n";
+  fetchContext.formattedChat.forEach((chat) => {
+    formattedChat += chat;
+  });
+  formattedChat += SEPERATOR;
+  //Iterate through the reranked text and add it to the final formatted chat
+  formattedChat += "#################### Reranked Text ####################\n";
+  fetchContext.rerankedText.forEach((reranked) => {
+    formattedChat += reranked;
+  });
+  formattedChat += SEPERATOR;
+  //Iterate through the ground truths and add it to the final formatted chat
+  formattedChat += "#################### Ground Truths ####################\n";
+  fetchContext.groundTruths.forEach((truth) => {
+    formattedChat += truth;
+  });
+  formattedChat += SEPERATOR;
+  return formattedChat;
+}