[spelunker] Use gpt-3.5-turbo for step 1 (proposing queries) (microso…

…ft#452)

Also log the time taken in steps 1, 2, 3.

ts/examples/spelunker/design.md

@@ -49,6 +49,7 @@ A user query is handled using the following steps:
 
 - How to measure the quality of the answers? This probably has to be a manual process.
   We might be able to have a standard set of queries about a particular code base and for each potential improvement decide which variant gives the better answer.
+- Anecdotally, GitHub Copilot in VS Code does a better job (possibly because it can see the project docs)
 
 ### Import process open questions
 

ts/examples/spelunker/src/chunkyIndex.ts

@@ -23,6 +23,7 @@ export type NamedIndex = [IndexType, knowLib.TextIndex<string, ChunkId>];
 // A bundle of object stores and indexes etc.
 export class ChunkyIndex {
     chatModel: ChatModel;
+    miniModel: ChatModel; // E.g. gpt-3.5-turbo or gpt-4-mini or o1-mini.
     embeddingModel: TextEmbeddingModel;
     fileDocumenter: FileDocumenter;
     queryMaker: TypeChatJsonTranslator<QuerySpecs>;
@@ -40,12 +41,18 @@ export class ChunkyIndex {
 
     private constructor() {
         this.chatModel = openai.createChatModelDefault("spelunkerChat");
+        this.miniModel = openai.createChatModel(
+            "GPT_35_TURBO",
+            undefined,
+            undefined,
+            ["spelunkerMini"],
+        );
         this.embeddingModel = knowLib.createEmbeddingCache(
             openai.createEmbeddingModel(),
             1000,
         );
         this.fileDocumenter = createFileDocumenter(this.chatModel);
-        this.queryMaker = createQueryMaker(this.chatModel);
+        this.queryMaker = createQueryMaker(this.miniModel);
         this.answerMaker = createAnswerMaker(this.chatModel);
     }
 

ts/examples/spelunker/src/queryInterface.ts

@@ -652,6 +652,8 @@ async function proposeQueries(
     io: iapp.InteractiveIo,
     queryOptions: QueryOptions,
 ): Promise<QuerySpecs | undefined> {
+    const t0 = Date.now();
+
     const promptPreamble = makeQueryMakerPrompt(recentAnswers);
     if (queryOptions.verbose) {
         for (const section of promptPreamble) {
@@ -663,7 +665,11 @@ async function proposeQueries(
         promptPreamble,
     );
     if (!result.success) {
-        writeError(io, `[Error: ${result.message}]`);
+        const t1 = Date.now();
+        writeError(
+            io,
+            `[Error: ${result.message} in ${((t1 - t0) * 0.001).toFixed(3)} seconds]`,
+        );
         return undefined;
     }
     const specs = result.data;
@@ -674,6 +680,12 @@ async function proposeQueries(
             util.inspect(specs, { depth: null, colors: true, compact: false }),
         );
     }
+    const t1 = Date.now();
+    writeNote(
+        io,
+        `[proposeQueries took ${((t1 - t0) * 0.001).toFixed(3)} seconds]`,
+    );
+
     return specs;
 }
 
@@ -683,6 +695,8 @@ async function runIndexQueries(
     io: iapp.InteractiveIo,
     queryOptions: QueryOptions,
 ): Promise<Map<ChunkId, ScoredItem<ChunkId>> | undefined> {
+    const t0 = Date.now();
+
     const chunkIdScores: Map<ChunkId, ScoredItem<ChunkId>> = new Map(); // Record score of each chunk id.
     const totalNumChunks = await chunkyIndex.chunkFolder.size(); // Nominator in IDF calculation.
 
@@ -765,6 +779,12 @@ async function runIndexQueries(
         );
     }
 
+    const t1 = Date.now();
+    writeNote(
+        io,
+        `[runIndexQueries took ${((t1 - t0) * 0.001).toFixed(3)} seconds]`,
+    );
+
     return chunkIdScores;
 }
 
@@ -776,6 +796,8 @@ async function generateAnswer(
     io: iapp.InteractiveIo,
     queryOptions: QueryOptions,
 ): Promise<AnswerSpecs | undefined> {
+    const t0 = Date.now();
+
     // Step 3a: Compute array of ids sorted by score, truncated to some limit.
     const scoredChunkIds: ScoredItem<ChunkId>[] = Array.from(
         chunkIdScores.values(),
@@ -831,6 +853,12 @@ async function generateAnswer(
         );
     }
 
+    const t1 = Date.now();
+    writeNote(
+        io,
+        `[generateAnswer took ${((t1 - t0) * 0.001).toFixed(3)} seconds]`,
+    );
+
     return answerResult.data;
 }