One approach to having a validator plan for AI questions, still sever…

…al to do items.
microsoft · Mar 28, 2024 · 2372ed4 · 2372ed4
1 parent a344259
commit 2372ed4
Show file tree

Hide file tree

Showing 11 changed files with 126 additions and 11 deletions.
diff --git a/common-docs/teachertool/test/catalog-shared.json b/common-docs/teachertool/test/catalog-shared.json
@@ -37,15 +37,27 @@
         },
         {
             "id": "499F3572-E655-4DEE-953B-5F26BF0191D7",
-            "use": "block_used_n_times",
-            "template": "Long String: ${question}",
-            "description": "This is just a test for long string inputs.",
+            "use": "ai_question",
+            "template": "Ask Copilot: ${question}",
+            "description": "Experimental: AI outputs are inherently nondeterministic and may not be accurate. Use with caution and always review responses.",
             "docPath": "/teachertool",
             "params": [
                 {
                     "name": "question",
                     "type": "longString",
-                    "paths": ["checks[0].blockCounts[0].blockId"]
+                    "paths": ["checks[0].question"]
+                },
+                {
+                    "name": "shareid",
+                    "type": "system",
+                    "default": "SHAREID",
+                    "paths": ["checks[0].shareId"]
+                },
+                {
+                    "name": "target",
+                    "type": "system",
+                    "default": "TARGET",
+                    "paths": ["checks[0].target"]
                 }
             ]
         },

diff --git a/common-docs/teachertool/test/validator-plans-shared.json b/common-docs/teachertool/test/validator-plans-shared.json
@@ -27,6 +27,19 @@
                     "count": 0
                 }
             ]
+        },
+        {
+            ".desc": "Ask Copilot a question",
+            "name": "ai_question",
+            "threshold": -1,
+            "checks": [
+                {
+                    "validator": "aiQuestion",
+                    "question": "",
+                    "shareId": "",
+                    "target": ""
+                }
+            ]
         }
     ]
 }
diff --git a/localtypings/validatorPlan.d.ts b/localtypings/validatorPlan.d.ts
@@ -41,7 +41,8 @@ declare namespace pxt.blocks {
     }
 
     export interface EvaluationResult {
-        result: boolean;
+        result?: boolean;
+        notes?: string;
     }
 
     export interface BlockFieldValueExistsCheck extends ValidatorCheckBase {
@@ -50,4 +51,11 @@ declare namespace pxt.blocks {
         fieldValue: string;
         blockType: string;
     }
+
+    export interface AiQuestionValidatorCheck extends ValidatorCheckBase {
+        validator: "aiQuestion";
+        question: string;
+        shareId: string;
+        target: string; // TODO thsparks : Just look this up from the share id within deepprompt itself? Or in our backend?
+    }
 }
diff --git a/pxteditor/code-validation/runValidatorPlan.ts b/pxteditor/code-validation/runValidatorPlan.ts
@@ -9,10 +9,19 @@ import { validateBlockCommentsExist } from "./validateCommentsExist";
 import { validateSpecificBlockCommentsExist } from "./validateSpecificBlockCommentsExist";
 import { getNestedChildBlocks } from "./getNestedChildBlocks";
 
-export function runValidatorPlan(usedBlocks: Blockly.Block[], plan: pxt.blocks.ValidatorPlan, planLib: pxt.blocks.ValidatorPlan[]): boolean {
+export async function runValidatorPlan(usedBlocks: Blockly.Block[], plan: pxt.blocks.ValidatorPlan, planLib: pxt.blocks.ValidatorPlan[]): Promise<pxt.blocks.EvaluationResult> {
     const startTime = Date.now();
     let checksSucceeded = 0;
     let successfulBlocks: Blockly.Block[] = [];
+    let notes: string = undefined;
+
+    function addToNote(note: string) {
+        if (!notes) {
+            notes = note;
+        } else {
+            notes += `\n${note}`;
+        }
+    }
 
     for (const check of plan.checks) {
         let checkPassed = false;
@@ -32,6 +41,10 @@ export function runValidatorPlan(usedBlocks: Blockly.Block[], plan: pxt.blocks.V
             case "blockFieldValueExists":
                 [successfulBlocks, checkPassed] = [...runBlockFieldValueExistsValidation(usedBlocks, check as pxt.blocks.BlockFieldValueExistsCheck)];
                 break;
+            case "aiQuestion":
+                const response = await runAiQuestionValidation(check as pxt.blocks.AiQuestionValidatorCheck);
+                addToNote(response);
+                break;
             default:
                 pxt.debug(`Unrecognized validator: ${check.validator}`);
                 checkPassed = false;
@@ -53,15 +66,16 @@ export function runValidatorPlan(usedBlocks: Blockly.Block[], plan: pxt.blocks.V
         checksSucceeded += checkPassed ? 1 : 0;
     }
 
-    const passed = checksSucceeded >= plan.threshold;
+    // If threshold is -1 then pass/fail does not apply.
+    const passed = plan.threshold < 0 ? undefined : checksSucceeded >= plan.threshold;
 
     pxt.tickEvent("validation.evaluation_complete", {
         plan: plan.name,
         durationMs: Date.now() - startTime,
         passed: `${passed}`,
     });
 
-    return passed;
+    return { result: passed, notes };
 }
 
 function runBlocksExistValidation(usedBlocks: Blockly.Block[], inputs: pxt.blocks.BlocksExistValidatorCheck): [Blockly.Block[], boolean] {
@@ -104,3 +118,34 @@ function runBlockFieldValueExistsValidation(usedBlocks: Blockly.Block[], inputs:
     });
     return  [blockResults.successfulBlocks, blockResults.passed];
 }
+
+
+// TODO thsparks - do we have a shared backend requests location in pxteditor? If not, should we make one?
+export async function askCopilotQuestion(shareId: string, target: string, question: string): Promise<string> {
+    // TODO thsparks - any kind of retry logic, error handling?
+    // TODO thsparks - use pxt.Cloud.apiRoot instead of my staging endpoint.
+    const url = `https://makecode-app-backend-ppe-thsparks.azurewebsites.net/api/copilot/question`;
+    const data = { id: shareId, target, question }
+    const request = await fetch(url, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(data),
+    });
+    const response = await request.text();
+
+    if (!response) {
+        throw new Error("Unable to reach copilot service.");
+    } else {
+        return response;
+    }
+}
+
+
+async function runAiQuestionValidation(inputs: pxt.blocks.AiQuestionValidatorCheck): Promise<string> {
+    // TODO thsparks - remove debug logs.
+    console.log(`Asking question: '${inputs.question}' on '${inputs.target}' project with shareId: '${inputs.shareId}'`);
+    const response = await askCopilotQuestion(inputs.shareId, inputs.target, inputs.question);
+    console.log(`Response: ${response}`);
+
+    return response;
+}
diff --git a/pxteditor/editorcontroller.ts b/pxteditor/editorcontroller.ts
@@ -183,7 +183,7 @@ export function bindEditorMessages(getEditorAsync: () => Promise<IProjectView>)
                                         const blocks = projectView.getBlocks();
                                         return runValidatorPlan(blocks, plan, planLib)})
                                     .then (results => {
-                                        resp = { result: results };
+                                        resp = results;
                                     });
                             }
                             case "gettoolboxcategories": {

diff --git a/teachertool/src/services/autorunService.ts b/teachertool/src/services/autorunService.ts
@@ -5,6 +5,8 @@ import { runEvaluateAsync } from "../transforms/runEvaluateAsync";
 let autorunTimer: NodeJS.Timeout | null = null;
 
 export function poke() {
+    // TODO thsparks - somehow don't autorun for ai? It's expensive...
+
     if (autorunTimer) {
         clearTimeout(autorunTimer);
     }

diff --git a/teachertool/src/transforms/runEvaluateAsync.ts b/teachertool/src/transforms/runEvaluateAsync.ts
@@ -10,6 +10,9 @@ import { showToast } from "./showToast";
 import { setActiveTab } from "./setActiveTab";
 import { setEvalResultOutcome } from "./setEvalResultOutcome";
 import jp from "jsonpath";
+import { getSystemParameter } from "../utils/getSystemParameter";
+import { setEvalResult } from "./setEvalResult";
+import { setEvalResultNotes } from "./setEvalResultNotes";
 
 function generateValidatorPlan(
     criteriaInstance: CriteriaInstance,
@@ -47,6 +50,10 @@ function generateValidatorPlan(
             return undefined;
         }
 
+        if (catalogParam.type === "system" && catalogParam.default) { // TODO thsparks - "keyword" instead of default?
+            param.value = getSystemParameter(catalogParam.default, teacherTool);
+        }
+
         if (!param.value) {
             // User didn't set a value for the parameter.
             if (showErrors) {
@@ -97,8 +104,12 @@ export async function runEvaluateAsync(fromUserInteraction: boolean) {
                 const planResult = await runValidatorPlanAsync(plan, loadedValidatorPlans);
 
                 if (planResult) {
-                    const result = planResult.result ? EvaluationStatus.Pass : EvaluationStatus.Fail;
+                    const result = planResult.result === undefined ? EvaluationStatus.CompleteWithNoResult : planResult.result ? EvaluationStatus.Pass : EvaluationStatus.Fail;
+
                     setEvalResultOutcome(criteriaInstance.instanceId, result);
+                    if (planResult.notes) {
+                        setEvalResultNotes(criteriaInstance.instanceId, planResult.notes);
+                    }
                     return resolve(true); // evaluation completed successfully, so return true (regardless of pass/fail)
                 } else {
                     dispatch(Actions.clearEvalResult(criteriaInstance.instanceId));

diff --git a/teachertool/src/transforms/setEvalResult.ts b/teachertool/src/transforms/setEvalResult.ts
@@ -0,0 +1,8 @@
+import { stateAndDispatch } from "../state";
+import * as Actions from "../state/actions";
+import { CriteriaResult } from "../types/criteria";
+
+export function setEvalResult(criteriaId: string, result: CriteriaResult) {
+    const { dispatch } = stateAndDispatch();
+    dispatch(Actions.setEvalResult(criteriaId, result));
+}
diff --git a/teachertool/src/types/criteria.ts b/teachertool/src/types/criteria.ts
@@ -17,7 +17,7 @@ export interface CriteriaInstance {
 }
 
 // Represents a parameter definition in a catalog criteria.
-export type CriteriaParameterType = "string" | "longString" | "number" | "block";
+export type CriteriaParameterType = "string" | "longString" | "number" | "block" | "system";
 export interface CriteriaParameter {
     name: string;
     type: CriteriaParameterType;

diff --git a/teachertool/src/types/errorCode.ts b/teachertool/src/types/errorCode.ts
@@ -23,4 +23,5 @@ export enum ErrorCode {
     fetchJsonDocAsync = "fetchJsonDocAsync",
     missingParameter = "missingParameter",
     selectedBlockWithoutOptions = "selectedBlockWithoutOptions",
+    unrecognizedSystemParameter = "unrecognizedSystemParameter",
 }
diff --git a/teachertool/src/utils/getSystemParameter.ts b/teachertool/src/utils/getSystemParameter.ts
@@ -0,0 +1,15 @@
+import { logError } from "../services/loggingService";
+import { AppState } from "../state/state";
+import { ErrorCode } from "../types/errorCode";
+
+export function getSystemParameter(keyword: string, state: AppState): string | undefined {
+    switch (keyword) {
+        case "SHAREID":
+            return state.projectMetadata?.id;
+        case "TARGET":
+            return state.projectMetadata?.target;
+        default:
+            logError(ErrorCode.unrecognizedSystemParameter, "Unrecognized system parameter", { keyword });
+            return undefined;
+    }
+};