Skip to content

Commit

Permalink
Very hacky wiring up to send notes as a response and enable N/A resul…
Browse files Browse the repository at this point in the history 
…t. Not sure why but unable to access enum values.
  • Loading branch information
@thsparks
thsparks committed Feb 15, 2024
1 parent 0a42686 commit edd158f
Show file tree
Hide file tree
Showing 12 changed files with 272 additions and 182 deletions.
2 changes: 1 addition & 1 deletion common-docs/teachertool/test/catalog-shared.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
{
"id": "499F3572-E655-4DEE-953B-5F26BF0191D7",
"use": "ai_question",
"template": "Ask AI: ${question}",
"template": "Ask Copilot: ${question}",
"description": "Ask the AI a question about the project.",
"docPath": "/teachertool",
"params": [
Expand Down
4 changes: 3 additions & 1 deletion localtypings/pxteditor.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,8 @@ declare namespace pxt.editor {

export interface EditorMessageRunEvalRequest extends EditorMessageRequest {
action: "runeval";
shareId: string, // todo thsparks : ideally, I think we'd get these from the project somehow.
target: string,
validatorPlan: pxt.blocks.ValidatorPlan;
}

Expand Down Expand Up @@ -1204,4 +1206,4 @@ declare namespace pxt.workspace {

fireEvent?: (ev: pxt.editor.EditorEvent) => void;
}
}
}
8 changes: 7 additions & 1 deletion localtypings/validatorPlan.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,13 @@ declare namespace pxt.blocks {
question: string;
}

export enum EvaluationResultStatus {
Success = 0,
Failure = 1,
NotApplicable = 2,
}
export interface EvaluationResult {
result: boolean;
notes: string | undefined;
result: EvaluationResultStatus;
}
}
324 changes: 183 additions & 141 deletions pxteditor/code-validation/runValidatorPlanAsync.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,169 +8,211 @@ import { validateSpecificBlockCommentsExist } from "./validateSpecificBlockComme
const maxConcurrentChecks = 4;

export async function runValidatorPlanAsync(
usedBlocks: Blockly.Block[],
plan: pxt.blocks.ValidatorPlan
): Promise<boolean> {
// Each plan can have multiple checks it needs to run.
// Run all of them in parallel, and then check if the number of successes is greater than the specified threshold.
// TBD if it's faster to run in parallel without short-circuiting once the threshold is reached, or if it's faster to run sequentially and short-circuit.
const startTime = Date.now();

const checkRuns = pxt.Util.promisePoolAsync(
maxConcurrentChecks,
plan.checks,
async (check: pxt.blocks.ValidatorCheckBase): Promise<boolean> => {
switch (check.validator) {
case "blocksExist":
return runBlocksExistValidation(
usedBlocks,
check as pxt.blocks.BlocksExistValidatorCheck
);
case "blockCommentsExist":
return runValidateBlockCommentsExist(
usedBlocks,
check as pxt.blocks.BlockCommentsExistValidatorCheck
);
case "specificBlockCommentsExist":
return runValidateSpecificBlockCommentsExist(
usedBlocks,
check as pxt.blocks.SpecificBlockCommentsExistValidatorCheck
);
case "blocksInSetExist":
return runBlocksInSetExistValidation(
usedBlocks,
check as pxt.blocks.BlocksInSetExistValidatorCheck
);
case "aiQuestion":
return runAiQuestionValidation(
check as pxt.blocks.AiQuestionValidatorCheck
);
default:
pxt.debug(`Unrecognized validator: ${check.validator}`);
return false;
}
}
);

const results = await checkRuns;
const successCount = results.filter((r) => r).length;
const passed = successCount >= plan.threshold;

pxt.tickEvent("validation.evaluation_complete", {
plan: plan.name,
durationMs: Date.now() - startTime,
passed: `${passed}`,
});

return passed;
shareId: string,
target: string,
usedBlocks: Blockly.Block[],
plan: pxt.blocks.ValidatorPlan
): Promise<pxt.blocks.EvaluationResult> {
// Each plan can have multiple checks it needs to run.
// Run all of them in parallel, and then check if the number of successes is greater than the specified threshold.
// TBD if it's faster to run in parallel without short-circuiting once the threshold is reached, or if it's faster to run sequentially and short-circuit.
const startTime = Date.now();

const checkRuns = pxt.Util.promisePoolAsync(
maxConcurrentChecks,
plan.checks,
async (check: pxt.blocks.ValidatorCheckBase): Promise<pxt.blocks.EvaluationResult> => {
let success: boolean | undefined = undefined;
let notes: string | undefined = undefined;
switch (check.validator) {
case "blocksExist":
success = runBlocksExistValidation(usedBlocks, check as pxt.blocks.BlocksExistValidatorCheck);
break;
case "blockCommentsExist":
success = runValidateBlockCommentsExist(
usedBlocks,
check as pxt.blocks.BlockCommentsExistValidatorCheck
);
break;
case "specificBlockCommentsExist":
success = runValidateSpecificBlockCommentsExist(
usedBlocks,
check as pxt.blocks.SpecificBlockCommentsExistValidatorCheck
);
break;
case "blocksInSetExist":
success = runBlocksInSetExistValidation(
usedBlocks,
check as pxt.blocks.BlocksInSetExistValidatorCheck
);
break;
case "aiQuestion":
notes = await runAiQuestionValidation(
shareId,
target,
check as pxt.blocks.AiQuestionValidatorCheck
);
break;
default:
pxt.debug(`Unrecognized validator: ${check.validator}`);
}

let result = 2;
if (success === true) {
result = 0;
} else if (success === false) {
result = 1;
}

return { result, notes };
}
);

const results = await checkRuns;
const successCount = results.filter((r) => r.result === 0).length;
const hasNotApplicable =
results.filter((r) => r.result === 2).length > 0;
const passed = successCount >= plan.threshold;

// todo thsparks - how to actually combine these?
const notes = results.map((r) => r.notes).join("\n");

pxt.tickEvent("validation.evaluation_complete", {
plan: plan.name,
durationMs: Date.now() - startTime,
hasNotApplicable: `${hasNotApplicable}`,
passed: `${passed}`,
});

return {
result: hasNotApplicable
? 2
: passed
? 0
: 1,
notes,
};
}

function runBlocksExistValidation(
usedBlocks: Blockly.Block[],
inputs: pxt.blocks.BlocksExistValidatorCheck
): boolean {
const blockResults = validateBlocksExist({
usedBlocks,
requiredBlockCounts: inputs.blockCounts,
});
const success =
blockResults.disabledBlocks.length === 0 &&
blockResults.missingBlocks.length === 0 &&
blockResults.insufficientBlocks.length === 0;
return success;
function runBlocksExistValidation(usedBlocks: Blockly.Block[], inputs: pxt.blocks.BlocksExistValidatorCheck): boolean {
const blockResults = validateBlocksExist({
usedBlocks,
requiredBlockCounts: inputs.blockCounts,
});
const success =
blockResults.disabledBlocks.length === 0 &&
blockResults.missingBlocks.length === 0 &&
blockResults.insufficientBlocks.length === 0;
return success;
}

function runValidateBlockCommentsExist(
usedBlocks: Blockly.Block[],
inputs: pxt.blocks.BlockCommentsExistValidatorCheck
usedBlocks: Blockly.Block[],
inputs: pxt.blocks.BlockCommentsExistValidatorCheck
): boolean {
const blockResults = validateBlockCommentsExist({
usedBlocks,
numRequired: inputs.count,
});
return blockResults.passed;
const blockResults = validateBlockCommentsExist({
usedBlocks,
numRequired: inputs.count,
});
return blockResults.passed;
}

function runValidateSpecificBlockCommentsExist(
usedBlocks: Blockly.Block[],
inputs: pxt.blocks.SpecificBlockCommentsExistValidatorCheck
usedBlocks: Blockly.Block[],
inputs: pxt.blocks.SpecificBlockCommentsExistValidatorCheck
): boolean {
const blockResults = validateSpecificBlockCommentsExist({
usedBlocks,
blockType: inputs.blockType,
});
return blockResults.passed;
const blockResults = validateSpecificBlockCommentsExist({
usedBlocks,
blockType: inputs.blockType,
});
return blockResults.passed;
}

function runBlocksInSetExistValidation(
usedBlocks: Blockly.Block[],
inputs: pxt.blocks.BlocksInSetExistValidatorCheck
usedBlocks: Blockly.Block[],
inputs: pxt.blocks.BlocksInSetExistValidatorCheck
): boolean {
const blockResults = validateBlocksInSetExist({
usedBlocks,
blockIdsToCheck: inputs.blocks,
count: inputs.count,
});
return blockResults.passed;
const blockResults = validateBlocksInSetExist({
usedBlocks,
blockIdsToCheck: inputs.blocks,
count: inputs.count,
});
return blockResults.passed;
}

async function runAiQuestionValidation(
inputs: pxt.blocks.AiQuestionValidatorCheck
): Promise<boolean> {
shareId: string,
target: string,
inputs: pxt.blocks.AiQuestionValidatorCheck
): Promise<string> {
// TODO thsparks - remove debug logs.
console.log(`Asking question: '${inputs.question}'`);

const data = {
query: inputs.question,
intent: "makecode_evaluation",
context: {
share_id: "_bwCXLsXHjfvK",
target: "microbit",
},
};

// Send a post request to deep prompt endpoint to get the response.
const initialRequest = await fetch(
"http://127.0.0.1:5000/api/v1/query_async",
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(data),
}
);

const requestStatusInfo = await initialRequest.json();

if (!requestStatusInfo) {
throw new Error("Failed to get response from deep prompt.");
}

const result_id = requestStatusInfo.result_id;
if (!result_id) {
throw new Error("No result id returned from deep prompt.");
}

// Poll for response
// TODO thsparks - timeout?
let response;
while (!response) {
const pollResponse = await fetch(`http://127.0.0.1:5000/api/v1/result/${result_id}`);
const pollData = await pollResponse.json();

if (pollData.executed === true) {
response = pollData;
} else {
// Wait for a bit before polling again to avoid spamming the server
await new Promise(resolve => setTimeout(resolve, 1000));
}
const data = {
query: inputs.question,
intent: "makecode_evaluation",
context: {
share_id: shareId,
target: target,
},
};

// // Send a post request to deep prompt endpoint to get the response.
// const initialRequest = await fetch(
// "http://127.0.0.1:5000/api/v1/query_async",
// {
// method: "POST",
// headers: { "Content-Type": "application/json" },
// body: JSON.stringify(data),
// }
// );

// const requestStatusInfo = await initialRequest.json();

// if (!requestStatusInfo) {
// throw new Error("Failed to get response from deep prompt.");
// }

// const result_id = requestStatusInfo.result_id;
// if (!result_id) {
// throw new Error("No result id returned from deep prompt.");
// }

// // Poll for response
// // TODO thsparks - timeout?
// let response;
// while (!response) {
// const pollResponse = await fetch(`http://127.0.0.1:5000/api/v1/result/${result_id}`);
// const pollData = await pollResponse.json();

// if (pollData.executed === true) {
// response = pollData;
// } else {
// // Wait for a bit before polling again to avoid spamming the server
// await new Promise(resolve => setTimeout(resolve, 1000));
// }
// }

// Send a post request to deep prompt endpoint to get the response.
console.log("Sending request with data " + JSON.stringify(data));
const request = await fetch("http://127.0.0.1:5000/api/v1/query", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(data),
});

const response = await request.json();

if (!response) {
throw new Error("Failed to get response from deep prompt.");
}

if (!response.successful) {
// TODO thsparks - can we get more info?
throw new Error("AI was unable to complete the request.");
}
// if (!response.successful) {
// // TODO thsparks - can we get more info?
// throw new Error("AI was unable to complete the request.");
// }

console.log(`Response: ${response.response_text}`)
console.log(`Response: ${response.response_text}`);

return true;
return response.response_text;
}
Loading

0 comments on commit edd158f

Please sign in to comment.