Teacher tool: first pass of nested validators

common-docs/teachertool/catalog-shared.json

@@ -35,6 +35,18 @@
             "use": "variable_accessed",
             "template": "At least one variable is accessed",
             "docPath": "/teachertool"
+        },
+        {
+            "id": "8E6F9A92-2D22-4BB0-A77A-BD7490D3CEF7",
+            "use": "variable_set_random",
+            "template": "A custom variable's value is set to a random number",
+            "docPath": "/teachertool"
+        },
+        {
+            "id": "E8DAD360-F4AB-4E6B-9981-C14BDEE1295B",
+            "use": "device_random_used",
+            "template": "The math random number generator block is used",
+            "docPath": "/teachertool"
         }
     ]
 }

common-docs/teachertool/validator-plans-shared.json

@@ -96,6 +96,33 @@
                     }
                 }
             ]
+        },
+        {
+            ".desc": "A custom variable's value is set to a random number",
+            "name": "variable_set_random",
+            "threshold": 1,
+            "checks": [
+                {
+                    "validator": "blocksExist",
+                    "blockCounts": {
+                        "variables_set": 1
+                    },
+                    "childValidatorPlans": ["device_random_used"]
+                }
+            ]
+        },
+        {
+            ".desc": "Random number block used in project",
+            "name": "device_random_used",
+            "threshold": 1,
+            "checks": [
+                {
+                    "validator": "blocksExist",
+                    "blockCounts": {
+                        "device_random": 1
+                    }
+                }
+            ]
         }
     ]
 }

localtypings/pxteditor.d.ts

@@ -298,6 +298,7 @@ declare namespace pxt.editor {
     export interface EditorMessageRunEvalRequest extends EditorMessageRequest {
         action: "runeval";
         validatorPlan: pxt.blocks.ValidatorPlan;
+        planBank: pxt.blocks.ValidatorPlan[];
     }
 
     export interface EditorMessageRenderBlocksResponse {

localtypings/validatorPlan.d.ts

@@ -12,6 +12,7 @@ declare namespace pxt.blocks {
     // Each type of validation will need to implement its own ValidatorCheck based on this.
     export interface ValidatorCheckBase {
         validator: string;
+        childValidatorPlans?: string[]
     }
 
     // Inputs for "Blocks Exist" validation.

pxteditor/code-validation/runValidatorPlanAsync.ts

@@ -5,33 +5,48 @@ import { validateBlocksInSetExist } from "./validateBlocksInSetExist";
 import { validateBlockCommentsExist } from "./validateCommentsExist";
 import { validateSpecificBlockCommentsExist } from "./validateSpecificBlockCommentsExist";
 
-const maxConcurrentChecks = 4;
-
-export async function runValidatorPlanAsync(usedBlocks: Blockly.Block[], plan: pxt.blocks.ValidatorPlan): Promise<boolean> {
-    // Each plan can have multiple checks it needs to run.
-    // Run all of them in parallel, and then check if the number of successes is greater than the specified threshold.
-    // TBD if it's faster to run in parallel without short-circuiting once the threshold is reached, or if it's faster to run sequentially and short-circuit.
+export function runValidatorPlanAsync(usedBlocks: Blockly.Block[], plan: pxt.blocks.ValidatorPlan, planBank: pxt.blocks.ValidatorPlan[]): boolean {
     const startTime = Date.now();
+    let checksSucceeded = 0;
+    let successfulBlocks: Blockly.Block[] = [];
 
-    const checkRuns = pxt.Util.promisePoolAsync(maxConcurrentChecks, plan.checks, async (check: pxt.blocks.ValidatorCheckBase): Promise<boolean> => {
+    for (const check of plan.checks) {
+        let checkPassed = false;
         switch (check.validator) {
             case "blocksExist":
-                return runBlocksExistValidation(usedBlocks, check as pxt.blocks.BlocksExistValidatorCheck);
+                [successfulBlocks, checkPassed] = [...runBlocksExistValidation(usedBlocks, check as pxt.blocks.BlocksExistValidatorCheck)];
+                break;
             case "blockCommentsExist":
-                return runValidateBlockCommentsExist(usedBlocks, check as pxt.blocks.BlockCommentsExistValidatorCheck);
+                checkPassed = runValidateBlockCommentsExist(usedBlocks, check as pxt.blocks.BlockCommentsExistValidatorCheck);
+                break;
             case "specificBlockCommentsExist":
-                return runValidateSpecificBlockCommentsExist(usedBlocks, check as pxt.blocks.SpecificBlockCommentsExistValidatorCheck);
+                checkPassed = runValidateSpecificBlockCommentsExist(usedBlocks, check as pxt.blocks.SpecificBlockCommentsExistValidatorCheck);
+                break;
             case "blocksInSetExist":
-                return runBlocksInSetExistValidation(usedBlocks, check as pxt.blocks.BlocksInSetExistValidatorCheck);
+                [successfulBlocks, checkPassed] = [...runBlocksInSetExistValidation(usedBlocks, check as pxt.blocks.BlocksInSetExistValidatorCheck)];
+                break;
             default:
                 pxt.debug(`Unrecognized validator: ${check.validator}`);
-                return false;
+                checkPassed = false;
+                break;
         }
-    });
 
-    const results = await checkRuns;
-    const successCount = results.filter((r) => r).length;
-    const passed = successCount >= plan.threshold;
+        if (checkPassed && check.childValidatorPlans) {
+            for (const planName of check.childValidatorPlans) {
+                let timesPassed = 0;
+                for (const parentBlock of successfulBlocks) {
+                    const blocksToUse = parentBlock.getChildren(true);
+                    const childPlan = planBank.find((plan) => plan.name === planName);
+                    const childPassed = runValidatorPlanAsync(blocksToUse, childPlan, planBank);
+                    timesPassed += childPassed ? 1 : 0;
+                }
+                checkPassed = timesPassed > 0;
+            }
+        }
+        checksSucceeded += checkPassed ? 1 : 0;
+    }
+
+    const passed = checksSucceeded >= plan.threshold;
 
     pxt.tickEvent("validation.evaluation_complete", {
         plan: plan.name,
@@ -42,13 +57,11 @@ export async function runValidatorPlanAsync(usedBlocks: Blockly.Block[], plan: p
     return passed;
 }
 
-function runBlocksExistValidation(usedBlocks: Blockly.Block[], inputs: pxt.blocks.BlocksExistValidatorCheck): boolean {
+function runBlocksExistValidation(usedBlocks: Blockly.Block[], inputs: pxt.blocks.BlocksExistValidatorCheck): [Blockly.Block[], boolean] {
     const blockResults = validateBlocksExist({ usedBlocks, requiredBlockCounts: inputs.blockCounts });
-    const success =
-        blockResults.disabledBlocks.length === 0 &&
-        blockResults.missingBlocks.length === 0 &&
-        blockResults.insufficientBlocks.length === 0;
-    return success;
+    const blockId = Object.keys(inputs.blockCounts)[0];
+    const successfulBlocks = blockResults.successfulBlocks.length ? blockResults.successfulBlocks[0][blockId] : [];
+    return [successfulBlocks, blockResults.passed];
 }
 
 function runValidateBlockCommentsExist(usedBlocks: Blockly.Block[], inputs: pxt.blocks.BlockCommentsExistValidatorCheck): boolean {
@@ -61,7 +74,7 @@ function runValidateSpecificBlockCommentsExist(usedBlocks: Blockly.Block[], inpu
     return blockResults.passed;
 }
 
-function runBlocksInSetExistValidation(usedBlocks: Blockly.Block[], inputs: pxt.blocks.BlocksInSetExistValidatorCheck): boolean {
+function runBlocksInSetExistValidation(usedBlocks: Blockly.Block[], inputs: pxt.blocks.BlocksInSetExistValidatorCheck): [Blockly.Block[], boolean] {
     const blockResults = validateBlocksInSetExist({ usedBlocks, blockIdsToCheck: inputs.blocks, count: inputs.count });
-    return blockResults.passed;
+    return  [blockResults.successfulBlocks, blockResults.passed];
 }

pxteditor/code-validation/validateBlocksExist.ts

@@ -6,17 +6,25 @@ export function validateBlocksExist({ usedBlocks, requiredBlockCounts }: {
     missingBlocks: string[],
     disabledBlocks: string[],
     insufficientBlocks: string[],
+    successfulBlocks: pxt.Map<Blockly.Block[]>[],
+    passed: boolean
 } {
     let missingBlocks: string[] = [];
     let disabledBlocks: string[] = [];
     let insufficientBlocks: string[] = [];
+    let successfulBlocks: pxt.Map<Blockly.Block[]>[]  = [];
     const userBlocksEnabledByType = usedBlocks?.reduce((acc: pxt.Map<number>, block) => {
         acc[block.type] = (acc[block.type] || 0) + (block.isEnabled() ? 1 : 0);
         return acc;
     }, {});
 
     for (const [requiredBlockId, requiredCount] of Object.entries(requiredBlockCounts || {})) {
         const countForBlock = userBlocksEnabledByType[requiredBlockId];
+        const passedBlocks = usedBlocks.filter((block) => block.type === requiredBlockId);
+        if (passedBlocks.length > 0) {
+            successfulBlocks.push({ [requiredBlockId]: passedBlocks})
+        }
+
         if (countForBlock === undefined) {
             // user did not use a specific block
             missingBlocks.push(requiredBlockId);
@@ -29,9 +37,16 @@ export function validateBlocksExist({ usedBlocks, requiredBlockCounts }: {
         }
     }
 
+    const passed =
+        missingBlocks.length === 0 &&
+        disabledBlocks.length === 0 &&
+        insufficientBlocks.length === 0;
+
     return {
         missingBlocks,
         disabledBlocks,
         insufficientBlocks,
+        successfulBlocks,
+        passed
     }
 }

pxteditor/editorcontroller.ts

@@ -157,10 +157,11 @@ export function bindEditorMessages(getEditorAsync: () => Promise<IProjectView>)
                             case "runeval": {
                                 const evalmsg = data as pxt.editor.EditorMessageRunEvalRequest;
                                 const plan = evalmsg.validatorPlan;
+                                const planBank = evalmsg.planBank;
                                 return Promise.resolve()
                                     .then(() => {
                                         const blocks = projectView.getBlocks();
-                                        return runValidatorPlanAsync(blocks, plan)})
+                                        return runValidatorPlanAsync(blocks, plan, planBank)})
                                     .then (results => {
                                         resp = { result: results };
                                     });

teachertool/src/services/makecodeEditorService.ts

@@ -92,7 +92,8 @@ export async function setHighContrastAsync(on: boolean) {
 }
 
 export async function runValidatorPlanAsync(
-    validatorPlan: pxt.blocks.ValidatorPlan
+    validatorPlan: pxt.blocks.ValidatorPlan,
+    planBank: pxt.blocks.ValidatorPlan[]
 ): Promise<pxt.blocks.EvaluationResult | undefined> {
     let evalResults = undefined;
 
@@ -101,6 +102,7 @@ export async function runValidatorPlanAsync(
             type: "pxteditor",
             action: "runeval",
             validatorPlan: validatorPlan,
+            planBank: planBank
         } as pxt.editor.EditorMessageRunEvalRequest);
         const result = response as pxt.editor.EditorMessageResponse;
         validateResponse(result, true); // Throws on failure

teachertool/src/transforms/runEvaluateAsync.ts

@@ -45,14 +45,21 @@ export async function runEvaluateAsync() {
             new Promise(async resolve => {
                 dispatch(Actions.setEvalResult(criteriaInstance.instanceId, CriteriaEvaluationResult.InProgress));
 
+                const loadedValidatorPlans = teacherTool.validatorPlans;
+                if (!loadedValidatorPlans) {
+                    logError(ErrorCode.validatorPlansNotFound, "Attempting to evaluate criteria without any plans");
+                    dispatch(Actions.clearEvalResult(criteriaInstance.instanceId));
+                    return resolve(false);
+                }
+
                 const plan = generateValidatorPlan(criteriaInstance);
 
                 if (!plan) {
                     dispatch(Actions.clearEvalResult(criteriaInstance.instanceId));
                     return resolve(false);
                 }
 
-                const planResult = await runValidatorPlanAsync(plan);
+                const planResult = await runValidatorPlanAsync(plan, loadedValidatorPlans);
 
                 if (planResult) {
                     dispatch(

teachertool/src/types/errorCode.ts

@@ -15,4 +15,5 @@ export enum ErrorCode {
     unableToReadRubricFile = "unableToReadRubricFile",
     localStorageReadError = "localStorageReadError",
     localStorageWriteError = "localStorageWriteError",
+    validatorPlansNotFound = "validatorPlansNotFound"
 }