Merge branch 'master' into eanders-ms/tt-autorun

localtypings/pxteditor.d.ts

@@ -1,6 +1,7 @@
 /// <reference path="../built/pxtlib.d.ts" />
 /// <reference path="../built/pxtblocks.d.ts" />
 /// <reference path="./projectheader.d.ts" />
+/// <reference path="./validatorPlan.d.ts" />
 
 declare namespace pxt.editor {
     export interface EditorMessage {

pxtblocks/code-validation/validatorPlans.ts → localtypings/validatorPlan.d.ts

@@ -1,4 +1,6 @@
-namespace pxt.blocks {
+/// <reference path="./pxtpackage.d.ts" />
+
+declare namespace pxt.blocks {
     // A set of validation checks (with inputs) to run for a given criteria.
     export interface ValidatorPlan {
         name: string;
@@ -32,4 +34,8 @@ namespace pxt.blocks {
         blocks: string[];
         count: number;
     }
+
+    export interface EvaluationResult {
+        result: boolean;
+    }
 }

pxteditor/code-validation/index.ts

@@ -0,0 +1,5 @@
+export * from "./runValidatorPlanAsync";
+export * from "./validateBlocksExist";
+export * from "./validateBlocksInSetExist";
+export * from "./validateCommentsExist";
+export * from "./validateSpecificBlockCommentsExist";

pxteditor/code-validation/runValidatorPlanAsync.ts

@@ -0,0 +1,67 @@
+/// <reference path="../../localtypings/validatorPlan.d.ts" />
+
+import { validateBlocksExist } from "./validateBlocksExist";
+import { validateBlocksInSetExist } from "./validateBlocksInSetExist";
+import { validateBlockCommentsExist } from "./validateCommentsExist";
+import { validateSpecificBlockCommentsExist } from "./validateSpecificBlockCommentsExist";
+
+const maxConcurrentChecks = 4;
+
+export async function runValidatorPlanAsync(usedBlocks: Blockly.Block[], plan: pxt.blocks.ValidatorPlan): Promise<boolean> {
+    // Each plan can have multiple checks it needs to run.
+    // Run all of them in parallel, and then check if the number of successes is greater than the specified threshold.
+    // TBD if it's faster to run in parallel without short-circuiting once the threshold is reached, or if it's faster to run sequentially and short-circuit.
+    const startTime = Date.now();
+
+    const checkRuns = pxt.Util.promisePoolAsync(maxConcurrentChecks, plan.checks, async (check: pxt.blocks.ValidatorCheckBase): Promise<boolean> => {
+        switch (check.validator) {
+            case "blocksExist":
+                return runBlocksExistValidation(usedBlocks, check as pxt.blocks.BlocksExistValidatorCheck);
+            case "blockCommentsExist":
+                return runValidateBlockCommentsExist(usedBlocks, check as pxt.blocks.BlockCommentsExistValidatorCheck);
+            case "specificBlockCommentsExist":
+                return runValidateSpecificBlockCommentsExist(usedBlocks, check as pxt.blocks.SpecificBlockCommentsExistValidatorCheck);
+            case "blocksInSetExist":
+                return runBlocksInSetExistValidation(usedBlocks, check as pxt.blocks.BlocksInSetExistValidatorCheck);
+            default:
+                pxt.debug(`Unrecognized validator: ${check.validator}`);
+                return false;
+        }
+    });
+
+    const results = await checkRuns;
+    const successCount = results.filter((r) => r).length;
+    const passed = successCount >= plan.threshold;
+
+    pxt.tickEvent("validation.evaluation_complete", {
+        plan: plan.name,
+        durationMs: Date.now() - startTime,
+        passed: `${passed}`,
+    });
+
+    return passed;
+}
+
+function runBlocksExistValidation(usedBlocks: Blockly.Block[], inputs: pxt.blocks.BlocksExistValidatorCheck): boolean {
+    const blockResults = validateBlocksExist({ usedBlocks, requiredBlockCounts: inputs.blockCounts });
+    const success =
+        blockResults.disabledBlocks.length === 0 &&
+        blockResults.missingBlocks.length === 0 &&
+        blockResults.insufficientBlocks.length === 0;
+    return success;
+}
+
+function runValidateBlockCommentsExist(usedBlocks: Blockly.Block[], inputs: pxt.blocks.BlockCommentsExistValidatorCheck): boolean {
+    const blockResults = validateBlockCommentsExist({ usedBlocks, numRequired: inputs.count });
+    return blockResults.passed;
+}
+
+function runValidateSpecificBlockCommentsExist(usedBlocks: Blockly.Block[], inputs: pxt.blocks.SpecificBlockCommentsExistValidatorCheck): boolean {
+    const blockResults = validateSpecificBlockCommentsExist({ usedBlocks, blockType: inputs.blockType });
+    return blockResults.passed;
+}
+
+function runBlocksInSetExistValidation(usedBlocks: Blockly.Block[], inputs: pxt.blocks.BlocksInSetExistValidatorCheck): boolean {
+    const blockResults = validateBlocksInSetExist({ usedBlocks, blockIdsToCheck: inputs.blocks, count: inputs.count });
+    return blockResults.passed;
+}

pxteditor/code-validation/validateBlocksExist.ts

@@ -0,0 +1,37 @@
+
+export function validateBlocksExist({ usedBlocks, requiredBlockCounts }: {
+    usedBlocks: Blockly.Block[],
+    requiredBlockCounts: pxt.Map<number>,
+}): {
+    missingBlocks: string[],
+    disabledBlocks: string[],
+    insufficientBlocks: string[],
+} {
+    let missingBlocks: string[] = [];
+    let disabledBlocks: string[] = [];
+    let insufficientBlocks: string[] = [];
+    const userBlocksEnabledByType = usedBlocks?.reduce((acc: pxt.Map<number>, block) => {
+        acc[block.type] = (acc[block.type] || 0) + (block.isEnabled() ? 1 : 0);
+        return acc;
+    }, {});
+
+    for (const [requiredBlockId, requiredCount] of Object.entries(requiredBlockCounts || {})) {
+        const countForBlock = userBlocksEnabledByType[requiredBlockId];
+        if (countForBlock === undefined) {
+            // user did not use a specific block
+            missingBlocks.push(requiredBlockId);
+        } else if (!countForBlock) {
+            // all instances of block are disabled
+            disabledBlocks.push(requiredBlockId);
+        } else if (countForBlock < requiredCount) {
+            // instances of block exists, but not enough.
+            insufficientBlocks.push(requiredBlockId);
+        }
+    }
+
+    return {
+        missingBlocks,
+        disabledBlocks,
+        insufficientBlocks,
+    }
+}

pxteditor/code-validation/validateBlocksInSetExist.ts

@@ -0,0 +1,23 @@
+// validates that a combination of blocks in the set satisfies the required count
+// returns the blocks that make the validator pass
+export function validateBlocksInSetExist({ usedBlocks, blockIdsToCheck, count, requireUnique }: {
+    usedBlocks: Blockly.Block[],
+    blockIdsToCheck: string[],
+    count: number,
+    requireUnique?: boolean
+}): {
+    successfulBlocks: Blockly.Block[],
+    passed: boolean
+} {
+    const successfulBlocks = [];
+    const enabledBlocks = usedBlocks.filter((block) => block.isEnabled());
+    for (const block of blockIdsToCheck) {
+        const blockInstances = enabledBlocks.filter((b) => b.type === block);
+        if (requireUnique && blockInstances.length >= 1) {
+            successfulBlocks.push(blockInstances[0]);
+        } else {
+            successfulBlocks.push(...blockInstances);
+        }
+    }
+    return { successfulBlocks, passed: successfulBlocks.length >= count };
+}

pxteditor/code-validation/validateCommentsExist.ts

@@ -0,0 +1,12 @@
+// validates that one or more blocks comments are in the project
+// returns the blocks that have comments for teacher tool scenario
+export function validateBlockCommentsExist({ usedBlocks, numRequired }: {
+    usedBlocks: Blockly.Block[],
+    numRequired: number,
+}): {
+    commentedBlocks: Blockly.Block[],
+    passed: boolean
+} {
+    const commentedBlocks = usedBlocks.filter((block) => !!block.getCommentText());
+    return { commentedBlocks, passed: commentedBlocks.length >= numRequired };
+}

pxteditor/code-validation/validateSpecificBlockCommentsExist.ts

@@ -0,0 +1,13 @@
+// validates that all of a specific block type have comments
+// returns the blocks that do not have comments for a tutorial validation scenario
+export function validateSpecificBlockCommentsExist({ usedBlocks, blockType }: {
+    usedBlocks: Blockly.Block[],
+    blockType: string,
+}): {
+    uncommentedBlocks: Blockly.Block[],
+    passed: boolean
+} {
+    const allSpecifcBlocks = usedBlocks.filter((block) => block.type === blockType);
+    const uncommentedBlocks = allSpecifcBlocks.filter((block) => !block.getCommentText());
+    return { uncommentedBlocks, passed: uncommentedBlocks.length === 0 };
+}

pxteditor/editorcontroller.ts

@@ -1,5 +1,6 @@
 /// <reference path="../localtypings/pxteditor.d.ts" />
 
+import { runValidatorPlanAsync } from "./code-validation/runValidatorPlanAsync";
 import IProjectView = pxt.editor.IProjectView;
 
 const pendingRequests: pxt.Map<{
@@ -159,7 +160,7 @@ export function bindEditorMessages(getEditorAsync: () => Promise<IProjectView>)
                                 return Promise.resolve()
                                     .then(() => {
                                         const blocks = projectView.getBlocks();
-                                        return pxt.blocks.runValidatorPlanAsync(blocks, plan)})
+                                        return runValidatorPlanAsync(blocks, plan)})
                                     .then (results => {
                                         resp = { result: results };
                                     });

pxteditor/index.ts

@@ -2,6 +2,7 @@ import * as history from "./history";
 import * as monaco from "./monaco";
 import * as workspace from "./workspace";
 import * as experiments from "./experiments";
+import * as validation from "./code-validation";
 
 export * from "./editor";
 export * from "./editorcontroller";
@@ -16,5 +17,6 @@ export {
     history,
     monaco,
     workspace,
-    experiments
+    experiments,
+    validation
 };