Teacher Tool: Only Auto-Run Modified Criteria (#9958)

With this change, auto-run will only evaluate criteria that have been modified. It will not re-evaluate everything unless the share link changes. If the user presses the run button directly, everything will re-run.

I also changed Pending to NotStarted because I kept confusing Pending with InProgress.

teachertool/src/components/CriteriaEvalResultDropdown.tsx

@@ -15,14 +15,14 @@ const itemIdToCriteriaResult: pxt.Map<EvaluationStatus> = {
     notevaluated: EvaluationStatus.CompleteWithNoResult,
     fail: EvaluationStatus.Fail,
     pass: EvaluationStatus.Pass,
-    pending: EvaluationStatus.Pending,
+    notstarted: EvaluationStatus.NotStarted,
 };
 
 const criteriaResultToItemId: pxt.Map<string> = {
     [EvaluationStatus.CompleteWithNoResult]: "notevaluated",
     [EvaluationStatus.Fail]: "fail",
     [EvaluationStatus.Pass]: "pass",
-    [EvaluationStatus.Pending]: "pending",
+    [EvaluationStatus.NotStarted]: "notstarted",
 };
 
 const dropdownItems: DropdownItem[] = [
@@ -42,7 +42,7 @@ const dropdownItems: DropdownItem[] = [
         label: lf("Looks good!"),
     },
     {
-        id: "pending",
+        id: "notstarted",
         title: lf("not started"),
         label: lf("Not started"),
     },

teachertool/src/transforms/initNewProjectResults.ts

@@ -1,9 +1,9 @@
 import { stateAndDispatch } from "../state";
-import { setEvalResultsPending } from "./setEvalResultsPending";
+import { setEvalResultsToNotStarted } from "./setEvalResultsToNotStarted";
 import * as Actions from "../state/actions";
 
 export function initNewProjectResults() {
     const { dispatch } = stateAndDispatch();
-    setEvalResultsPending({ overwriteExistingEntries: true });
+    setEvalResultsToNotStarted({ overwriteExistingEntries: true });
     dispatch(Actions.clearAllEvalResultNotes());
 }

teachertool/src/transforms/loadProjectMetadataAsync.ts

@@ -4,7 +4,6 @@ import { getProjectMetaAsync } from "../services/backendRequests";
 import { logDebug } from "../services/loggingService";
 import { showToast } from "./showToast";
 import { makeToast } from "../utils";
-import { setEvalResultsPending } from "./setEvalResultsPending";
 import { initNewProjectResults } from "./initNewProjectResults";
 
 export async function loadProjectMetadataAsync(inputText: string, shareLink: string) {

teachertool/src/transforms/runEvaluateAsync.ts

@@ -88,6 +88,16 @@ export async function runEvaluateAsync(fromUserInteraction: boolean) {
     const evalRequests = teacherTool.rubric.criteria.map(
         criteriaInstance =>
             new Promise(async resolve => {
+                const existingOutcome = teacherTool.evalResults[criteriaInstance.instanceId]?.result;
+                if (
+                    !fromUserInteraction &&
+                    existingOutcome !== undefined &&
+                    existingOutcome !== EvaluationStatus.NotStarted
+                ) {
+                    // The criteria has not changed since it was last evaluated, so we can skip it (unless user specifically clicked run).
+                    return resolve(true);
+                }
+
                 setEvalResultOutcome(criteriaInstance.instanceId, EvaluationStatus.InProgress);
 
                 const loadedValidatorPlans = teacherTool.validatorPlans;

teachertool/src/transforms/setEvalResultsPending.ts → teachertool/src/transforms/setEvalResultsToNotStarted.ts

@@ -3,7 +3,7 @@ import { EvaluationStatus, CriteriaResult } from "../types/criteria";
 import { Rubric } from "../types/rubric";
 import * as Actions from "../state/actions";
 
-export function setEvalResultsPending({
+export function setEvalResultsToNotStarted({
     overwriteExistingEntries,
     rubric,
 }: {
@@ -16,7 +16,7 @@ export function setEvalResultsPending({
     for (const criteria of usedRubric.criteria) {
         const instanceId = criteria.instanceId;
         if (!teachertool.evalResults[instanceId] || overwriteExistingEntries) {
-            allEvalResults[instanceId] = { result: EvaluationStatus.Pending };
+            allEvalResults[instanceId] = { result: EvaluationStatus.NotStarted };
         }
     }
     if (!overwriteExistingEntries) {

teachertool/src/transforms/setParameterValue.ts

@@ -1,7 +1,9 @@
 import { logDebug, logError } from "../services/loggingService";
 import { stateAndDispatch } from "../state";
 import { getCriteriaInstanceWithId } from "../state/helpers";
+import { EvaluationStatus } from "../types/criteria";
 import { ErrorCode } from "../types/errorCode";
+import { setEvalResultOutcome } from "./setEvalResultOutcome";
 import { setRubric } from "./setRubric";
 
 export function setParameterValue(instanceId: string, paramName: string, newValue: any) {
@@ -34,4 +36,5 @@ export function setParameterValue(instanceId: string, paramName: string, newValu
     const newRubric = { ...teacherTool.rubric, criteria: newInstanceSet };
 
     setRubric(newRubric);
+    setEvalResultOutcome(instanceId, EvaluationStatus.NotStarted);
 }

teachertool/src/transforms/setRubric.ts

@@ -2,11 +2,11 @@ import { stateAndDispatch } from "../state";
 import { Rubric } from "../types/rubric";
 import * as Actions from "../state/actions";
 import * as AutorunService from "../services/autorunService";
-import { setEvalResultsPending } from "./setEvalResultsPending";
+import { setEvalResultsToNotStarted } from "./setEvalResultsToNotStarted";
 
 export function setRubric(rubric: Rubric) {
     const { dispatch } = stateAndDispatch();
     dispatch(Actions.setRubric(rubric));
-    setEvalResultsPending({ rubric });
+    setEvalResultsToNotStarted({ rubric });
     AutorunService.poke();
 }

teachertool/src/types/criteria.ts

@@ -38,7 +38,7 @@ export enum EvaluationStatus {
     Fail,
     CompleteWithNoResult,
     InProgress,
-    Pending,
+    NotStarted,
 }
 
 export interface CriteriaResult {