Teacher Tool: Copilot Criteria Support (#9953)

This change adds the front-end support for copilot criteria. In essence, we call into the backend with the Share ID of the project and the question being asked, then the backend forwards that information to DeepPrompt and returns the result to us.

The backend changes will not be checked in until we get more feedback and a better sense for funding, so this change has a few workarounds to compensate while still enabling easy demos and experimentation. Notably, there is a new "copilot" flag that can be set to point the copilot backend request to a staging slot. For example, http://localhost:3232/eval?copilot=thsparks sets it to the thsparks staging slot.

common-docs/teachertool/test/catalog-shared.json

@@ -1,5 +1,25 @@
 {
     "criteria": [
+        {
+            "id": "499F3572-E655-4DEE-953B-5F26BF0191D7",
+            "use": "ai_question",
+            "template": "Ask Copilot: ${question}",
+            "description": "Experimental: AI outputs may not be accurate. Use with caution and always review responses.",
+            "docPath": "/teachertool",
+            "params": [
+                {
+                    "name": "question",
+                    "type": "longString",
+                    "paths": ["checks[0].question"]
+                },
+                {
+                    "name": "shareid",
+                    "type": "system",
+                    "key": "SHARE_ID",
+                    "paths": ["checks[0].shareId"]
+                }
+            ]
+        },
         {
             "id": "7AE7EA2A-3AC8-42DC-89DB-65E3AE157156",
             "use": "block_comment_used",
@@ -35,26 +55,6 @@
                 }
             ]
         },
-        {
-            "id": "499F3572-E655-4DEE-953B-5F26BF0191D7",
-            "use": "ai_question",
-            "template": "Ask Copilot: ${question}",
-            "description": "Experimental: AI outputs are inherently nondeterministic and may not be accurate. Use with caution and always review responses.",
-            "docPath": "/teachertool",
-            "params": [
-                {
-                    "name": "question",
-                    "type": "longString",
-                    "paths": ["checks[0].question"]
-                },
-                {
-                    "name": "shareid",
-                    "type": "system",
-                    "key": "SHARE_ID",
-                    "paths": ["checks[0].shareId"]
-                }
-            ]
-        },
         {
             "id": "B8987394-1531-4C71-8661-BE4086CE0C6E",
             "use": "n_loops",

localtypings/validatorPlan.d.ts

@@ -41,7 +41,8 @@ declare namespace pxt.blocks {
     }
 
     export interface EvaluationResult {
-        result: boolean;
+        result?: boolean;
+        notes?: string;
     }
 
     export interface BlockFieldValueExistsCheck extends ValidatorCheckBase {
@@ -50,4 +51,10 @@ declare namespace pxt.blocks {
         fieldValue: string;
         blockType: string;
     }
+
+    export interface AiQuestionValidatorCheck extends ValidatorCheckBase {
+        validator: "aiQuestion";
+        question: string;
+        shareId: string;
+    }
 }

react-common/components/controls/Textarea.tsx

@@ -46,11 +46,47 @@ export const Textarea = (props: TextareaProps) => {
 
     const [value, setValue] = React.useState(initialValue || "");
     const textareaRef = React.useRef<HTMLTextAreaElement>(null);
+    const previousWidthRef = React.useRef<number>(0);
+
+    const fitVerticalSizeToContent = () => {
+        if (!textareaRef.current) {
+            return;
+        }
+
+        textareaRef.current.style.height = "1px";
+        textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`;
+    }
 
     React.useEffect(() => {
         setValue(initialValue)
+
+        if (autoResize && textareaRef.current) {
+            fitVerticalSizeToContent();
+        }
     }, [initialValue])
 
+    React.useEffect(() => {
+        if (!autoResize) {
+            return () => {};
+        }
+
+        const observer = new ResizeObserver((entries) => {
+            // If the width has changed, we need to update the vertical height to account for it.
+            const width = entries[0].contentRect.width;
+            if (previousWidthRef.current != width) {
+                requestAnimationFrame(() => fitVerticalSizeToContent());
+                previousWidthRef.current = width;
+            }
+        });
+
+        if (textareaRef.current) {
+            observer.observe(textareaRef.current);
+        }
+
+        return () => {
+            observer.disconnect();
+        }
+    }, [autoResize]);
 
     const changeHandler = (e: React.ChangeEvent<any>) => {
         const newValue = (e.target as any).value;
@@ -61,8 +97,7 @@ export const Textarea = (props: TextareaProps) => {
             onChange(newValue);
         }
         if (autoResize && textareaRef.current) {
-            textareaRef.current.style.height = "1px";
-            textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`;
+            fitVerticalSizeToContent();
         }
     }
 
@@ -108,4 +143,4 @@ export const Textarea = (props: TextareaProps) => {
             </div>
         </div>
     );
-}
+}

teachertool/src/components/CriteriaEvalResultDropdown.tsx

@@ -3,55 +3,48 @@ import { useMemo } from "react";
 import { setEvalResultOutcome } from "../transforms/setEvalResultOutcome";
 import { Dropdown, DropdownItem } from "react-common/components/controls/Dropdown";
 import { EvaluationStatus } from "../types/criteria";
-import css from "./styling/EvalResultDisplay.module.scss";
 import { classList } from "react-common/components/util";
+import css from "./styling/EvalResultDisplay.module.scss";
 
 interface CriteriaEvalResultProps {
     result: EvaluationStatus;
     criteriaId: string;
 }
 
 const itemIdToCriteriaResult: pxt.Map<EvaluationStatus> = {
-    evaluating: EvaluationStatus.InProgress,
     notevaluated: EvaluationStatus.CompleteWithNoResult,
     fail: EvaluationStatus.Fail,
     pass: EvaluationStatus.Pass,
     pending: EvaluationStatus.Pending,
 };
 
 const criteriaResultToItemId: pxt.Map<string> = {
-    [EvaluationStatus.InProgress]: "evaluating",
     [EvaluationStatus.CompleteWithNoResult]: "notevaluated",
     [EvaluationStatus.Fail]: "fail",
     [EvaluationStatus.Pass]: "pass",
     [EvaluationStatus.Pending]: "pending",
 };
 
 const dropdownItems: DropdownItem[] = [
-    {
-        id: "evaluating",
-        title: lf("evaluating..."),
-        label: lf("evaluating..."),
-    },
     {
         id: "notevaluated",
-        title: lf("not evaluated"),
-        label: lf("not evaluated"),
+        title: lf("not applicable"),
+        label: lf("N/A"),
     },
     {
         id: "fail",
         title: lf("needs work"),
-        label: lf("needs work"),
+        label: lf("Needs work"),
     },
     {
         id: "pass",
         title: lf("looks good!"),
-        label: lf("looks good!"),
+        label: lf("Looks good!"),
     },
     {
         id: "pending",
         title: lf("not started"),
-        label: lf("not started"),
+        label: lf("Not started"),
     },
 ];
 
@@ -62,7 +55,11 @@ export const CriteriaEvalResultDropdown: React.FC<CriteriaEvalResultProps> = ({
         <Dropdown
             id="project-eval-result-dropdown"
             selectedId={selectedResult}
-            className={classList("rounded", selectedResult)}
+            className={classList(
+                "rounded",
+                selectedResult,
+                selectedResult === "notevaluated" ? css["no-print"] : undefined
+            )}
             items={dropdownItems}
             onItemSelected={id => setEvalResultOutcome(criteriaId, itemIdToCriteriaResult[id])}
         />

teachertool/src/components/CriteriaResultEntry.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { useState, useContext, useRef } from "react";
+import { useState, useContext, useEffect } from "react";
 import css from "./styling/EvalResultDisplay.module.scss";
 import { AppStateContext } from "../state/appStateContext";
 import { classList } from "react-common/components/util";
@@ -10,6 +10,8 @@ import { CriteriaEvalResultDropdown } from "./CriteriaEvalResultDropdown";
 import { DebouncedTextarea } from "./DebouncedTextarea";
 import { getCatalogCriteriaWithId, getCriteriaInstanceWithId } from "../state/helpers";
 import { ReadOnlyCriteriaDisplay } from "./ReadonlyCriteriaDisplay";
+import { EvaluationStatus } from "../types/criteria";
+import { ThreeDotsLoadingDisplay } from "./ThreeDotsLoadingDisplay";
 
 interface AddNotesButtonProps {
     criteriaId: string;
@@ -36,11 +38,11 @@ const AddNotesButton: React.FC<AddNotesButtonProps> = ({ criteriaId, setShowInpu
 
 interface CriteriaResultNotesProps {
     criteriaId: string;
-    notes?: string;
 }
 
-const CriteriaResultNotes: React.FC<CriteriaResultNotesProps> = ({ criteriaId, notes }) => {
+const CriteriaResultNotes: React.FC<CriteriaResultNotesProps> = ({ criteriaId }) => {
     const { state: teacherTool } = useContext(AppStateContext);
+
     const onTextChange = (str: string) => {
         setEvalResultNotes(criteriaId, str);
     };
@@ -52,7 +54,7 @@ const CriteriaResultNotes: React.FC<CriteriaResultNotesProps> = ({ criteriaId, n
                 ariaLabel={lf("Feedback regarding the criteria result")}
                 label={lf("Feedback")}
                 title={lf("Write your notes here")}
-                initialValue={teacherTool.evalResults[criteriaId]?.notes ?? undefined}
+                initialValue={teacherTool.evalResults[criteriaId]?.notes}
                 autoResize={true}
                 onChange={onTextChange}
                 autoComplete={false}
@@ -73,30 +75,43 @@ export const CriteriaResultEntry: React.FC<CriteriaResultEntryProps> = ({ criter
     const criteriaInstance = getCriteriaInstanceWithId(teacherTool, criteriaId);
     const catalogCriteria = criteriaInstance ? getCatalogCriteriaWithId(criteriaInstance.catalogCriteriaId) : undefined;
 
+    useEffect(() => {
+        if (!showInput && teacherTool.evalResults[criteriaId]?.notes) {
+            setShowInput(true);
+        }
+    }, [teacherTool.evalResults[criteriaId]?.notes]);
+
+    const isInProgress = teacherTool.evalResults[criteriaId].result === EvaluationStatus.InProgress;
     return (
         <>
             {catalogCriteria && (
-                <div className={css["specific-criteria-result"]} key={criteriaId}>
+                <div className={css["specific-criteria-result"]} key={criteriaId} aria-busy={isInProgress}>
                     <div className={css["result-details"]}>
                         <ReadOnlyCriteriaDisplay
                             catalogCriteria={catalogCriteria}
                             criteriaInstance={criteriaInstance}
                             showDescription={false}
                         />
-                        <CriteriaEvalResultDropdown
-                            result={teacherTool.evalResults[criteriaId].result}
-                            criteriaId={criteriaId}
-                        />
-                    </div>
-                    <div
-                        className={classList(
-                            css["result-notes"],
-                            teacherTool.evalResults[criteriaId]?.notes ? undefined : css["no-print"]
+                        {!isInProgress && (
+                            <CriteriaEvalResultDropdown
+                                result={teacherTool.evalResults[criteriaId].result}
+                                criteriaId={criteriaId}
+                            />
                         )}
-                    >
-                        {!showInput && <AddNotesButton criteriaId={criteriaId} setShowInput={setShowInput} />}
-                        {showInput && <CriteriaResultNotes criteriaId={criteriaId} />}
                     </div>
+                    {isInProgress ? (
+                        <ThreeDotsLoadingDisplay className={css["loading-display"]} />
+                    ) : (
+                        <div
+                            className={classList(
+                                css["result-notes"],
+                                teacherTool.evalResults[criteriaId]?.notes ? undefined : css["no-print"]
+                            )}
+                        >
+                            {!showInput && <AddNotesButton criteriaId={criteriaId} setShowInput={setShowInput} />}
+                            {showInput && <CriteriaResultNotes criteriaId={criteriaId} />}
+                        </div>
+                    )}
                 </div>
             )}
         </>

teachertool/src/components/ThreeDotsLoadingDisplay.tsx

@@ -0,0 +1,17 @@
+import { classList } from "react-common/components/util";
+import css from "./styling/ThreeDotsLoadingDisplay.module.scss";
+import { Strings } from "../constants";
+
+export interface ThreeDotsLoadingDisplayProps {
+    className?: string;
+}
+// Three dots that move up and down in a wave pattern.
+export const ThreeDotsLoadingDisplay: React.FC<ThreeDotsLoadingDisplayProps> = ({ className }) => {
+    return (
+        <div className={classList(className, css["loading-ellipsis"])} aria-label={Strings.Loading}>
+            <i className={classList("far fa-circle", css["circle"], css["circle-1"])} aria-hidden={true} />
+            <i className={classList("far fa-circle", css["circle"], css["circle-2"])} aria-hidden={true} />
+            <i className={classList("far fa-circle", css["circle"], css["circle-3"])} aria-hidden={true} />
+        </div>
+    );
+};

teachertool/src/components/styling/EvalResultDisplay.module.scss

@@ -14,6 +14,11 @@
     min-height: 9rem;
     border-bottom: solid 1px var(--pxt-content-accent);
     break-inside: avoid;
+
+    .loading-display {
+        padding-top: 0;
+        flex-grow: 1;
+    }
 }
 
 .result-details {

teachertool/src/components/styling/ThreeDotsLoadingDisplay.module.scss

@@ -0,0 +1,44 @@
+.loading-ellipsis {
+    display: flex;
+    flex-direction: row;
+    gap: 1rem;
+    padding-top: 1rem;
+    align-items: center;
+    color: var(--pxt-content-foreground);
+
+    .circle {
+        font-size: 1rem;
+
+        @keyframes bounce {
+            0% {
+                transform: translateY(0);
+            }
+
+            25% {
+                transform: translateY(-0.5rem);
+            }
+
+            50% {
+                transform: translateY(0);
+            }
+
+            100% {
+                transform: translateY(0);
+            }
+        }
+
+        animation: bounce 750ms infinite ease-in-out;
+    }
+
+    .circle-1 {
+        animation-delay: 0ms;
+    }
+
+    .circle-2 {
+        animation-delay: 125ms;
+    }
+
+    .circle-3 {
+        animation-delay: 250ms;
+    }
+}

teachertool/src/constants.ts

@@ -25,6 +25,7 @@ export namespace Strings {
     export const ValueRequired = lf("Value Required");
     export const AddSelected = lf("Add Selected");
     export const Continue = lf("Continue");
+    export const Loading = lf("Loading...");
 }
 
 export namespace Ticks {