From cf4f60bcba1316c11f4e6cb530798d9dcabd6040 Mon Sep 17 00:00:00 2001
From: Thomas Sparks <69657545+thsparks@users.noreply.github.com>
Date: Fri, 5 Apr 2024 16:22:03 -0700
Subject: [PATCH] Add catalog and validator plans for ai questions. (#9950)

The implementation is not included here, but the docs changes are still needed to have the criteria appear in uploaded targets.

Of note: this does introduce the idea of a "system" parameter, which is an input to the validator plan which can change based on context, but which the user doesn't really need to specify in the crtitera template. In this case, that's the share id. The plan is for the teacher tool to automatically populate system parameters based on the provided key.
---
 common-docs/teachertool/test/catalog-shared.json   | 14 ++++++++++----
 .../teachertool/test/validator-plans-shared.json   | 12 ++++++++++++
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/common-docs/teachertool/test/catalog-shared.json b/common-docs/teachertool/test/catalog-shared.json
index 95196242ba81..62e1188d558b 100644
--- a/common-docs/teachertool/test/catalog-shared.json
+++ b/common-docs/teachertool/test/catalog-shared.json
@@ -37,15 +37,21 @@
         },
         {
             "id": "499F3572-E655-4DEE-953B-5F26BF0191D7",
-            "use": "block_used_n_times",
-            "template": "Long String: ${question}",
-            "description": "This is just a test for long string inputs.",
+            "use": "ai_question",
+            "template": "Ask Copilot: ${question}",
+            "description": "Experimental: AI outputs are inherently nondeterministic and may not be accurate. Use with caution and always review responses.",
             "docPath": "/teachertool",
             "params": [
                 {
                     "name": "question",
                     "type": "longString",
-                    "paths": ["checks[0].blockCounts[0].blockId"]
+                    "paths": ["checks[0].question"]
+                },
+                {
+                    "name": "shareid",
+                    "type": "system",
+                    "key": "SHARE_ID",
+                    "paths": ["checks[0].shareId"]
                 }
             ]
         },
diff --git a/common-docs/teachertool/test/validator-plans-shared.json b/common-docs/teachertool/test/validator-plans-shared.json
index 1cd2ffb53cc0..a9a0830f7031 100644
--- a/common-docs/teachertool/test/validator-plans-shared.json
+++ b/common-docs/teachertool/test/validator-plans-shared.json
@@ -27,6 +27,18 @@
                     "count": 0
                 }
             ]
+        },
+        {
+            ".desc": "Ask Copilot a question",
+            "name": "ai_question",
+            "threshold": -1,
+            "checks": [
+                {
+                    "validator": "aiQuestion",
+                    "question": "",
+                    "shareId": ""
+                }
+            ]
         }
     ]
 }