Update default interpretation of string/objects

YAML strings and objects are now interpreted as literal YAML values by
default.

We introduce `!expression` and `!oracle` to "cast" strings and objects
to expressions and oracles respectively. The list of arguments in an
oracle uses the same logic: YAML types by default, expressions when
tagged as such.

Using the same logic for arguments is not that obvious: we could also
force the use of the expression syntax, as we do with actual expressions
and statements. However, it felt more natural to be consistent with the
return value (which is in the same YAML object) than with the
expressions and statements.

This is basically a reversal of the previous behaviour, where strings
and objects needed a `!v` tag. This tag has now been removed.

tested/dsl/schema.json

@@ -378,10 +378,7 @@
             },
             "arguments" : {
               "type" : "array",
-              "description" : "List of 'Python' values to use as arguments to the function.",
-              "items" : {
-                "type" : "string"
-              }
+              "description" : "List of YAML (or tagged expression) values to use as arguments to the function."
             }
           }
         }
@@ -446,10 +443,7 @@
                 },
                 "arguments" : {
                   "type" : "array",
-                  "description" : "List of 'Python' values to use as arguments to the function.",
-                  "items" : {
-                    "type" : "string"
-                  }
+                  "description" : "List of YAML (or tagged expression) values to use as arguments to the function."
                 }
               }
             }

tested/dsl/schema_draft7.json

@@ -373,10 +373,7 @@
             },
             "arguments" : {
               "type" : "array",
-              "description" : "List of 'Python' values to use as arguments to the function.",
-              "items" : {
-                "type" : "string"
-              }
+              "description" : "List of YAML (or tagged expression) values to use as arguments to the function."
             }
           }
         }
@@ -440,10 +437,7 @@
                 },
                 "arguments" : {
                   "type" : "array",
-                  "description" : "List of 'Python' values to use as arguments to the function.",
-                  "items" : {
-                    "type" : "string"
-                  }
+                  "description" : "List of YAML (or tagged expression) values to use as arguments to the function."
                 }
               }
             }

tested/dsl/translate_parser.py

@@ -65,9 +65,7 @@
 )
 from tested.utils import get_args
 
-OptionDict = dict[str, int | bool]
 YamlDict = dict[str, "YamlObject"]
-YamlObject = YamlDict | list | bool | float | int | str | None
 
 
 @define
@@ -77,8 +75,19 @@ class TestedType:
 
 
 @define
-class YamlValue:
-    value: Any
+class ExpressionString:
+    expression: str
+
+
+@define
+class ReturnOracle:
+    value: YamlDict
+
+
+OptionDict = dict[str, int | bool]
+YamlObject = (
+    YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle
+)
 
 
 def _parse_yaml_value(loader: yaml.Loader, node: yaml.Node) -> Any:
@@ -98,8 +107,18 @@ def _custom_type_constructors(loader: yaml.Loader, node: yaml.Node) -> TestedTyp
     return TestedType(type=tested_tag, value=base_result)
 
 
-def _yaml_value_constructor(loader: yaml.Loader, node: yaml.Node) -> YamlValue:
-    return YamlValue(value=_parse_yaml_value(loader, node))
+def _expression_string(loader: yaml.Loader, node: yaml.Node) -> ExpressionString:
+    result = _parse_yaml_value(loader, node)
+    assert isinstance(result, str), f"An expression must be a string, got {result}"
+    return ExpressionString(expression=result)
+
+
+def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle:
+    result = _parse_yaml_value(loader, node)
+    assert isinstance(
+        result, dict
+    ), f"A custom oracle must be an object, got {result} which is a {type(result)}."
+    return ReturnOracle(value=result)
 
 
 def _parse_yaml(yaml_stream: str) -> YamlObject:
@@ -110,8 +129,8 @@ def _parse_yaml(yaml_stream: str) -> YamlObject:
     for types in get_args(AllTypes):
         for actual_type in types:
             yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader)
-    yaml.add_constructor("!v", _yaml_value_constructor, loader)
-    yaml.add_constructor("!value", _yaml_value_constructor, loader)
+    yaml.add_constructor("!expression", _expression_string, loader)
+    yaml.add_constructor("!oracle", _return_oracle, loader)
 
     try:
         return yaml.load(yaml_stream, loader)
@@ -352,15 +371,12 @@ def _convert_text_output_channel(stream: YamlObject) -> TextOutputChannel:
 
 
 def _convert_yaml_value(stream: YamlObject) -> Value | None:
-    if isinstance(stream, YamlValue):
-        # A normal yaml type tagged explicitly.
-        value = _convert_value(stream.value)
-    elif isinstance(stream, (int, float, bool, TestedType, list, set)):
+    if isinstance(stream, ExpressionString):
+        # We have an expression string.
+        value = parse_string(stream.expression, is_return=True)
+    elif isinstance(stream, (int, float, bool, TestedType, list, set, str, dict)):
         # Simple values where no confusion is possible.
         value = _convert_value(stream)
-    elif isinstance(stream, str):
-        # A normal YAML string is considered a "Python" string.
-        value = parse_string(stream, is_return=True)
     else:
         return None
     assert isinstance(
@@ -370,22 +386,21 @@ def _convert_yaml_value(stream: YamlObject) -> Value | None:
 
 
 def _convert_advanced_value_output_channel(stream: YamlObject) -> ValueOutputChannel:
-    yaml_value = _convert_yaml_value(stream)
-    if yaml_value:
-        return ValueOutputChannel(value=yaml_value)
-    else:
-        # We have an object, which means we have an output channel.
-        assert isinstance(stream, dict)
-        value = _convert_yaml_value(stream["value"])
-        assert isinstance(value, Value)
-        if "oracle" not in stream or stream["oracle"] == "builtin":
+    if isinstance(stream, ReturnOracle):
+        return_object = stream.value
+        value = _convert_yaml_value(return_object["value"])
+        assert isinstance(value, Value), "You must specify a value for a return oracle."
+        if "oracle" not in return_object or return_object["oracle"] == "builtin":
             return ValueOutputChannel(value=value)
-        elif stream["oracle"] == "custom_check":
+        elif return_object["oracle"] == "custom_check":
             return ValueOutputChannel(
                 value=value,
-                oracle=_convert_custom_check_oracle(stream),
+                oracle=_convert_custom_check_oracle(return_object),
             )
-        raise TypeError(f"Unknown value oracle type: {stream['oracle']}")
+        raise TypeError(f"Unknown value oracle type: {return_object['oracle']}")
+    else:
+        yaml_value = _convert_yaml_value(stream)
+        return ValueOutputChannel(value=yaml_value)
 
 
 def _validate_testcase_combinations(testcase: YamlDict):

tests/exercises/echo-function/evaluation/expected_return_and_got_some.yaml

@@ -1,4 +1,4 @@
 - tab: "My tab"
   testcases:
     - expression: 'echo("input")'
-      return: !v "input"
+      return: "input"

tests/exercises/echo-function/evaluation/expected_return_but_got_none.yaml

@@ -1,4 +1,4 @@
 - tab: "My tab"
   testcases:
     - expression: 'no_echo("input")'
-      return: !v "input"
+      return: "input"

tests/exercises/echo-function/evaluation/one-language-literals.yaml

@@ -9,4 +9,4 @@
         kotlin: "toString(1+1)"
         python: "submission.to_string(1+1)"
         csharp: "Submission.toString(1+1)"
-      return: !v "2"
+      return: "2"

tests/exercises/echo-function/evaluation/one-nested.yaml

@@ -1,4 +1,4 @@
 - tab: "My tab"
   testcases:
     - expression: 'echo(echo("input"))'
-      return: !v "input"
+      return: "input"

tests/exercises/global/evaluation/plan.yaml

@@ -1,7 +1,7 @@
 - tab: "Global variable"
   testcases:
     - expression: "GLOBAL_VAR"
-      return: !v "GLOBAL"
+      return: "GLOBAL"
       description:
         description: "Hallo"
         format: "code"

tests/exercises/objects/evaluation/missing_key_types.yaml

@@ -1,4 +1,4 @@
 - tab: "Feedback"
   testcases:
     - expression: '{{"a"}: [int32(1)], {"b"}: "a.txt"}'
-      return: '{{"a"}: [int32(1)], {"b"}: "a.txt"}'
+      return: !expression '{{"a"}: [int32(1)], {"b"}: "a.txt"}'

tests/exercises/objects/evaluation/no-test.yaml

@@ -1,11 +1,11 @@
 - tab: "Feedback"
   testcases:
     - statement: '{["a", "b"], ["c"]}'
-      return: '{["a", "b"], ["a"]}'
+      return: !expression '{["a", "b"], ["a"]}'
     - statement: 'x = {{"a"}: [int16(1)], {"b"}: [int16(0)]}'
     - statement: 'x = {{"a"}: [int32(1)], {"b"}: "a"}'
     - expression: '{{"a"}: [int32(1)], {"b"}: "a.txt"}'
-      return: '{{"a"}: [int32(1)], {"b"}: "a.txt"}'
+      return: !expression '{{"a"}: [int32(1)], {"b"}: "a.txt"}'
       files:
         - name: "a.txt"
           url: "a.txt"

tests/test_dsl_yaml.py

@@ -261,7 +261,7 @@ def test_statements():
         data: "New safe"
         config: *stdout
     - expression: 'safe.content()'
-      return: !v "Ignore whitespace"
+      return: "Ignore whitespace"
   - testcases:
     - statement: 'safe: Safe = Safe(uint8(5))'
       stdout:
@@ -270,7 +270,7 @@ def test_statements():
           <<: *stdout
           ignoreWhitespace: false
     - expression: 'safe.content()'
-      return: 'uint8(5)'
+      return: !expression 'uint8(5)'
     """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -385,7 +385,7 @@ def test_invalid_yaml():
     stderr: []
     testcases:
     - statement: 'data = () ()'
-      return: '() {}'
+      return: !expression '() {}'
     """
     with pytest.raises(Exception):
         translate_to_test_suite(yaml_str)
@@ -408,7 +408,7 @@ def test_statement_with_yaml_dict():
 - tab: "Feedback"
   testcases:
   - expression: "get_dict()"
-    return: !v
+    return:
         alpha: 5
         beta: 6
 """
@@ -462,7 +462,7 @@ def test_expression_raw_return():
   contexts:
     - testcases:
         - expression: 'test()'
-          return: '[(4, 4), (4, 3), (4, 2), (4, 1), (4, 0), (3, 0), (3, 1), (4, 1)]'
+          return: !expression '[(4, 4), (4, 3), (4, 2), (4, 1), (4, 0), (3, 0), (3, 1), (4, 1)]'
 """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -494,7 +494,7 @@ def test_empty_constructor(function_name, result):
   contexts:
     - testcases:
         - expression: 'test()'
-          return: '{function_name}()'
+          return: !expression '{function_name}()'
 """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -524,7 +524,7 @@ def test_empty_constructor_with_param(function_name, result):
   contexts:
     - testcases:
         - expression: 'test()'
-          return: '{function_name}([])'
+          return: !expression '{function_name}([])'
 """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -598,7 +598,7 @@ def test_text_custom_checks_correct():
                 language: "python"
                 file: "test.py"
                 name: "evaluate_test"
-                arguments: ["'yes'", "5", "set([5, 5])"]
+                arguments: [!expression "'yes'", 5, !expression "set([5, 5])"]
     """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -635,8 +635,8 @@ def test_value_built_in_checks_implied():
       contexts:
         - testcases:
             - expression: 'test()'
-              return:
-                value: "'hallo'"
+              return: !oracle
+                value: !expression "'hallo'"
     """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)
@@ -660,8 +660,8 @@ def test_value_built_in_checks_explicit():
       contexts:
         - testcases:
             - expression: 'test()'
-              return:
-                value: "'hallo'"
+              return: !oracle
+                value: "hallo"
                 oracle: "builtin"
     """
     json_str = translate_to_test_suite(yaml_str)
@@ -686,13 +686,13 @@ def test_value_custom_checks_correct():
       contexts:
         - testcases:
             - expression: 'test()'
-              return:
-                value: "'hallo'"
+              return: !oracle
+                value: "hallo"
                 oracle: "custom_check"
                 language: "python"
                 file: "test.py"
                 name: "evaluate_test"
-                arguments: ["'yes'", "5", "set([5, 5])"]
+                arguments: ['yes', 5, !expression "set([5, 5])"]
     """
     json_str = translate_to_test_suite(yaml_str)
     suite = parse_test_suite(json_str)