From ba0ee3c3772118857bf1001ab951a3f9af386b3c Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Thu, 16 Nov 2023 14:54:31 +0100 Subject: [PATCH] Update default interpretation of string/objects YAML strings and objects are now interpreted as literal YAML values by default. We introduce `!expression` and `!oracle` to "cast" strings and objects to expressions and oracles respectively. The list of arguments in an oracle uses the same logic: YAML types by default, expressions when tagged as such. Using the same logic for arguments is not that obvious: we could also force the use of the expression syntax, as we do with actual expressions and statements. However, it felt more natural to be consistent with the return value (which is in the same YAML object) than with the expressions and statements. This is basically a reversal of the previous behaviour, where strings and objects needed a `!v` tag. This tag has now been removed. --- tested/dsl/schema.json | 10 +-- tested/dsl/schema_draft7.json | 10 +-- tested/dsl/translate_parser.py | 69 +++++++++++-------- .../expected_return_and_got_some.yaml | 2 +- .../expected_return_but_got_none.yaml | 2 +- .../evaluation/one-language-literals.yaml | 2 +- .../echo-function/evaluation/one-nested.yaml | 2 +- tests/exercises/global/evaluation/plan.yaml | 2 +- .../objects/evaluation/missing_key_types.yaml | 2 +- .../exercises/objects/evaluation/no-test.yaml | 4 +- tests/test_dsl_yaml.py | 30 ++++---- 11 files changed, 69 insertions(+), 66 deletions(-) diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 0d510c0e..f8aec849 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -378,10 +378,7 @@ }, "arguments" : { "type" : "array", - "description" : "List of 'Python' values to use as arguments to the function.", - "items" : { - "type" : "string" - } + "description" : "List of YAML (or tagged expression) values to use as arguments to the function." } } } @@ -446,10 +443,7 @@ }, "arguments" : { "type" : "array", - "description" : "List of 'Python' values to use as arguments to the function.", - "items" : { - "type" : "string" - } + "description" : "List of YAML (or tagged expression) values to use as arguments to the function." } } } diff --git a/tested/dsl/schema_draft7.json b/tested/dsl/schema_draft7.json index e1bbf03f..c8a9b234 100644 --- a/tested/dsl/schema_draft7.json +++ b/tested/dsl/schema_draft7.json @@ -373,10 +373,7 @@ }, "arguments" : { "type" : "array", - "description" : "List of 'Python' values to use as arguments to the function.", - "items" : { - "type" : "string" - } + "description" : "List of YAML (or tagged expression) values to use as arguments to the function." } } } @@ -440,10 +437,7 @@ }, "arguments" : { "type" : "array", - "description" : "List of 'Python' values to use as arguments to the function.", - "items" : { - "type" : "string" - } + "description" : "List of YAML (or tagged expression) values to use as arguments to the function." } } } diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 8273e40c..18960eb5 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -65,9 +65,7 @@ ) from tested.utils import get_args -OptionDict = dict[str, int | bool] YamlDict = dict[str, "YamlObject"] -YamlObject = YamlDict | list | bool | float | int | str | None @define @@ -77,8 +75,19 @@ class TestedType: @define -class YamlValue: - value: Any +class ExpressionString: + expression: str + + +@define +class ReturnOracle: + value: YamlDict + + +OptionDict = dict[str, int | bool] +YamlObject = ( + YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle +) def _parse_yaml_value(loader: yaml.Loader, node: yaml.Node) -> Any: @@ -98,8 +107,18 @@ def _custom_type_constructors(loader: yaml.Loader, node: yaml.Node) -> TestedTyp return TestedType(type=tested_tag, value=base_result) -def _yaml_value_constructor(loader: yaml.Loader, node: yaml.Node) -> YamlValue: - return YamlValue(value=_parse_yaml_value(loader, node)) +def _expression_string(loader: yaml.Loader, node: yaml.Node) -> ExpressionString: + result = _parse_yaml_value(loader, node) + assert isinstance(result, str), f"An expression must be a string, got {result}" + return ExpressionString(expression=result) + + +def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle: + result = _parse_yaml_value(loader, node) + assert isinstance( + result, dict + ), f"A custom oracle must be an object, got {result} which is a {type(result)}." + return ReturnOracle(value=result) def _parse_yaml(yaml_stream: str) -> YamlObject: @@ -110,8 +129,8 @@ def _parse_yaml(yaml_stream: str) -> YamlObject: for types in get_args(AllTypes): for actual_type in types: yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader) - yaml.add_constructor("!v", _yaml_value_constructor, loader) - yaml.add_constructor("!value", _yaml_value_constructor, loader) + yaml.add_constructor("!expression", _expression_string, loader) + yaml.add_constructor("!oracle", _return_oracle, loader) try: return yaml.load(yaml_stream, loader) @@ -352,15 +371,12 @@ def _convert_text_output_channel(stream: YamlObject) -> TextOutputChannel: def _convert_yaml_value(stream: YamlObject) -> Value | None: - if isinstance(stream, YamlValue): - # A normal yaml type tagged explicitly. - value = _convert_value(stream.value) - elif isinstance(stream, (int, float, bool, TestedType, list, set)): + if isinstance(stream, ExpressionString): + # We have an expression string. + value = parse_string(stream.expression, is_return=True) + elif isinstance(stream, (int, float, bool, TestedType, list, set, str, dict)): # Simple values where no confusion is possible. value = _convert_value(stream) - elif isinstance(stream, str): - # A normal YAML string is considered a "Python" string. - value = parse_string(stream, is_return=True) else: return None assert isinstance( @@ -370,22 +386,21 @@ def _convert_yaml_value(stream: YamlObject) -> Value | None: def _convert_advanced_value_output_channel(stream: YamlObject) -> ValueOutputChannel: - yaml_value = _convert_yaml_value(stream) - if yaml_value: - return ValueOutputChannel(value=yaml_value) - else: - # We have an object, which means we have an output channel. - assert isinstance(stream, dict) - value = _convert_yaml_value(stream["value"]) - assert isinstance(value, Value) - if "oracle" not in stream or stream["oracle"] == "builtin": + if isinstance(stream, ReturnOracle): + return_object = stream.value + value = _convert_yaml_value(return_object["value"]) + assert isinstance(value, Value), "You must specify a value for a return oracle." + if "oracle" not in return_object or return_object["oracle"] == "builtin": return ValueOutputChannel(value=value) - elif stream["oracle"] == "custom_check": + elif return_object["oracle"] == "custom_check": return ValueOutputChannel( value=value, - oracle=_convert_custom_check_oracle(stream), + oracle=_convert_custom_check_oracle(return_object), ) - raise TypeError(f"Unknown value oracle type: {stream['oracle']}") + raise TypeError(f"Unknown value oracle type: {return_object['oracle']}") + else: + yaml_value = _convert_yaml_value(stream) + return ValueOutputChannel(value=yaml_value) def _validate_testcase_combinations(testcase: YamlDict): diff --git a/tests/exercises/echo-function/evaluation/expected_return_and_got_some.yaml b/tests/exercises/echo-function/evaluation/expected_return_and_got_some.yaml index fa84fa48..f53ba87f 100644 --- a/tests/exercises/echo-function/evaluation/expected_return_and_got_some.yaml +++ b/tests/exercises/echo-function/evaluation/expected_return_and_got_some.yaml @@ -1,4 +1,4 @@ - tab: "My tab" testcases: - expression: 'echo("input")' - return: !v "input" + return: "input" diff --git a/tests/exercises/echo-function/evaluation/expected_return_but_got_none.yaml b/tests/exercises/echo-function/evaluation/expected_return_but_got_none.yaml index 7c687441..d6cde337 100644 --- a/tests/exercises/echo-function/evaluation/expected_return_but_got_none.yaml +++ b/tests/exercises/echo-function/evaluation/expected_return_but_got_none.yaml @@ -1,4 +1,4 @@ - tab: "My tab" testcases: - expression: 'no_echo("input")' - return: !v "input" + return: "input" diff --git a/tests/exercises/echo-function/evaluation/one-language-literals.yaml b/tests/exercises/echo-function/evaluation/one-language-literals.yaml index 7b5acfdb..20718509 100644 --- a/tests/exercises/echo-function/evaluation/one-language-literals.yaml +++ b/tests/exercises/echo-function/evaluation/one-language-literals.yaml @@ -9,4 +9,4 @@ kotlin: "toString(1+1)" python: "submission.to_string(1+1)" csharp: "Submission.toString(1+1)" - return: !v "2" + return: "2" diff --git a/tests/exercises/echo-function/evaluation/one-nested.yaml b/tests/exercises/echo-function/evaluation/one-nested.yaml index 3ce7336b..126fe164 100644 --- a/tests/exercises/echo-function/evaluation/one-nested.yaml +++ b/tests/exercises/echo-function/evaluation/one-nested.yaml @@ -1,4 +1,4 @@ - tab: "My tab" testcases: - expression: 'echo(echo("input"))' - return: !v "input" + return: "input" diff --git a/tests/exercises/global/evaluation/plan.yaml b/tests/exercises/global/evaluation/plan.yaml index 4aea33a7..5fd954bc 100644 --- a/tests/exercises/global/evaluation/plan.yaml +++ b/tests/exercises/global/evaluation/plan.yaml @@ -1,7 +1,7 @@ - tab: "Global variable" testcases: - expression: "GLOBAL_VAR" - return: !v "GLOBAL" + return: "GLOBAL" description: description: "Hallo" format: "code" diff --git a/tests/exercises/objects/evaluation/missing_key_types.yaml b/tests/exercises/objects/evaluation/missing_key_types.yaml index 030651a3..e7bc98bc 100644 --- a/tests/exercises/objects/evaluation/missing_key_types.yaml +++ b/tests/exercises/objects/evaluation/missing_key_types.yaml @@ -1,4 +1,4 @@ - tab: "Feedback" testcases: - expression: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' - return: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' + return: !expression '{{"a"}: [int32(1)], {"b"}: "a.txt"}' diff --git a/tests/exercises/objects/evaluation/no-test.yaml b/tests/exercises/objects/evaluation/no-test.yaml index c92b18eb..17657f62 100644 --- a/tests/exercises/objects/evaluation/no-test.yaml +++ b/tests/exercises/objects/evaluation/no-test.yaml @@ -1,11 +1,11 @@ - tab: "Feedback" testcases: - statement: '{["a", "b"], ["c"]}' - return: '{["a", "b"], ["a"]}' + return: !expression '{["a", "b"], ["a"]}' - statement: 'x = {{"a"}: [int16(1)], {"b"}: [int16(0)]}' - statement: 'x = {{"a"}: [int32(1)], {"b"}: "a"}' - expression: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' - return: '{{"a"}: [int32(1)], {"b"}: "a.txt"}' + return: !expression '{{"a"}: [int32(1)], {"b"}: "a.txt"}' files: - name: "a.txt" url: "a.txt" diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 4dee2c59..d9b7ebef 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -261,7 +261,7 @@ def test_statements(): data: "New safe" config: *stdout - expression: 'safe.content()' - return: !v "Ignore whitespace" + return: "Ignore whitespace" - testcases: - statement: 'safe: Safe = Safe(uint8(5))' stdout: @@ -270,7 +270,7 @@ def test_statements(): <<: *stdout ignoreWhitespace: false - expression: 'safe.content()' - return: 'uint8(5)' + return: !expression 'uint8(5)' """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) @@ -385,7 +385,7 @@ def test_invalid_yaml(): stderr: [] testcases: - statement: 'data = () ()' - return: '() {}' + return: !expression '() {}' """ with pytest.raises(Exception): translate_to_test_suite(yaml_str) @@ -408,7 +408,7 @@ def test_statement_with_yaml_dict(): - tab: "Feedback" testcases: - expression: "get_dict()" - return: !v + return: alpha: 5 beta: 6 """ @@ -462,7 +462,7 @@ def test_expression_raw_return(): contexts: - testcases: - expression: 'test()' - return: '[(4, 4), (4, 3), (4, 2), (4, 1), (4, 0), (3, 0), (3, 1), (4, 1)]' + return: !expression '[(4, 4), (4, 3), (4, 2), (4, 1), (4, 0), (3, 0), (3, 1), (4, 1)]' """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) @@ -494,7 +494,7 @@ def test_empty_constructor(function_name, result): contexts: - testcases: - expression: 'test()' - return: '{function_name}()' + return: !expression '{function_name}()' """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) @@ -524,7 +524,7 @@ def test_empty_constructor_with_param(function_name, result): contexts: - testcases: - expression: 'test()' - return: '{function_name}([])' + return: !expression '{function_name}([])' """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) @@ -598,7 +598,7 @@ def test_text_custom_checks_correct(): language: "python" file: "test.py" name: "evaluate_test" - arguments: ["'yes'", "5", "set([5, 5])"] + arguments: [!expression "'yes'", 5, !expression "set([5, 5])"] """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) @@ -635,8 +635,8 @@ def test_value_built_in_checks_implied(): contexts: - testcases: - expression: 'test()' - return: - value: "'hallo'" + return: !oracle + value: !expression "'hallo'" """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) @@ -660,8 +660,8 @@ def test_value_built_in_checks_explicit(): contexts: - testcases: - expression: 'test()' - return: - value: "'hallo'" + return: !oracle + value: "hallo" oracle: "builtin" """ json_str = translate_to_test_suite(yaml_str) @@ -686,13 +686,13 @@ def test_value_custom_checks_correct(): contexts: - testcases: - expression: 'test()' - return: - value: "'hallo'" + return: !oracle + value: "hallo" oracle: "custom_check" language: "python" file: "test.py" name: "evaluate_test" - arguments: ["'yes'", "5", "set([5, 5])"] + arguments: ['yes', 5, !expression "set([5, 5])"] """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str)