fixed an edge case and made an extra test for it.

tested/dsl/schema-strict-nat-translation.json

@@ -736,7 +736,7 @@
         },
         {
           "description" : "Programming-language-specific statement or expression.",
-          "type" : "object",
+          "type" : "programming_language",
           "minProperties" : 1,
           "propertyNames" : {
             "$ref" : "#/definitions/programmingLanguage"
@@ -756,7 +756,7 @@
         },
         {
           "description" : "Programming-language-specific statement or expression.",
-          "type" : "object",
+          "type" : "programming_language",
           "minProperties" : 1,
           "propertyNames" : {
             "$ref" : "#/definitions/programmingLanguage"
@@ -1308,7 +1308,8 @@
         "type" : [
           "oracle",
           "expression",
-          "natural_language"
+          "natural_language",
+          "programming_language"
         ]
       }
     },

tested/dsl/translate_parser.py

@@ -228,6 +228,9 @@ def is_expression(_checker: TypeChecker, instance: Any) -> bool:
 def is_natural_language_map(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, NaturalLanguageMap)
 
+def is_programming_language_map(_checker: TypeChecker, instance: Any) -> bool:
+    return isinstance(instance, ProgrammingLanguageMap)
+
 
 def test(value: object) -> bool:
     if not isinstance(value, str):
@@ -251,6 +254,7 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator:
         original_validator.TYPE_CHECKER.redefine("oracle", is_oracle)
         .redefine("expression", is_expression)
         .redefine("natural_language", is_natural_language_map)
+        .redefine("programming_language", is_programming_language_map)
     )
     format_checker = original_validator.FORMAT_CHECKER
     format_checker.checks("tested-dsl-expression", SyntaxError)(test)

tested/nat_translation.py

@@ -75,8 +75,8 @@ def translate_input_files(
 
 
 def parse_value(value: YamlObject, flattened_stack: dict) -> YamlObject:
-    # Will format the strings in different values.
 
+    # Will format the strings in different values.
     if isinstance(value, str):
         return format_string(value, flattened_stack)
     elif isinstance(value, dict):

tests/test_dsl_yaml.py

@@ -21,7 +21,8 @@
 )
 from tested.dsl import parse_dsl, translate_to_test_suite
 from tested.dsl.translate_parser import _parse_yaml, load_schema_validator
-from tested.nat_translation import convert_to_yaml, parse_value, translate_dsl
+from tested.nat_translation import convert_to_yaml, parse_value, translate_dsl, \
+    validate_pre_dsl
 from tested.serialisation import (
     FunctionCall,
     NumberType,
@@ -1573,3 +1574,27 @@ def test_translate_parse():
     }
     parsed_result = parse_value(value, flattened_stack)
     assert parsed_result == expected_value
+
+def test_wrong_natural_translation_suite():
+    yaml_str = """
+tabs:
+- tab: animals
+  testcases:
+  - expression: tests(11)
+    return: 11
+  - expression:
+      javascript: animals_javascript(1 + 1)
+      typescript: animals_typescript(1 + 1)
+      java: Submission.animals_java(1 + 1)
+      python:
+        en: animals_python_en(1 + 1)
+        nl: animals_python_nl(1 + 1)
+    return: 2
+    """.strip()
+    parsed_yaml = _parse_yaml(yaml_str)
+    try:
+        validate_pre_dsl(parsed_yaml)
+    except ExceptionGroup:
+        print("As expected")
+    else:
+        assert False, "Expected ExceptionGroup, but no exception was raised"