Initial generated test cases for complex input structures and markdow…

…n inputs (#1808)

* generated

Signed-off-by: dbczumar <[email protected]>

* fix

Signed-off-by: dbczumar <[email protected]>

* fix

Signed-off-by: dbczumar <[email protected]>

* fix

Signed-off-by: dbczumar <[email protected]>

---------

Signed-off-by: dbczumar <[email protected]>

tests/reliability/complex_types/generated/test_many_types_1/inputs/input1.json

@@ -0,0 +1,26 @@
+{
+  "assertions": [
+    "The 'processedTupleField' should be a tuple containing a string and a number. Note that 'processedNestedObjectField.tupleField' should NOT actually be a tuple.",
+    "The 'processedEnumField' should be one of the allowed enum values: 'option1', 'option2', or 'option3'.",
+    "The 'processedDatetimeField' should be a date-time",
+    "The 'processedLiteralField' should be exactly 'literalValue'.",
+    "The 'processedObjectField' should contain 'subField1' (string), 'subField2' (number), and an additional boolean field 'additionalField'.",
+    "The 'processedNestedObjectField' should contain 'tupleField' (which is actually a list with a string and a number - the name is misleading), 'enumField' (one of the allowed enum values), 'datetimeField' (string formatted as date-time), 'literalField' (exactly 'literalValue'), and an additional boolean field 'additionalField'."
+  ],
+  "input": {
+    "datetimeField": "2023-10-12T07:20:50.52Z",
+    "enumField": "option1",
+    "literalField": "literalValue",
+    "nestedObjectField": {
+      "datetimeField": "2023-10-12T07:20:50.52Z",
+      "enumField": "option2",
+      "literalField": "literalValue",
+      "tupleField": ["nestedString", 789]
+    },
+    "objectField": {
+      "subField1": "example",
+      "subField2": 456
+    },
+    "tupleField": ["string1", 123]
+  }
+}

tests/reliability/complex_types/generated/test_many_types_1/inputs/input2.json

@@ -0,0 +1,26 @@
+{
+  "assertions": [
+    "The 'processedTupleField' should be an tuple with exactly two elements: the first element being a string and the second element being a number. Note that 'processedNestedObjectField.tupleField' should NOT actually be a tuple",
+    "The 'processedEnumField' should be one of the predefined options: 'option1', 'option2', or 'option3'.",
+    "The 'processedDatetimeField' should be a date-time",
+    "The 'processedLiteralField' should be the enum 'literalValue'.",
+    "The 'processedObjectField' should be an object containing 'subField1' as a string, 'subField2' as a number, and an 'additionalField' as a boolean.",
+    "The 'processedNestedObjectField' should be an object containing 'tupleField' as a list (NOT a tuple) with exactly two elements (a string and a number), 'enumField' as one of the predefined options (option1, option2, or option3), 'datetimeField' as a 'date-time' object, 'literalField' as the string 'literalValue', and an 'additionalField' as a boolean."
+  ],
+  "input": {
+    "datetimeField": "2023-10-01T12:00:00Z",
+    "enumField": "option1",
+    "literalField": "literalValue",
+    "nestedObjectField": {
+      "datetimeField": "2023-11-01T12:00:00Z",
+      "enumField": "option2",
+      "literalField": "literalValue",
+      "tupleField": ["nestedString", 789]
+    },
+    "objectField": {
+      "subField1": "Patriotism is a feeling of love, devotion, and sense of attachment to one's country. This attachment can be a combination of many different feelings relating to one's homeland, including ethnic, cultural, political or historical aspects. It encompasses a set of concepts closely related to those of nationalism. In the context of patriotism, people may express their feelings in a variety of ways, including supporting their country's interests and policies, celebrating national holidays, and participating in civic activities. Patriotism often involves a sense of pride in one's country and a willingness to defend it against any threats. It can also include a commitment to improving the country and making it a better place for future generations. The concept of patriotism is often linked with the idea of national identity, which is the sense of a nation as a cohesive whole, as represented by distinctive traditions, culture, language, and politics. Patriots may feel a strong sense of loyalty and duty to their country, and they may take actions to support and protect it. However, it is important to note that patriotism can also be a complex and sometimes controversial concept. While it can inspire positive actions and a sense of community, it can also lead to exclusionary or aggressive behaviors if taken to an extreme. In some cases, excessive patriotism can result in nationalism, which can lead to conflicts with other nations or groups. Despite these potential issues, many people view patriotism as a positive force that can unite people and inspire them to work together for the common good. It can foster a sense of belonging and purpose, and it can motivate individuals to contribute to the well-being of their country. Overall, patriotism is a multifaceted and deeply personal sentiment that can manifest in many different ways, depending on an individual's experiences, beliefs, and values.",
+      "subField2": 456
+    },
+    "tupleField": ["exampleString", 123]
+  }
+}

tests/reliability/complex_types/generated/test_many_types_1/program.py

@@ -0,0 +1,120 @@
+### Input models ###
+
+
+from datetime import datetime
+from enum import Enum
+from typing import List, Tuple
+
+from pydantic import BaseModel, Field
+
+
+class EnumField(Enum):
+    option1 = "option1"
+    option2 = "option2"
+    option3 = "option3"
+
+
+class LiteralField(Enum):
+    literalValue = "literalValue"
+
+
+class ObjectField(BaseModel):
+    subField1: str
+    subField2: float
+
+
+class NestedObjectField(BaseModel):
+    tupleField: Tuple[str, float]
+    enumField: EnumField
+    datetimeField: datetime
+    literalField: LiteralField
+
+
+class ProgramInputs(BaseModel):
+    tupleField: Tuple[str, float]
+    enumField: EnumField
+    datetimeField: datetime
+    literalField: LiteralField
+    objectField: ObjectField
+    nestedObjectField: NestedObjectField
+
+
+### Output models ###
+
+
+from datetime import datetime
+from enum import Enum
+from typing import List, Tuple, Union
+
+from pydantic import BaseModel, Field
+
+
+class ProcessedEnumField(Enum):
+    option1 = "option1"
+    option2 = "option2"
+    option3 = "option3"
+
+
+class ProcessedLiteralField(Enum):
+    literalValue = "literalValue"
+
+
+class ProcessedObjectField(BaseModel):
+    subField1: str
+    subField2: float
+    additionalField: bool
+
+
+class EnumField(Enum):
+    option1 = "option1"
+    option2 = "option2"
+    option3 = "option3"
+
+
+class LiteralField(Enum):
+    literalValue = "literalValue"
+
+
+class ProcessedNestedObjectField(BaseModel):
+    tupleField: List[Union[str, float]] = Field(..., max_items=2, min_items=2)
+    enumField: EnumField
+    datetimeField: datetime
+    literalField: LiteralField
+    additionalField: bool
+
+
+class ProgramOutputs(BaseModel):
+    processedTupleField: Tuple[str, float]
+    processedEnumField: ProcessedEnumField
+    processedDatetimeField: datetime
+    processedLiteralField: ProcessedLiteralField
+    processedObjectField: ProcessedObjectField
+    processedNestedObjectField: ProcessedNestedObjectField
+
+
+### Program definition ###
+
+import dspy
+
+
+class BaseSignature(dspy.Signature):
+    """
+    The program is designed to process various data types including tuples, enums, datetime values, literals, objects, and nested objects containing these types. The program will accept inputs of these types, perform specified operations on them, and return the results. The operations could include validation, transformation, and extraction of information from these inputs.
+    """
+
+
+program_signature = BaseSignature
+for input_field_name, input_field in ProgramInputs.model_fields.items():
+    program_signature = program_signature.append(
+        name=input_field_name,
+        field=dspy.InputField(description=input_field.description),
+        type_=input_field.annotation,
+    )
+for output_field_name, output_field in ProgramOutputs.model_fields.items():
+    program_signature = program_signature.append(
+        name=output_field_name,
+        field=dspy.OutputField(description=input_field.description),
+        type_=output_field.annotation,
+    )
+
+program = dspy.Predict(program_signature)

tests/reliability/complex_types/generated/test_many_types_1/schema.json

@@ -0,0 +1,184 @@
+{
+  "description": "The program is designed to process various data types including tuples, enums, datetime values, literals, objects, and nested objects containing these types. The program will accept inputs of these types, perform specified operations on them, and return the results. The operations could include validation, transformation, and extraction of information from these inputs.",
+  "properties": {
+    "datetimeField": {
+      "desc": null,
+      "format": "date-time",
+      "prefix": "Datetime Field:",
+      "type": "string"
+    },
+    "enumField": {
+      "enum": ["option1", "option2", "option3"],
+      "type": "string"
+    },
+    "literalField": {
+      "const": "literalValue",
+      "enum": ["literalValue"],
+      "type": "string"
+    },
+    "nestedObjectField": {
+      "properties": {
+        "datetimeField": {
+          "format": "date-time",
+          "type": "string"
+        },
+        "enumField": {
+          "enum": ["option1", "option2", "option3"],
+          "type": "string"
+        },
+        "literalField": {
+          "const": "literalValue",
+          "enum": ["literalValue"],
+          "type": "string"
+        },
+        "tupleField": {
+          "items": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "number"
+              }
+            ]
+          },
+          "maxItems": 2,
+          "minItems": 2,
+          "type": "array"
+        }
+      },
+      "required": ["tupleField", "enumField", "datetimeField", "literalField"],
+      "type": "object"
+    },
+    "objectField": {
+      "properties": {
+        "subField1": {
+          "type": "string"
+        },
+        "subField2": {
+          "type": "number"
+        }
+      },
+      "required": ["subField1", "subField2"],
+      "type": "object"
+    },
+    "processedDatetimeField": {
+      "desc": null,
+      "format": "date-time",
+      "prefix": "Processed Datetime Field:",
+      "type": "string"
+    },
+    "processedEnumField": {
+      "enum": ["option1", "option2", "option3"],
+      "type": "string"
+    },
+    "processedLiteralField": {
+      "const": "literalValue",
+      "enum": ["literalValue"],
+      "type": "string"
+    },
+    "processedNestedObjectField": {
+      "properties": {
+        "additionalField": {
+          "type": "boolean"
+        },
+        "datetimeField": {
+          "format": "date-time",
+          "type": "string"
+        },
+        "enumField": {
+          "enum": ["option1", "option2", "option3"],
+          "type": "string"
+        },
+        "literalField": {
+          "const": "literalValue",
+          "enum": ["literalValue"],
+          "type": "string"
+        },
+        "tupleField": {
+          "items": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "number"
+              }
+            ]
+          },
+          "maxItems": 2,
+          "minItems": 2,
+          "type": "array"
+        }
+      },
+      "required": [
+        "tupleField",
+        "enumField",
+        "datetimeField",
+        "literalField",
+        "additionalField"
+      ],
+      "type": "object"
+    },
+    "processedObjectField": {
+      "properties": {
+        "additionalField": {
+          "type": "boolean"
+        },
+        "subField1": {
+          "type": "string"
+        },
+        "subField2": {
+          "type": "number"
+        }
+      },
+      "required": ["subField1", "subField2", "additionalField"],
+      "type": "object"
+    },
+    "processedTupleField": {
+      "desc": null,
+      "items": {
+        "anyOf": [
+          {
+            "type": "string"
+          },
+          {
+            "type": "number"
+          }
+        ]
+      },
+      "prefix": "Processed Tuple Field:",
+      "type": "array"
+    },
+    "tupleField": {
+      "desc": null,
+      "items": {
+        "anyOf": [
+          {
+            "type": "string"
+          },
+          {
+            "type": "number"
+          }
+        ]
+      },
+      "prefix": "Tuple Field:",
+      "type": "array"
+    }
+  },
+  "required": [
+    "tupleField",
+    "enumField",
+    "datetimeField",
+    "literalField",
+    "objectField",
+    "nestedObjectField",
+    "processedTupleField",
+    "processedEnumField",
+    "processedDatetimeField",
+    "processedLiteralField",
+    "processedObjectField",
+    "processedNestedObjectField"
+  ],
+  "type": "object"
+}

tests/reliability/complex_types/generated/test_nesting_1/inputs/input1.json

@@ -0,0 +1,27 @@
+{
+  "assertions": [
+    "The top-level output should contain the key 'resultLevel1'.",
+    "'resultLevel1' should contain the key 'resultLevel2'.",
+    "'resultLevel2' should contain the key 'resultLevel3'.",
+    "'resultLevel3' should contain the key 'resultLevel4'.",
+    "'resultLevel4' should contain the key 'resultLevel5'.",
+    "'resultLevel5' should contain the key 'outputField1' which should be of type boolean.",
+    "'resultLevel5' should contain the key 'outputField2' which should be an array of strings.",
+    "'outputField1' should indicate success or failure with a boolean value.",
+    "'outputField2' should contain messages represented as strings."
+  ],
+  "input": {
+    "level1": {
+      "level2": {
+        "level3": {
+          "level4": {
+            "level5": {
+              "field1": "test_string",
+              "field2": 42
+            }
+          }
+        }
+      }
+    }
+  }
+}