diff --git a/tests/reliability/complex_types/generated/test_many_types_1/inputs/input1.json b/tests/reliability/complex_types/generated/test_many_types_1/inputs/input1.json new file mode 100644 index 000000000..91b2530aa --- /dev/null +++ b/tests/reliability/complex_types/generated/test_many_types_1/inputs/input1.json @@ -0,0 +1,26 @@ +{ + "assertions": [ + "The 'processedTupleField' should be a tuple containing a string and a number. Note that 'processedNestedObjectField.tupleField' should NOT actually be a tuple.", + "The 'processedEnumField' should be one of the allowed enum values: 'option1', 'option2', or 'option3'.", + "The 'processedDatetimeField' should be a date-time", + "The 'processedLiteralField' should be exactly 'literalValue'.", + "The 'processedObjectField' should contain 'subField1' (string), 'subField2' (number), and an additional boolean field 'additionalField'.", + "The 'processedNestedObjectField' should contain 'tupleField' (which is actually a list with a string and a number - the name is misleading), 'enumField' (one of the allowed enum values), 'datetimeField' (string formatted as date-time), 'literalField' (exactly 'literalValue'), and an additional boolean field 'additionalField'." + ], + "input": { + "datetimeField": "2023-10-12T07:20:50.52Z", + "enumField": "option1", + "literalField": "literalValue", + "nestedObjectField": { + "datetimeField": "2023-10-12T07:20:50.52Z", + "enumField": "option2", + "literalField": "literalValue", + "tupleField": ["nestedString", 789] + }, + "objectField": { + "subField1": "example", + "subField2": 456 + }, + "tupleField": ["string1", 123] + } +} diff --git a/tests/reliability/complex_types/generated/test_many_types_1/inputs/input2.json b/tests/reliability/complex_types/generated/test_many_types_1/inputs/input2.json new file mode 100644 index 000000000..bca9c80f9 --- /dev/null +++ b/tests/reliability/complex_types/generated/test_many_types_1/inputs/input2.json @@ -0,0 +1,26 @@ +{ + "assertions": [ + "The 'processedTupleField' should be an tuple with exactly two elements: the first element being a string and the second element being a number. Note that 'processedNestedObjectField.tupleField' should NOT actually be a tuple", + "The 'processedEnumField' should be one of the predefined options: 'option1', 'option2', or 'option3'.", + "The 'processedDatetimeField' should be a date-time", + "The 'processedLiteralField' should be the enum 'literalValue'.", + "The 'processedObjectField' should be an object containing 'subField1' as a string, 'subField2' as a number, and an 'additionalField' as a boolean.", + "The 'processedNestedObjectField' should be an object containing 'tupleField' as a list (NOT a tuple) with exactly two elements (a string and a number), 'enumField' as one of the predefined options (option1, option2, or option3), 'datetimeField' as a 'date-time' object, 'literalField' as the string 'literalValue', and an 'additionalField' as a boolean." + ], + "input": { + "datetimeField": "2023-10-01T12:00:00Z", + "enumField": "option1", + "literalField": "literalValue", + "nestedObjectField": { + "datetimeField": "2023-11-01T12:00:00Z", + "enumField": "option2", + "literalField": "literalValue", + "tupleField": ["nestedString", 789] + }, + "objectField": { + "subField1": "Patriotism is a feeling of love, devotion, and sense of attachment to one's country. This attachment can be a combination of many different feelings relating to one's homeland, including ethnic, cultural, political or historical aspects. It encompasses a set of concepts closely related to those of nationalism. In the context of patriotism, people may express their feelings in a variety of ways, including supporting their country's interests and policies, celebrating national holidays, and participating in civic activities. Patriotism often involves a sense of pride in one's country and a willingness to defend it against any threats. It can also include a commitment to improving the country and making it a better place for future generations. The concept of patriotism is often linked with the idea of national identity, which is the sense of a nation as a cohesive whole, as represented by distinctive traditions, culture, language, and politics. Patriots may feel a strong sense of loyalty and duty to their country, and they may take actions to support and protect it. However, it is important to note that patriotism can also be a complex and sometimes controversial concept. While it can inspire positive actions and a sense of community, it can also lead to exclusionary or aggressive behaviors if taken to an extreme. In some cases, excessive patriotism can result in nationalism, which can lead to conflicts with other nations or groups. Despite these potential issues, many people view patriotism as a positive force that can unite people and inspire them to work together for the common good. It can foster a sense of belonging and purpose, and it can motivate individuals to contribute to the well-being of their country. Overall, patriotism is a multifaceted and deeply personal sentiment that can manifest in many different ways, depending on an individual's experiences, beliefs, and values.", + "subField2": 456 + }, + "tupleField": ["exampleString", 123] + } +} diff --git a/tests/reliability/complex_types/generated/test_many_types_1/program.py b/tests/reliability/complex_types/generated/test_many_types_1/program.py new file mode 100644 index 000000000..52798903b --- /dev/null +++ b/tests/reliability/complex_types/generated/test_many_types_1/program.py @@ -0,0 +1,120 @@ +### Input models ### + + +from datetime import datetime +from enum import Enum +from typing import List, Tuple + +from pydantic import BaseModel, Field + + +class EnumField(Enum): + option1 = "option1" + option2 = "option2" + option3 = "option3" + + +class LiteralField(Enum): + literalValue = "literalValue" + + +class ObjectField(BaseModel): + subField1: str + subField2: float + + +class NestedObjectField(BaseModel): + tupleField: Tuple[str, float] + enumField: EnumField + datetimeField: datetime + literalField: LiteralField + + +class ProgramInputs(BaseModel): + tupleField: Tuple[str, float] + enumField: EnumField + datetimeField: datetime + literalField: LiteralField + objectField: ObjectField + nestedObjectField: NestedObjectField + + +### Output models ### + + +from datetime import datetime +from enum import Enum +from typing import List, Tuple, Union + +from pydantic import BaseModel, Field + + +class ProcessedEnumField(Enum): + option1 = "option1" + option2 = "option2" + option3 = "option3" + + +class ProcessedLiteralField(Enum): + literalValue = "literalValue" + + +class ProcessedObjectField(BaseModel): + subField1: str + subField2: float + additionalField: bool + + +class EnumField(Enum): + option1 = "option1" + option2 = "option2" + option3 = "option3" + + +class LiteralField(Enum): + literalValue = "literalValue" + + +class ProcessedNestedObjectField(BaseModel): + tupleField: List[Union[str, float]] = Field(..., max_items=2, min_items=2) + enumField: EnumField + datetimeField: datetime + literalField: LiteralField + additionalField: bool + + +class ProgramOutputs(BaseModel): + processedTupleField: Tuple[str, float] + processedEnumField: ProcessedEnumField + processedDatetimeField: datetime + processedLiteralField: ProcessedLiteralField + processedObjectField: ProcessedObjectField + processedNestedObjectField: ProcessedNestedObjectField + + +### Program definition ### + +import dspy + + +class BaseSignature(dspy.Signature): + """ + The program is designed to process various data types including tuples, enums, datetime values, literals, objects, and nested objects containing these types. The program will accept inputs of these types, perform specified operations on them, and return the results. The operations could include validation, transformation, and extraction of information from these inputs. + """ + + +program_signature = BaseSignature +for input_field_name, input_field in ProgramInputs.model_fields.items(): + program_signature = program_signature.append( + name=input_field_name, + field=dspy.InputField(description=input_field.description), + type_=input_field.annotation, + ) +for output_field_name, output_field in ProgramOutputs.model_fields.items(): + program_signature = program_signature.append( + name=output_field_name, + field=dspy.OutputField(description=input_field.description), + type_=output_field.annotation, + ) + +program = dspy.Predict(program_signature) diff --git a/tests/reliability/complex_types/generated/test_many_types_1/schema.json b/tests/reliability/complex_types/generated/test_many_types_1/schema.json new file mode 100644 index 000000000..1e102dc8b --- /dev/null +++ b/tests/reliability/complex_types/generated/test_many_types_1/schema.json @@ -0,0 +1,184 @@ +{ + "description": "The program is designed to process various data types including tuples, enums, datetime values, literals, objects, and nested objects containing these types. The program will accept inputs of these types, perform specified operations on them, and return the results. The operations could include validation, transformation, and extraction of information from these inputs.", + "properties": { + "datetimeField": { + "desc": null, + "format": "date-time", + "prefix": "Datetime Field:", + "type": "string" + }, + "enumField": { + "enum": ["option1", "option2", "option3"], + "type": "string" + }, + "literalField": { + "const": "literalValue", + "enum": ["literalValue"], + "type": "string" + }, + "nestedObjectField": { + "properties": { + "datetimeField": { + "format": "date-time", + "type": "string" + }, + "enumField": { + "enum": ["option1", "option2", "option3"], + "type": "string" + }, + "literalField": { + "const": "literalValue", + "enum": ["literalValue"], + "type": "string" + }, + "tupleField": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + } + }, + "required": ["tupleField", "enumField", "datetimeField", "literalField"], + "type": "object" + }, + "objectField": { + "properties": { + "subField1": { + "type": "string" + }, + "subField2": { + "type": "number" + } + }, + "required": ["subField1", "subField2"], + "type": "object" + }, + "processedDatetimeField": { + "desc": null, + "format": "date-time", + "prefix": "Processed Datetime Field:", + "type": "string" + }, + "processedEnumField": { + "enum": ["option1", "option2", "option3"], + "type": "string" + }, + "processedLiteralField": { + "const": "literalValue", + "enum": ["literalValue"], + "type": "string" + }, + "processedNestedObjectField": { + "properties": { + "additionalField": { + "type": "boolean" + }, + "datetimeField": { + "format": "date-time", + "type": "string" + }, + "enumField": { + "enum": ["option1", "option2", "option3"], + "type": "string" + }, + "literalField": { + "const": "literalValue", + "enum": ["literalValue"], + "type": "string" + }, + "tupleField": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + } + }, + "required": [ + "tupleField", + "enumField", + "datetimeField", + "literalField", + "additionalField" + ], + "type": "object" + }, + "processedObjectField": { + "properties": { + "additionalField": { + "type": "boolean" + }, + "subField1": { + "type": "string" + }, + "subField2": { + "type": "number" + } + }, + "required": ["subField1", "subField2", "additionalField"], + "type": "object" + }, + "processedTupleField": { + "desc": null, + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + }, + "prefix": "Processed Tuple Field:", + "type": "array" + }, + "tupleField": { + "desc": null, + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + }, + "prefix": "Tuple Field:", + "type": "array" + } + }, + "required": [ + "tupleField", + "enumField", + "datetimeField", + "literalField", + "objectField", + "nestedObjectField", + "processedTupleField", + "processedEnumField", + "processedDatetimeField", + "processedLiteralField", + "processedObjectField", + "processedNestedObjectField" + ], + "type": "object" +} diff --git a/tests/reliability/complex_types/generated/test_nesting_1/inputs/input1.json b/tests/reliability/complex_types/generated/test_nesting_1/inputs/input1.json new file mode 100644 index 000000000..13fbe99ae --- /dev/null +++ b/tests/reliability/complex_types/generated/test_nesting_1/inputs/input1.json @@ -0,0 +1,27 @@ +{ + "assertions": [ + "The top-level output should contain the key 'resultLevel1'.", + "'resultLevel1' should contain the key 'resultLevel2'.", + "'resultLevel2' should contain the key 'resultLevel3'.", + "'resultLevel3' should contain the key 'resultLevel4'.", + "'resultLevel4' should contain the key 'resultLevel5'.", + "'resultLevel5' should contain the key 'outputField1' which should be of type boolean.", + "'resultLevel5' should contain the key 'outputField2' which should be an array of strings.", + "'outputField1' should indicate success or failure with a boolean value.", + "'outputField2' should contain messages represented as strings." + ], + "input": { + "level1": { + "level2": { + "level3": { + "level4": { + "level5": { + "field1": "test_string", + "field2": 42 + } + } + } + } + } + } +} diff --git a/tests/reliability/complex_types/generated/test_nesting_1/inputs/input2.json b/tests/reliability/complex_types/generated/test_nesting_1/inputs/input2.json new file mode 100644 index 000000000..1d2cdfd00 --- /dev/null +++ b/tests/reliability/complex_types/generated/test_nesting_1/inputs/input2.json @@ -0,0 +1,25 @@ +{ + "assertions": [ + "The output should have a top-level field named 'resultLevel1'.", + "Within 'resultLevel1', there should be a nested field named 'resultLevel2'.", + "Within 'resultLevel2', there should be a nested field named 'resultLevel3'.", + "Within 'resultLevel3', there should be a nested field named 'resultLevel4'.", + "Within 'resultLevel4', there should be a nested field named 'resultLevel5'.", + "Within 'resultLevel5', there should be a field named 'outputField1' which must be of type boolean.", + "Within 'resultLevel5', there should be a field named 'outputField2' which must be an array of strings." + ], + "input": { + "level1": { + "level2": { + "level3": { + "level4": { + "level5": { + "field1": "test string", + "field2": 123.45 + } + } + } + } + } + } +} diff --git a/tests/reliability/complex_types/generated/test_nesting_1/program.py b/tests/reliability/complex_types/generated/test_nesting_1/program.py new file mode 100644 index 000000000..607f1675d --- /dev/null +++ b/tests/reliability/complex_types/generated/test_nesting_1/program.py @@ -0,0 +1,90 @@ +### Input models ### + + +from pydantic import BaseModel, Field + + +class Level5(BaseModel): + field1: str = Field(..., description="A string field at the deepest level") + field2: float = Field(..., description="A numerical field at the deepest level") + + +class Level4(BaseModel): + level5: Level5 + + +class Level3(BaseModel): + level4: Level4 + + +class Level2(BaseModel): + level3: Level3 + + +class Level1(BaseModel): + level2: Level2 + + +class ProgramInputs(BaseModel): + level1: Level1 + + +### Output models ### + + +from typing import List + +from pydantic import BaseModel, Field + + +class ResultLevel5(BaseModel): + outputField1: bool = Field(..., description="A boolean field indicating success or failure") + outputField2: List[str] = Field(..., description="An array of strings representing messages") + + +class ResultLevel4(BaseModel): + resultLevel5: ResultLevel5 + + +class ResultLevel3(BaseModel): + resultLevel4: ResultLevel4 + + +class ResultLevel2(BaseModel): + resultLevel3: ResultLevel3 + + +class ResultLevel1(BaseModel): + resultLevel2: ResultLevel2 + + +class ProgramOutputs(BaseModel): + resultLevel1: ResultLevel1 + + +### Program definition ### + +import dspy + + +class BaseSignature(dspy.Signature): + """ + The AI program is designed to process hierarchical data structures with multiple levels of nesting. The program will take a deeply nested input structure representing a complex dataset, perform specific transformations, validations, and computations, and then produce an equally complex nested output structure. The program is suitable for applications that require detailed data processing, such as multi-level data aggregation, hierarchical data validation, and nested data transformation. + """ + + +program_signature = BaseSignature +for input_field_name, input_field in ProgramInputs.model_fields.items(): + program_signature = program_signature.append( + name=input_field_name, + field=dspy.InputField(description=input_field.description), + type_=input_field.annotation, + ) +for output_field_name, output_field in ProgramOutputs.model_fields.items(): + program_signature = program_signature.append( + name=output_field_name, + field=dspy.OutputField(description=input_field.description), + type_=output_field.annotation, + ) + +program = dspy.Predict(program_signature) diff --git a/tests/reliability/complex_types/generated/test_nesting_1/schema.json b/tests/reliability/complex_types/generated/test_nesting_1/schema.json new file mode 100644 index 000000000..7cac60390 --- /dev/null +++ b/tests/reliability/complex_types/generated/test_nesting_1/schema.json @@ -0,0 +1,86 @@ +{ + "description": "The AI program is designed to process hierarchical data structures with multiple levels of nesting. The program will take a deeply nested input structure representing a complex dataset, perform specific transformations, validations, and computations, and then produce an equally complex nested output structure. The program is suitable for applications that require detailed data processing, such as multi-level data aggregation, hierarchical data validation, and nested data transformation.", + "properties": { + "level1": { + "properties": { + "level2": { + "properties": { + "level3": { + "properties": { + "level4": { + "properties": { + "level5": { + "properties": { + "field1": { + "description": "A string field at the deepest level", + "type": "string" + }, + "field2": { + "description": "A numerical field at the deepest level", + "type": "number" + } + }, + "required": ["field1", "field2"], + "type": "object" + } + }, + "required": ["level5"], + "type": "object" + } + }, + "required": ["level4"], + "type": "object" + } + }, + "required": ["level3"], + "type": "object" + } + }, + "required": ["level2"], + "type": "object" + }, + "resultLevel1": { + "properties": { + "resultLevel2": { + "properties": { + "resultLevel3": { + "properties": { + "resultLevel4": { + "properties": { + "resultLevel5": { + "properties": { + "outputField1": { + "description": "A boolean field indicating success or failure", + "type": "boolean" + }, + "outputField2": { + "description": "An array of strings representing messages", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": ["outputField1", "outputField2"], + "type": "object" + } + }, + "required": ["resultLevel5"], + "type": "object" + } + }, + "required": ["resultLevel4"], + "type": "object" + } + }, + "required": ["resultLevel3"], + "type": "object" + } + }, + "required": ["resultLevel2"], + "type": "object" + } + }, + "required": ["level1", "resultLevel1"], + "type": "object" +} diff --git a/tests/reliability/complex_types/generated/test_nesting_2/inputs/input1.json b/tests/reliability/complex_types/generated/test_nesting_2/inputs/input1.json new file mode 100644 index 000000000..fc0694b74 --- /dev/null +++ b/tests/reliability/complex_types/generated/test_nesting_2/inputs/input1.json @@ -0,0 +1,30 @@ +{ + "assertions": [ + "The output should contain a 'customer_summary' object with the required properties: 'customer_id', 'customer_type', and 'value'.", + "'customer_summary.customer_id' should be a string and match the 'customer_id' from the input.", + "'customer_summary.customer_type' should be an object containing 'is_premium' (a boolean) and 'category' (a string).", + "'customer_summary.value' should be a string and reflect the 'value' from the input's customer details.", + "The output should contain a 'transaction_summary' object with the required properties: 'transaction_id', 'total_amount', and 'details'.", + "'transaction_summary.transaction_id' should be a string and match the 'transaction_id' from the input.", + "'transaction_summary.total_amount' should be a number and match the 'amount' from the input.", + "'transaction_summary.details' should be an object containing 'value' (a number) and 'timestamp' (a date-time value)." + ], + "input": { + "customer": { + "customer_id": "C12345", + "customer_type": true, + "details": { + "age": 30, + "value": "Gold" + } + }, + "transaction": { + "amount": 150.75, + "details": { + "timestamp": "2023-10-01T10:00:00Z", + "value": 150.75 + }, + "transaction_id": "T98765" + } + } +} diff --git a/tests/reliability/complex_types/generated/test_nesting_2/program.py b/tests/reliability/complex_types/generated/test_nesting_2/program.py new file mode 100644 index 000000000..ebf848e43 --- /dev/null +++ b/tests/reliability/complex_types/generated/test_nesting_2/program.py @@ -0,0 +1,96 @@ +### Input models ### + + +from datetime import datetime + +from pydantic import BaseModel, Field + + +class Details(BaseModel): + value: str = Field(..., description="Customer's value category") + age: int = Field(..., description="Customer's age") + + +class Customer(BaseModel): + customer_id: str = Field(..., description="Unique identifier for the customer") + customer_type: bool = Field(..., description="Indicates if the customer is a premium member") + details: Details + + +class Details1(BaseModel): + value: float = Field(..., description="Monetary value of the transaction") + timestamp: datetime = Field(..., description="Timestamp of the transaction") + + +class Transaction(BaseModel): + transaction_id: str = Field(..., description="Unique identifier for the transaction") + amount: float = Field(..., description="Transaction amount") + details: Details1 + + +class ProgramInputs(BaseModel): + customer: Customer + transaction: Transaction + + +### Output models ### + + +from datetime import datetime + +from pydantic import BaseModel, Field + + +class CustomerType(BaseModel): + is_premium: bool = Field(..., description="Indicates if the customer is a premium member") + category: str = Field(..., description="Customer's membership category") + + +class CustomerSummary(BaseModel): + customer_id: str = Field(..., description="Unique identifier for the customer") + customer_type: CustomerType + value: str = Field(..., description="Customer's value category") + + +class Details(BaseModel): + value: float = Field(..., description="Monetary value of the transaction") + timestamp: datetime = Field(..., description="Timestamp of the transaction") + + +class TransactionSummary(BaseModel): + transaction_id: str = Field(..., description="Unique identifier for the transaction") + total_amount: float = Field(..., description="Total transaction amount") + details: Details + + +class ProgramOutputs(BaseModel): + customer_summary: CustomerSummary + transaction_summary: TransactionSummary + + +### Program definition ### + +import dspy + + +class BaseSignature(dspy.Signature): + """ + This AI program is designed to process complex datasets with multiple nested input fields and produce structured output fields. It can handle cases where nested fields have the same name but different types, ensuring that the data is accurately processed and transformed. The program is particularly useful for applications that require detailed data analysis, integration of multiple data sources, and handling of heterogeneous data types. + """ + + +program_signature = BaseSignature +for input_field_name, input_field in ProgramInputs.model_fields.items(): + program_signature = program_signature.append( + name=input_field_name, + field=dspy.InputField(description=input_field.description), + type_=input_field.annotation, + ) +for output_field_name, output_field in ProgramOutputs.model_fields.items(): + program_signature = program_signature.append( + name=output_field_name, + field=dspy.OutputField(description=input_field.description), + type_=output_field.annotation, + ) + +program = dspy.ChainOfThought(program_signature) diff --git a/tests/reliability/complex_types/generated/test_nesting_2/schema.json b/tests/reliability/complex_types/generated/test_nesting_2/schema.json new file mode 100644 index 000000000..1ff44d265 --- /dev/null +++ b/tests/reliability/complex_types/generated/test_nesting_2/schema.json @@ -0,0 +1,126 @@ +{ + "description": "This AI program is designed to process complex datasets with multiple nested input fields and produce structured output fields. It can handle cases where nested fields have the same name but different types, ensuring that the data is accurately processed and transformed. The program is particularly useful for applications that require detailed data analysis, integration of multiple data sources, and handling of heterogeneous data types.", + "properties": { + "customer": { + "properties": { + "customer_id": { + "description": "Unique identifier for the customer", + "type": "string" + }, + "customer_type": { + "description": "Indicates if the customer is a premium member", + "type": "boolean" + }, + "details": { + "properties": { + "age": { + "description": "Customer's age", + "type": "integer" + }, + "value": { + "description": "Customer's value category", + "type": "string" + } + }, + "required": ["value", "age"], + "type": "object" + } + }, + "required": ["customer_id", "customer_type", "details"], + "type": "object" + }, + "customer_summary": { + "properties": { + "customer_id": { + "description": "Unique identifier for the customer", + "type": "string" + }, + "customer_type": { + "properties": { + "category": { + "description": "Customer's membership category", + "type": "string" + }, + "is_premium": { + "description": "Indicates if the customer is a premium member", + "type": "boolean" + } + }, + "required": ["is_premium", "category"], + "type": "object" + }, + "value": { + "description": "Customer's value category", + "type": "string" + } + }, + "required": ["customer_id", "customer_type", "value"], + "type": "object" + }, + "transaction": { + "properties": { + "amount": { + "description": "Transaction amount", + "type": "number" + }, + "details": { + "properties": { + "timestamp": { + "description": "Timestamp of the transaction", + "format": "date-time", + "type": "string" + }, + "value": { + "description": "Monetary value of the transaction", + "type": "number" + } + }, + "required": ["value", "timestamp"], + "type": "object" + }, + "transaction_id": { + "description": "Unique identifier for the transaction", + "type": "string" + } + }, + "required": ["transaction_id", "amount", "details"], + "type": "object" + }, + "transaction_summary": { + "properties": { + "details": { + "properties": { + "timestamp": { + "description": "Timestamp of the transaction", + "format": "date-time", + "type": "string" + }, + "value": { + "description": "Monetary value of the transaction", + "type": "number" + } + }, + "required": ["value", "timestamp"], + "type": "object" + }, + "total_amount": { + "description": "Total transaction amount", + "type": "number" + }, + "transaction_id": { + "description": "Unique identifier for the transaction", + "type": "string" + } + }, + "required": ["transaction_id", "total_amount", "details"], + "type": "object" + } + }, + "required": [ + "customer", + "transaction", + "customer_summary", + "transaction_summary" + ], + "type": "object" +} diff --git a/tests/reliability/input_formats/generated/test_markdown_1/inputs/input1.json b/tests/reliability/input_formats/generated/test_markdown_1/inputs/input1.json new file mode 100644 index 000000000..b8c5d5ad3 --- /dev/null +++ b/tests/reliability/input_formats/generated/test_markdown_1/inputs/input1.json @@ -0,0 +1,11 @@ +{ + "assertions": [ + "Each top-level heading (indicated by `#`) should appear as a top-level entry in the TOC.", + "Each second-level heading (indicated by `##`) should be nested under the appropriate top-level heading in the TOC.", + "Each third-level heading (indicated by `###`) should be nested under the appropriate second-level heading in the TOC.", + "Each entry in the TOC should be linked to the corresponding section in the document, using markdown link syntax." + ], + "input": { + "markdown_content": "# The American Space Program\n\nThe American space program has a rich history of exploration and discovery.\n\n## Early Beginnings\n\nThe journey began in the late 1950s with the launch of the first artificial satellite.\n\n### The Space Race\n\nThe competition between the United States and the Soviet Union led to rapid advancements in space technology.\n\n## Moon Landing\n\nIn 1969, NASA successfully landed the first humans on the moon.\n\n### Apollo Missions\n\nThe Apollo missions were a series of spaceflights that landed humans on the moon and brought them back safely.\n\n## Space Shuttle Era\n\nThe development of the Space Shuttle program marked a new era in space exploration.\n\n### Reusable Spacecraft\n\nThe Space Shuttle was the first reusable spacecraft, capable of multiple missions.\n\n## International Space Station\n\nThe International Space Station (ISS) is a collaborative effort between multiple countries.\n\n### Living in Space\n\nAstronauts live and work on the ISS for extended periods, conducting scientific research.\n\n## Future Missions\n\nNASA continues to plan for future missions to Mars and beyond.\n\n### Mars Exploration\n\nExploration of Mars is a key objective for NASA's future missions.\n\n### Beyond Mars\n\nThe ultimate goal is to explore beyond Mars and into the outer reaches of the solar system.\n\n## Conclusion\n\nThe American space program has achieved many milestones and continues to push the boundaries of space exploration." + } +} diff --git a/tests/reliability/input_formats/generated/test_markdown_1/inputs/input2.json b/tests/reliability/input_formats/generated/test_markdown_1/inputs/input2.json new file mode 100644 index 000000000..2ad412b87 --- /dev/null +++ b/tests/reliability/input_formats/generated/test_markdown_1/inputs/input2.json @@ -0,0 +1,11 @@ +{ + "assertions": [ + "Each entry in the TOC should be a markdown link pointing to the corresponding section in the document.", + "The hierarchy of the TOC should match the levels of headings in the input markdown content (e.g., H1 headings as top-level, H2 headings nested under H1, etc.).", + "The TOC should include all headings from the input markdown content, in the order they appear.", + "The TOC should not include any non-heading content from the input markdown document." + ], + "input": { + "markdown_content": "# Introduction\n\nThis is the introduction section.\n\n## Overview\n\nAn overview of the document.\n\n### Details\n\nMore detailed information.\n\n#### Subdetails\n\nEven more detailed information.\n\n## Another Section\n\nContent of another section.\n\n### Subsection\n\nDetails of the subsection.\n\n```python\ndef example_function():\n print(\"Hello, World!\")\n```\n\n# Conclusion\n\nFinal thoughts." + } +} diff --git a/tests/reliability/input_formats/generated/test_markdown_1/program.py b/tests/reliability/input_formats/generated/test_markdown_1/program.py new file mode 100644 index 000000000..435a47aea --- /dev/null +++ b/tests/reliability/input_formats/generated/test_markdown_1/program.py @@ -0,0 +1,49 @@ +### Input models ### + + +from pydantic import BaseModel, Field + + +class ProgramInputs(BaseModel): + markdown_content: str = Field( + ..., + description="The content of the markdown document from which the table of contents will be generated.", + ) + + +### Output models ### + + +from pydantic import BaseModel, Field + + +class ProgramOutputs(BaseModel): + table_of_contents: str = Field(..., description="The generated table of contents in markdown format.") + + +### Program definition ### + +import dspy + + +class BaseSignature(dspy.Signature): + """ + The program is designed to generate a table of contents (TOC) from a given markdown document. It will parse the markdown content, identify headings, and create a hierarchical TOC based on the heading levels. The TOC will be presented in markdown format, with each entry linked to the corresponding section in the document. + """ + + +program_signature = BaseSignature +for input_field_name, input_field in ProgramInputs.model_fields.items(): + program_signature = program_signature.append( + name=input_field_name, + field=dspy.InputField(description=input_field.description), + type_=input_field.annotation, + ) +for output_field_name, output_field in ProgramOutputs.model_fields.items(): + program_signature = program_signature.append( + name=output_field_name, + field=dspy.OutputField(description=input_field.description), + type_=output_field.annotation, + ) + +program = dspy.ChainOfThought(program_signature) diff --git a/tests/reliability/input_formats/generated/test_markdown_1/schema.json b/tests/reliability/input_formats/generated/test_markdown_1/schema.json new file mode 100644 index 000000000..56d9790f8 --- /dev/null +++ b/tests/reliability/input_formats/generated/test_markdown_1/schema.json @@ -0,0 +1,19 @@ +{ + "description": "The program is designed to generate a table of contents (TOC) from a given markdown document. It will parse the markdown content, identify headings, and create a hierarchical TOC based on the heading levels. The TOC will be presented in markdown format, with each entry linked to the corresponding section in the document.", + "properties": { + "markdown_content": { + "desc": "The content of the markdown document from which the table of contents will be generated.", + "description": "The content of the markdown document from which the table of contents will be generated.", + "prefix": "Markdown Content:", + "type": "string" + }, + "table_of_contents": { + "desc": "The content of the markdown document from which the table of contents will be generated.", + "description": "The content of the markdown document from which the table of contents will be generated.", + "prefix": "Table Of Contents:", + "type": "string" + } + }, + "required": ["markdown_content", "table_of_contents"], + "type": "object" +}