flow.schema.json

{
  "$schema": "https://json-schema.org/draft/2019-09/schema",
  "title": "Catalog",
  "description": "Each catalog source defines a portion of a Flow Catalog, by defining collections, derivations, tests, and materializations of the Catalog. Catalog sources may reference and import other sources, in order to collections and other entities that source defines.",
  "type": "object",
  "properties": {
    "$schema": {
      "title": "JSON-Schema against which the Catalog is validated.",
      "type": "string"
    },
    "captures": {
      "title": "Captures of this Catalog.",
      "examples": [
        {
          "acmeCo/capture": {
            "autoDiscover": {
              "addNewBindings": true,
              "evolveIncompatibleCollections": true
            },
            "bindings": [
              {
                "resource": {
                  "stream": "a_stream"
                },
                "target": "target/collection"
              }
            ],
            "endpoint": {
              "connector": {
                "config": "connector-config.yaml",
                "image": "connector/image:tag"
              }
            }
          }
        }
      ],
      "type": "object",
      "patternProperties": {
        "^[\\p{Letter}\\p{Number}\\-_\\.]+(/[\\p{Letter}\\p{Number}\\-_\\.]+)*$": {
          "$ref": "#/definitions/CaptureDef"
        }
      },
      "additionalProperties": false
    },
    "collections": {
      "title": "Collections of this Catalog.",
      "examples": [
        {
          "acmeCo/collection": {
            "key": [
              "/json/ptr"
            ],
            "schema": {
              "properties": {
                "bar": {
                  "const": 42
                },
                "foo": {
                  "type": "integer"
                }
              },
              "type": "object"
            }
          }
        }
      ],
      "type": "object",
      "patternProperties": {
        "^[\\p{Letter}\\p{Number}\\-_\\.]+(/[\\p{Letter}\\p{Number}\\-_\\.]+)*$": {
          "$ref": "#/definitions/CollectionDef"
        }
      },
      "additionalProperties": false
    },
    "import": {
      "title": "Import other Flow catalog sources.",
      "description": "By importing another Flow catalog source, its collections, schemas, and derivations are bundled into the publication context of this specification. Imports are relative or absolute URLs, relative to this specification's location.",
      "type": "array",
      "items": {
        "$ref": "#/definitions/RelativeUrl"
      }
    },
    "materializations": {
      "title": "Materializations of this Catalog.",
      "examples": [
        {
          "acmeCo/materialization": {
            "bindings": [
              {
                "fields": {
                  "recommended": true
                },
                "resource": {
                  "table": "a_table"
                },
                "source": "source/collection"
              }
            ],
            "endpoint": {
              "connector": {
                "config": "connector-config.yaml",
                "image": "connector/image:tag"
              }
            }
          }
        }
      ],
      "type": "object",
      "patternProperties": {
        "^[\\p{Letter}\\p{Number}\\-_\\.]+(/[\\p{Letter}\\p{Number}\\-_\\.]+)*$": {
          "$ref": "#/definitions/MaterializationDef"
        }
      },
      "additionalProperties": false
    },
    "tests": {
      "title": "Tests of this Catalog.",
      "examples": [
        {
          "acmeCo/conversions/test": [
            {
              "ingest": {
                "collection": "acmeCo/collection",
                "description": "Description of the ingestion.",
                "documents": [
                  {
                    "a": "document"
                  },
                  {
                    "another": "document"
                  }
                ]
              }
            },
            {
              "verify": {
                "collection": "acmeCo/collection",
                "description": "Description of the verification.",
                "documents": [
                  {
                    "a": "document"
                  },
                  {
                    "another": "document"
                  }
                ]
              }
            }
          ]
        }
      ],
      "type": "object",
      "patternProperties": {
        "^[\\p{Letter}\\p{Number}\\-_\\.]+(/[\\p{Letter}\\p{Number}\\-_\\.]+)*$": {
          "$ref": "#/definitions/TestDef"
        }
      },
      "additionalProperties": false
    }
  },
  "additionalProperties": false,
  "definitions": {
    "AutoDiscover": {
      "description": "Settings to determine how Flow should stay abreast of ongoing changes to collections and schemas.",
      "type": "object",
      "properties": {
        "addNewBindings": {
          "description": "Automatically add new bindings discovered from the source.",
          "default": false,
          "type": "boolean"
        },
        "evolveIncompatibleCollections": {
          "description": "Whether to automatically evolve collections and/or materialization bindings to handle changes to collections that would otherwise be incompatible with the existing catalog.",
          "default": false,
          "type": "boolean"
        }
      },
      "additionalProperties": false
    },
    "Capture": {
      "description": "Capture names are paths of Unicode letters, numbers, '-', '_', or '.'. Each path component is separated by a slash '/', and a name may not begin or end in a '/'.",
      "examples": [
        "acmeCo/capture"
      ],
      "type": "string",
      "pattern": "^[\\p{Letter}\\p{Number}\\-_\\.]+(/[\\p{Letter}\\p{Number}\\-_\\.]+)*$"
    },
    "CaptureBinding": {
      "examples": [
        {
          "resource": {
            "stream": "a_stream"
          },
          "target": "target/collection"
        }
      ],
      "type": "object",
      "required": [
        "resource",
        "target"
      ],
      "properties": {
        "backfill": {
          "title": "Backfill counter for this binding.",
          "description": "Every increment of this counter will result in a new backfill of this binding from the captured endpoint. For example when capturing from a SQL table, incrementing this counter will cause the table to be re-captured in its entirety from the source database.\n\nNote that a backfill does *not* truncate the target collection, and documents published by a backfilled binding will coexist with (and be ordered after) any documents which were published as part of a preceding backfill.",
          "type": "integer",
          "format": "uint32",
          "minimum": 0.0
        },
        "disable": {
          "title": "Whether to disable the binding",
          "description": "Disabled bindings are inactive, and not validated. They can be used to represent discovered resources that are intentionally not being captured.",
          "type": "boolean"
        },
        "resource": {
          "title": "Endpoint resource to capture from."
        },
        "target": {
          "title": "Name of the collection to capture into.",
          "$ref": "#/definitions/Collection"
        }
      },
      "additionalProperties": false
    },
    "CaptureDef": {
      "description": "A Capture binds an external system and target (e.x., a SQL table or cloud storage bucket) from which data should be continuously captured, with a Flow collection into that captured data is ingested. Multiple Captures may be bound to a single collection, but only one capture may exist for a given endpoint and target.",
      "type": "object",
      "required": [
        "bindings",
        "endpoint"
      ],
      "properties": {
        "autoDiscover": {
          "title": "Continuously keep the collection spec and schema up-to-date",
          "$ref": "#/definitions/AutoDiscover"
        },
        "bindings": {
          "title": "Bound collections to capture from the endpoint.",
          "type": "array",
          "items": {
            "$ref": "#/definitions/CaptureBinding"
          }
        },
        "delete": {
          "title": "Delete this capture within the control plane.",
          "description": "When true, a publication will delete this capture.",
          "type": "boolean"
        },
        "endpoint": {
          "title": "Endpoint to capture from.",
          "$ref": "#/definitions/CaptureEndpoint"
        },
        "expectPubId": {
          "title": "Expected publication ID of this capture within the control plane.",
          "description": "When present, a publication of the capture will fail if the last publication ID in the control plane doesn't match this value.",
          "$ref": "#/definitions/Id"
        },
        "interval": {
          "title": "Interval of time between invocations of the capture.",
          "description": "Configured intervals are applicable only to connectors which are unable to continuously tail their source, and which instead produce a current quantity of output and then exit. Flow will start the connector again after the given interval of time has passed.\n\nIntervals are relative to the start of an invocation and not its completion. For example, if the interval is five minutes, and an invocation of the capture finishes after two minutes, then the next invocation will be started after three additional minutes.",
          "type": [
            "string",
            "null"
          ],
          "pattern": "^\\d+(s|m|h)$"
        },
        "shards": {
          "title": "Template for shards of this capture task.",
          "$ref": "#/definitions/ShardTemplate"
        }
      },
      "additionalProperties": false
    },
    "CaptureEndpoint": {
      "description": "An endpoint from which Flow will capture.",
      "oneOf": [
        {
          "title": "A Connector.",
          "type": "object",
          "required": [
            "connector"
          ],
          "properties": {
            "connector": {
              "$ref": "#/definitions/ConnectorConfig"
            }
          },
          "additionalProperties": false
        },
        {
          "title": "A local command (development only).",
          "type": "object",
          "required": [
            "local"
          ],
          "properties": {
            "local": {
              "$ref": "#/definitions/LocalConfig"
            }
          },
          "additionalProperties": false
        }
      ]
    },
    "Collection": {
      "description": "Collection names are paths of Unicode letters, numbers, '-', '_', or '.'. Each path component is separated by a slash '/', and a name may not begin or end in a '/'.",
      "examples": [
        "acmeCo/collection"
      ],
      "type": "string",
      "pattern": "^[\\p{Letter}\\p{Number}\\-_\\.]+(/[\\p{Letter}\\p{Number}\\-_\\.]+)*$"
    },
    "CollectionDef": {
      "description": "Collection describes a set of related documents, where each adheres to a common schema and grouping key. Collections are append-only: once a document is added to a collection, it is never removed. However, it may be replaced or updated (either in whole, or in part) by a future document sharing its key. Each new document of a given key is \"reduced\" into existing documents of the key. By default, this reduction is achieved by completely replacing the previous document, but much richer reduction behaviors can be specified through the use of annotated reduction strategies of the collection schema.",
      "examples": [
        {
          "key": [
            "/json/ptr"
          ],
          "schema": {
            "properties": {
              "bar": {
                "const": 42
              },
              "foo": {
                "type": "integer"
              }
            },
            "type": "object"
          }
        }
      ],
      "type": "object",
      "oneOf": [
        {
          "required": [
            "schema"
          ],
          "properties": {
            "readSchema": false,
            "writeSchema": false
          }
        },
        {
          "required": [
            "readSchema",
            "writeSchema"
          ],
          "properties": {
            "schema": false
          }
        }
      ],
      "required": [
        "key"
      ],
      "properties": {
        "delete": {
          "title": "Delete this collection within the control plane.",
          "description": "When true, a publication will delete this collection.",
          "type": "boolean"
        },
        "derive": {
          "$ref": "#/definitions/Derivation"
        },
        "expectPubId": {
          "title": "Expected publication ID of this collection within the control plane.",
          "description": "When present, a publication of the collection will fail if the last publication ID in the control plane doesn't match this value.",
          "$ref": "#/definitions/Id"
        },
        "journals": {
          "title": "Template for journals of this collection.",
          "$ref": "#/definitions/JournalTemplate"
        },
        "key": {
          "title": "Composite key of this collection.",
          "$ref": "#/definitions/CompositeKey"
        },
        "projections": {
          "title": "Projections and logical partitions of this collection.",
          "examples": [
            {
              "a_field": "/json/ptr",
              "a_partition": {
                "location": "/json/ptr",
                "partition": true
              }
            }
          ],
          "type": "object",
          "patternProperties": {
            "^([^/~]|(~[01]))+(/([^/~]|(~[01]))+)*$": {
              "$ref": "#/definitions/Projection"
            }
          },
          "additionalProperties": false
        },
        "readSchema": {
          "title": "Schema against which collection documents are validated and reduced on read.",
          "$ref": "#/definitions/Schema"
        },
        "schema": {
          "title": "Schema against which collection documents are validated and reduced on write and read.",
          "examples": [
            "../path/to/schema#/$defs/subPath"
          ],
          "$ref": "#/definitions/Schema"
        },
        "writeSchema": {
          "title": "Schema against which collection documents are validated and reduced on write.",
          "$ref": "#/definitions/Schema"
        }
      },
      "additionalProperties": false
    },
    "CompositeKey": {
      "description": "Ordered JSON-Pointers which define how a composite key may be extracted from a collection document.",
      "examples": [
        [
          "/json/ptr"
        ]
      ],
      "type": "array",
      "items": {
        "$ref": "#/definitions/JsonPointer"
      }
    },
    "CompressionCodec": {
      "description": "A CompressionCodec may be applied to compress journal fragments before they're persisted to cloud stoage. The compression applied to a journal fragment is included in its filename, such as \".gz\" for GZIP. A collection's compression may be changed at any time, and will affect newly-written journal fragments.",
      "examples": [
        "GZIP_OFFLOAD_DECOMPRESSION"
      ],
      "type": "string",
      "enum": [
        "NONE",
        "GZIP",
        "ZSTANDARD",
        "SNAPPY",
        "GZIP_OFFLOAD_DECOMPRESSION"
      ]
    },
    "ConnectorConfig": {
      "description": "Connector image and configuration specification.",
      "type": "object",
      "required": [
        "config",
        "image"
      ],
      "properties": {
        "config": {
          "title": "Configuration of the connector."
        },
        "image": {
          "title": "Image of the connector.",
          "type": "string"
        }
      }
    },
    "DekafConfig": {
      "description": "Dekaf service configuration",
      "type": "object",
      "required": [
        "config",
        "variant"
      ],
      "properties": {
        "config": {
          "title": "Dekaf endpoint config."
        },
        "variant": {
          "title": "Dekaf variant type",
          "type": "string"
        }
      }
    },
    "Derivation": {
      "description": "Derive specifies how a collection is derived from other collections.",
      "type": "object",
      "required": [
        "transforms",
        "using"
      ],
      "properties": {
        "shards": {
          "title": "Template for shards of this derivation task.",
          "$ref": "#/definitions/ShardTemplate"
        },
        "shuffleKeyTypes": {
          "title": "Key component types of the shuffle keys used by derivation lambdas.",
          "description": "Typically you omit this and Flow infers it from your transform shuffle keys. In some circumstances, Flow may require that you explicitly tell it of your shuffled key types.",
          "type": "array",
          "items": {
            "$ref": "#/definitions/ShuffleType"
          }
        },
        "transforms": {
          "title": "Transforms which make up this derivation.",
          "type": "array",
          "items": {
            "$ref": "#/definitions/TransformDef"
          }
        },
        "using": {
          "title": "The selected runtime for this derivation.",
          "$ref": "#/definitions/DeriveUsing"
        }
      },
      "additionalProperties": false
    },
    "DeriveUsing": {
      "description": "A derivation runtime implementation.",
      "oneOf": [
        {
          "title": "A Connector.",
          "type": "object",
          "required": [
            "connector"
          ],
          "properties": {
            "connector": {
              "$ref": "#/definitions/ConnectorConfig"
            }
          },
          "additionalProperties": false
        },
        {
          "title": "A SQLite derivation.",
          "type": "object",
          "required": [
            "sqlite"
          ],
          "properties": {
            "sqlite": {
              "$ref": "#/definitions/DeriveUsingSqlite"
            }
          },
          "additionalProperties": false
        },
        {
          "title": "A TypeScript derivation.",
          "type": "object",
          "required": [
            "typescript"
          ],
          "properties": {
            "typescript": {
              "$ref": "#/definitions/DeriveUsingTypescript"
            }
          },
          "additionalProperties": false
        },
        {
          "title": "A local command (development only).",
          "type": "object",
          "required": [
            "local"
          ],
          "properties": {
            "local": {
              "$ref": "#/definitions/LocalConfig"
            }
          },
          "additionalProperties": false
        }
      ]
    },
    "DeriveUsingSqlite": {
      "type": "object",
      "properties": {
        "migrations": {
          "title": "Ordered migrations which are used to initialize the database.",
          "description": "Migrations may be provided as an inline string, or as a relative URL to a file containing the migration SQL.",
          "type": "array",
          "items": {
            "oneOf": [
              {
                "description": "A URL identifying a resource, which may be a relative local path with respect to the current resource (i.e, ../path/to/flow.yaml), or may be an external absolute URL (i.e., http://example/flow.yaml).",
                "examples": [
                  "https://example/resource"
                ],
                "type": "string",
                "pattern": "^[^ ]+$"
              },
              {
                "type": "string",
                "contentMediaType": "text/x.sql"
              }
            ]
          }
        }
      },
      "additionalProperties": false
    },
    "DeriveUsingTypescript": {
      "type": "object",
      "required": [
        "module"
      ],
      "properties": {
        "module": {
          "title": "TypeScript module implementing this derivation.",
          "description": "Module is either a relative URL of a TypeScript module file, or is an inline representation of a Typescript module. The module must have a exported Derivation variable which is an instance implementing the corresponding Derivation interface.",
          "oneOf": [
            {
              "description": "A URL identifying a resource, which may be a relative local path with respect to the current resource (i.e, ../path/to/flow.yaml), or may be an external absolute URL (i.e., http://example/flow.yaml).",
              "examples": [
                "https://example/resource"
              ],
              "type": "string",
              "pattern": "^[^ ]+$"
            },
            {
              "type": "string",
              "contentMediaType": "text/x.typescript"
            }
          ]
        }
      },
      "additionalProperties": false
    },
    "Field": {
      "description": "Field names a projection of a document location. They may include '/', but cannot begin or end with one. Many Fields are automatically inferred by Flow from a collection JSON Schema, and are the JSON Pointer of the document location with the leading '/' removed. User-provided Fields which act as a logical partitions are restricted to Unicode letters, numbers, '-', '_', or '.'",
      "examples": [
        "my_field"
      ],
      "type": "string",
      "pattern": "^([^/~]|(~[01]))+(/([^/~]|(~[01]))+)*$"
    },
    "FragmentTemplate": {
      "description": "A FragmentTemplate configures how journal fragment files are produced as part of a collection.",
      "examples": [
        {
          "compressionCodec": "ZSTANDARD",
          "flushInterval": "1h"
        }
      ],
      "type": "object",
      "properties": {
        "compressionCodec": {
          "title": "Codec used to compress Journal Fragments.",
          "$ref": "#/definitions/CompressionCodec"
        },
        "flushInterval": {
          "title": "Maximum flush delay before in-progress fragments are closed and persisted",
          "description": "into cloud storage. Intervals are converted into uniform time segments: 24h will \"roll\" all fragments at midnight UTC every day, 1h at the top of every hour, 15m a :00, :15, :30, :45 past the hour, and so on. If not set, then fragments are not flushed on time-based intervals.",
          "type": [
            "string",
            "null"
          ],
          "pattern": "^\\d+(s|m|h)$"
        },
        "length": {
          "title": "Desired content length of each fragment, in megabytes before compression.",
          "description": "When a collection journal fragment reaches this threshold, it will be closed off and pushed to cloud storage. If not set, a default of 512MB is used.",
          "type": "integer",
          "format": "uint32",
          "maximum": 4096.0,
          "minimum": 32.0
        },
        "retention": {
          "title": "Duration for which historical fragments of a collection should be kept.",
          "description": "If not set, then fragments are retained indefinitely.",
          "type": [
            "string",
            "null"
          ],
          "pattern": "^\\d+(s|m|h)$"
        }
      },
      "additionalProperties": false
    },
    "FullSource": {
      "description": "A source collection and details of how it's read.",
      "examples": [
        {
          "name": "source/collection"
        }
      ],
      "type": "object",
      "required": [
        "name"
      ],
      "properties": {
        "name": {
          "title": "Name of the collection to be read.",
          "$ref": "#/definitions/Collection"
        },
        "notAfter": {
          "title": "Upper bound date-time for documents which should be processed.",
          "description": "Source collection documents published after this date-time are filtered. `notAfter` is *only* a filter. Updating its value will not cause Flow to re-process documents that have already been read. Optional. Default is to process all documents.",
          "type": [
            "string",
            "null"
          ],
          "format": "date-time"
        },
        "notBefore": {
          "title": "Lower bound date-time for documents which should be processed.",
          "description": "Source collection documents published before this date-time are filtered. `notBefore` is *only* a filter. Updating its value will not cause Flow to re-process documents that have already been read. Optional. Default is to process all documents.",
          "type": [
            "string",
            "null"
          ],
          "format": "date-time"
        },
        "partitions": {
          "title": "Selector over partition of the source collection to read.",
          "examples": [
            {
              "exclude": {
                "other_partition": [
                  32,
                  64
                ]
              },
              "include": {
                "a_partition": [
                  "A",
                  "B"
                ]
              }
            }
          ],
          "$ref": "#/definitions/PartitionSelector"
        }
      },
      "additionalProperties": false
    },
    "Id": {
      "type": "string"
    },
    "JournalTemplate": {
      "description": "A JournalTemplate configures the journals which make up the physical partitions of a collection.",
      "examples": [
        {
          "fragments": {
            "compressionCodec": "ZSTANDARD",
            "flushInterval": "1h"
          }
        }
      ],
      "type": "object",
      "required": [
        "fragments"
      ],
      "properties": {
        "fragments": {
          "title": "Fragment configuration of collection journals.",
          "$ref": "#/definitions/FragmentTemplate"
        }
      },
      "additionalProperties": false
    },
    "JsonPointer": {
      "description": "JSON Pointer which identifies a location in a document.",
      "examples": [
        "/json/ptr"
      ],
      "type": "string",
      "pattern": "^(/([^/~]|(~[01]))+)*$"
    },
    "LocalConfig": {
      "description": "Local command and its configuration.",
      "type": "object",
      "required": [
        "command",
        "config"
      ],
      "properties": {
        "command": {
          "title": "Command to execute",
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "config": {
          "title": "Configuration of the command."
        },
        "env": {
          "title": "Environment variables",
          "type": "object",
          "additionalProperties": {
            "type": "string"
          }
        },
        "protobuf": {
          "title": "Use protobuf codec instead of JSON.",
          "type": "boolean"
        }
      }
    },
    "Materialization": {
      "description": "Materialization names are paths of Unicode letters, numbers, '-', '_', or '.'. Each path component is separated by a slash '/', and a name may not begin or end in a '/'.",
      "examples": [
        "acmeCo/materialization"
      ],
      "type": "string",
      "pattern": "^[\\p{Letter}\\p{Number}\\-_\\.]+(/[\\p{Letter}\\p{Number}\\-_\\.]+)*$"
    },
    "MaterializationBinding": {
      "examples": [
        {
          "fields": {
            "recommended": true
          },
          "resource": {
            "table": "a_table"
          },
          "source": "source/collection"
        }
      ],
      "type": "object",
      "required": [
        "resource",
        "source"
      ],
      "properties": {
        "backfill": {
          "title": "Backfill counter for this binding.",
          "description": "Every increment of this counter will result in a new backfill of this binding from its source collection to its materialized resource. For example when materializing to a SQL table, incrementing this counter causes the table to be dropped and then rebuilt by re-reading the source collection.",
          "type": "integer",
          "format": "uint32",
          "minimum": 0.0
        },
        "disable": {
          "title": "Whether to disable the binding",
          "description": "Disabled bindings are inactive, and not validated.",
          "type": "boolean"
        },
        "fields": {
          "title": "Selected projections for this materialization.",
          "default": {
            "recommended": true
          },
          "$ref": "#/definitions/MaterializationFields"
        },
        "onIncompatibleSchemaChange": {
          "title": "Action to take when a schema change is rejected due to incompatibility.",
          "description": "This setting is used to determine the action to take when a schema change is rejected due to incompatibility with the target resource. By default, the binding will have its `backfill` counter incremented, causing it to be re-materialized from the source collection.",
          "$ref": "#/definitions/OnIncompatibleSchemaChange"
        },
        "priority": {
          "title": "Priority applied to documents processed by this binding.",
          "description": "When all bindings are of equal priority, Flow processes documents according to their associated publishing time, as encoded in the document UUID.\n\nHowever, when one binding has a higher priority than others, then *all* ready documents are processed through the binding before *any* documents of other bindings are processed.",
          "type": "integer",
          "format": "uint32",
          "minimum": 0.0
        },
        "resource": {
          "title": "Endpoint resource to materialize into."
        },
        "source": {
          "title": "The collection to be materialized.",
          "$ref": "#/definitions/Source"
        }
      },
      "additionalProperties": false
    },
    "MaterializationDef": {
      "description": "A Materialization binds a Flow collection with an external system & target (e.x, a SQL table) into which the collection is to be continuously materialized.",
      "type": "object",
      "required": [
        "bindings",
        "endpoint"
      ],
      "properties": {
        "bindings": {
          "title": "Bound collections to materialize into the endpoint.",
          "type": "array",
          "items": {
            "$ref": "#/definitions/MaterializationBinding"
          }
        },
        "delete": {
          "title": "Delete this materialization within the control plane.",
          "description": "When true, a publication will delete this materialization.",
          "type": "boolean"
        },
        "endpoint": {
          "title": "Endpoint to materialize into.",
          "$ref": "#/definitions/MaterializationEndpoint"
        },
        "expectPubId": {
          "title": "Expected publication ID of this materialization within the control plane.",
          "description": "When present, a publication of the materialization will fail if the last publication ID in the control plane doesn't match this value.",
          "$ref": "#/definitions/Id"
        },
        "onIncompatibleSchemaChange": {
          "title": "Default handling of schema changes that are incompatible with the target resource.",
          "description": "This can be overridden on a per-binding basis.",
          "$ref": "#/definitions/OnIncompatibleSchemaChange"
        },
        "shards": {
          "title": "Template for shards of this materialization task.",
          "$ref": "#/definitions/ShardTemplate"
        },
        "sourceCapture": {
          "title": "Automatically materialize new bindings from a named capture",
          "$ref": "#/definitions/SourceCapture"
        }
      },
      "additionalProperties": false
    },
    "MaterializationEndpoint": {
      "description": "An Endpoint connector used for Flow materializations.",
      "oneOf": [
        {
          "title": "A Connector.",
          "type": "object",
          "required": [
            "connector"
          ],
          "properties": {
            "connector": {
              "$ref": "#/definitions/ConnectorConfig"
            }
          },
          "additionalProperties": false
        },
        {
          "title": "A local command (development only).",
          "type": "object",
          "required": [
            "local"
          ],
          "properties": {
            "local": {
              "$ref": "#/definitions/LocalConfig"
            }
          },
          "additionalProperties": false
        },
        {
          "title": "A Dekaf connection",
          "type": "object",
          "required": [
            "dekaf"
          ],
          "properties": {
            "dekaf": {
              "$ref": "#/definitions/DekafConfig"
            }
          },
          "additionalProperties": false
        }
      ]
    },
    "MaterializationFields": {
      "description": "MaterializationFields defines a selection of projections to materialize, as well as optional per-projection, driver-specific configuration.",
      "examples": [
        {
          "exclude": [
            "removed"
          ],
          "include": {
            "added": {}
          },
          "recommended": true
        }
      ],
      "type": "object",
      "required": [
        "recommended"
      ],
      "properties": {
        "exclude": {
          "title": "Fields to exclude.",
          "description": "This removes from recommended projections, where enabled.",
          "type": "array",
          "items": {
            "$ref": "#/definitions/Field"
          }
        },
        "include": {
          "title": "Fields to include.",
          "description": "This supplements any recommended fields, where enabled. Values are passed through to the driver, e.x. for customization of the driver's schema generation or runtime behavior with respect to the field.",
          "type": "object",
          "additionalProperties": true
        },
        "recommended": {
          "title": "Should recommended projections for the endpoint be used?",
          "type": "boolean"
        }
      },
      "additionalProperties": false
    },
    "OnIncompatibleSchemaChange": {
      "description": "Determines how to handle incompatible schema changes for a given binding.",
      "examples": [
        "backfill"
      ],
      "oneOf": [
        {
          "description": "Fail the publication of the incompatible schema change. This prevents any schema change from being applied if it is incompatible with the existing schema, as determined by the connector.",
          "type": "string",
          "enum": [
            "abort"
          ]
        },
        {
          "description": "Increment the backfill counter of the binding, causing it to start over from the beginning.",
          "type": "string",
          "enum": [
            "backfill"
          ]
        },
        {
          "description": "Disable the binding, which will be effectively excluded from the task until it is re-enabled.",
          "type": "string",
          "enum": [
            "disableBinding"
          ]
        },
        {
          "description": "Disable the entire task, preventing it from running until it is re-enabled.",
          "type": "string",
          "enum": [
            "disableTask"
          ]
        }
      ]
    },
    "PartitionSelector": {
      "description": "Partition selectors identify a desired subset of the available logical partitions of a collection.",
      "examples": [
        {
          "exclude": {
            "other_partition": [
              32,
              64
            ]
          },
          "include": {
            "a_partition": [
              "A",
              "B"
            ]
          }
        }
      ],
      "type": "object",
      "properties": {
        "exclude": {
          "description": "Partition field names and values which are excluded from the source collection. Any documents matching *any one* of the partition values will be excluded.",
          "default": {},
          "type": "object",
          "additionalProperties": {
            "type": "array",
            "items": true
          }
        },
        "include": {
          "description": "Partition field names and corresponding values which must be matched from the Source collection. Only documents having one of the specified values across all specified partition names will be matched. For example, source: [App, Web] region: [APAC] would mean only documents of 'App' or 'Web' source and also occurring in the 'APAC' region will be processed.",
          "default": {},
          "type": "object",
          "additionalProperties": {
            "type": "array",
            "items": true
          }
        }
      },
      "additionalProperties": false
    },
    "Projection": {
      "description": "Projections are named locations within a collection document which may be used for logical partitioning or directly exposed to databases into which collections are materialized.",
      "anyOf": [
        {
          "$ref": "#/definitions/JsonPointer"
        },
        {
          "type": "object",
          "required": [
            "location"
          ],
          "properties": {
            "location": {
              "title": "Location of this projection.",
              "$ref": "#/definitions/JsonPointer"
            },
            "partition": {
              "title": "Is this projection a logical partition?",
              "default": false,
              "type": "boolean"
            }
          },
          "additionalProperties": false
        }
      ]
    },
    "RelativeUrl": {
      "description": "A URL identifying a resource, which may be a relative local path with respect to the current resource (i.e, ../path/to/flow.yaml), or may be an external absolute URL (i.e., http://example/flow.yaml).",
      "examples": [
        "https://example/resource"
      ],
      "type": "string",
      "pattern": "^[^ ]+$"
    },
    "Schema": {
      "description": "A schema is a draft 2020-12 JSON Schema which validates Flow documents. Schemas also provide annotations at document locations, such as reduction strategies for combining one document into another.\n\nSchemas may be defined inline to the catalog, or given as a relative or absolute URI. URIs may optionally include a JSON fragment pointer that locates a specific sub-schema therein.\n\nFor example, \"schemas/marketing.yaml#/$defs/campaign\" would reference the schema at location {\"$defs\": {\"campaign\": ...}} within ./schemas/marketing.yaml.",
      "examples": [
        "http://example/schema#/$defs/subPath",
        "../path/to/schema#/$defs/subPath",
        {
          "properties": {
            "bar": {
              "const": 42
            },
            "foo": {
              "type": "integer"
            }
          },
          "type": "object"
        }
      ]
    },
    "ShardTemplate": {
      "description": "A ShardTemplate configures how shards process a catalog task.",
      "examples": [
        {
          "hotStandbys": 1,
          "maxTxnDuration": "30s"
        }
      ],
      "type": "object",
      "properties": {
        "disable": {
          "title": "Disable processing of the task's shards.",
          "type": "boolean"
        },
        "hotStandbys": {
          "title": "Number of hot standbys to keep for each task shard.",
          "description": "Hot standbys of a shard actively replicate the shard's state to another machine, and are able to be quickly promoted to take over processing for the shard should its current primary fail. If not set, then no hot standbys are maintained. EXPERIMENTAL: this field MAY be removed.",
          "type": "integer",
          "format": "uint32",
          "minimum": 0.0
        },
        "logLevel": {
          "title": "Log level of this tasks's shards.",
          "description": "Log levels may currently be \"error\", \"warn\", \"info\", \"debug\", or \"trace\". If not set, the effective log level is \"info\".",
          "type": "string"
        },
        "maxTxnDuration": {
          "title": "Maximum duration of task transactions.",
          "description": "This duration upper-bounds the amount of time during which a transaction may process documents before it must flush and commit. It may run for less time if there aren't additional ready documents for it to process. If not set, the maximum duration defaults to five minutes for materializations, and one second for captures and derivations. EXPERIMENTAL: this field MAY be removed.",
          "type": [
            "string",
            "null"
          ],
          "pattern": "^\\d+(s|m|h)$"
        },
        "minTxnDuration": {
          "title": "Minimum duration of task transactions.",
          "description": "This duration lower-bounds the amount of time during which a transaction must process documents before it must flush and commit. It may run for more time if additional documents are available. The default value is zero seconds. Larger values may result in more data reduction, at the cost of more latency. EXPERIMENTAL: this field MAY be removed.",
          "type": [
            "string",
            "null"
          ],
          "pattern": "^\\d+(s|m|h)$"
        },
        "readChannelSize": {
          "title": "Size of the reader channel used for decoded documents.",
          "description": "Larger values are recommended for tasks having more than one shard split and long, bursty transaction durations. If not set, a reasonable default (currently 4,096) is used. EXPERIMENTAL: this field is LIKELY to be removed.",
          "type": "integer",
          "format": "uint32",
          "minimum": 0.0
        },
        "ringBufferSize": {
          "title": "Size of the ring buffer used to sequence documents for exactly-once semantics.",
          "description": "The ring buffer is a performance optimization only: catalog tasks will replay portions of journals as needed when messages aren't available in the buffer. It can remain small if upstream task transactions are small, but larger transactions will achieve better performance with a larger ring. If not set, a reasonable default (currently 65,536) is used. EXPERIMENTAL: this field is LIKELY to be removed.",
          "type": "integer",
          "format": "uint32",
          "minimum": 0.0
        }
      },
      "additionalProperties": false
    },
    "Shuffle": {
      "description": "A Shuffle specifies how a shuffling key is to be extracted from collection documents.",
      "examples": [
        {
          "key": [
            "/json/ptr"
          ]
        }
      ],
      "oneOf": [
        {
          "title": "A Document may be shuffled to any task shard.",
          "description": "Use 'any' if your transformation does not rely on internal task state, or if your derivation is not intended to scale beyond a single shard.",
          "type": "string",
          "enum": [
            "any"
          ]
        },
        {
          "title": "Key which identifies fields of sourced documents to extract and shuffle upon.",
          "type": "object",
          "required": [
            "key"
          ],
          "properties": {
            "key": {
              "$ref": "#/definitions/CompositeKey"
            }
          },
          "additionalProperties": false
        },
        {
          "title": "Lambda which extracts a shuffle key from the sourced documents of this transform.",
          "description": "Lambdas may be provided inline, or as a relative URL to a file containing the lambda.",
          "type": "object",
          "required": [
            "lambda"
          ],
          "properties": {
            "lambda": true
          },
          "additionalProperties": false
        }
      ]
    },
    "ShuffleType": {
      "description": "Type of a shuffled key component.",
      "type": "string",
      "enum": [
        "boolean",
        "integer",
        "string"
      ]
    },
    "Source": {
      "description": "A source collection and details of how it's read.",
      "examples": [
        "source/collection"
      ],
      "anyOf": [
        {
          "$ref": "#/definitions/FullSource"
        },
        {
          "$ref": "#/definitions/Collection"
        }
      ]
    },
    "SourceCapture": {
      "anyOf": [
        {
          "$ref": "#/definitions/Capture"
        },
        {
          "$ref": "#/definitions/SourceCaptureDef"
        }
      ]
    },
    "SourceCaptureDef": {
      "description": "SourceCaptureDef specifies configuration for source captures",
      "type": "object",
      "required": [
        "capture"
      ],
      "properties": {
        "capture": {
          "description": "Capture name",
          "$ref": "#/definitions/Capture"
        },
        "deltaUpdates": {
          "description": "When adding new bindings from a source capture to a materialization, should the new bindings be marked as delta updates",
          "type": "boolean"
        },
        "targetSchema": {
          "description": "When adding new bindings from a source capture to a materialization, how should the schema of the materialization binding be set",
          "$ref": "#/definitions/SourceCaptureSchemaMode"
        }
      },
      "additionalProperties": false
    },
    "SourceCaptureSchemaMode": {
      "oneOf": [
        {
          "description": "Leave the materialization binding's schema field empty, therefore falling back to the default schema of the materialization",
          "type": "string",
          "enum": [
            "leaveEmpty"
          ]
        },
        {
          "description": "Use the 2nd-to-last component of the collection name as the schema of the materialization binding",
          "type": "string",
          "enum": [
            "fromSourceName"
          ]
        }
      ]
    },
    "Test": {
      "description": "Test names are paths of Unicode letters, numbers, '-', '_', or '.'. Each path component is separated by a slash '/', and a name may not begin or end in a '/'.",
      "examples": [
        "acmeCo/conversions/test"
      ],
      "type": "string",
      "pattern": "^[\\p{Letter}\\p{Number}\\-_\\.]+(/[\\p{Letter}\\p{Number}\\-_\\.]+)*$"
    },
    "TestDef": {
      "description": "Test the behavior of reductions and derivations, through a sequence of test steps.",
      "examples": [
        {
          "description": "An example test",
          "steps": [
            {
              "ingest": {
                "collection": "acmeCo/collection",
                "description": "Description of the ingestion.",
                "documents": [
                  {
                    "a": "document"
                  },
                  {
                    "another": "document"
                  }
                ]
              }
            },
            {
              "verify": {
                "collection": "acmeCo/collection",
                "description": "Description of the verification.",
                "documents": [
                  {
                    "a": "document"
                  },
                  {
                    "another": "document"
                  }
                ]
              }
            }
          ]
        }
      ],
      "type": "object",
      "required": [
        "steps"
      ],
      "properties": {
        "delete": {
          "title": "Delete this test within the control plane.",
          "description": "When true, a publication will delete this test.",
          "type": "boolean"
        },
        "description": {
          "title": "Description of this test.",
          "type": "string"
        },
        "expectPubId": {
          "title": "Expected publication ID of this test within the control plane.",
          "description": "When present, a publication of the test will fail if the last publication ID in the control plane doesn't match this value.",
          "$ref": "#/definitions/Id"
        },
        "steps": {
          "title": "Sequential steps of this test.",
          "type": "array",
          "items": {
            "$ref": "#/definitions/TestStep"
          }
        }
      },
      "additionalProperties": false
    },
    "TestDocuments": {
      "description": "A test step describes either an \"ingest\" of document fixtures into a collection, or a \"verify\" of expected document fixtures from a collection.",
      "examples": [
        "../path/to/test-documents.json",
        [
          {
            "a": "document"
          },
          {
            "another": "document"
          }
        ]
      ]
    },
    "TestStep": {
      "description": "A test step describes either an \"ingest\" of document fixtures into a collection, or a \"verify\" of expected document fixtures from a collection.",
      "examples": [
        {
          "ingest": {
            "collection": "acmeCo/collection",
            "description": "Description of the ingestion.",
            "documents": [
              {
                "a": "document"
              },
              {
                "another": "document"
              }
            ]
          }
        },
        {
          "verify": {
            "collection": "acmeCo/collection",
            "description": "Description of the verification.",
            "documents": [
              {
                "a": "document"
              },
              {
                "another": "document"
              }
            ]
          }
        }
      ],
      "oneOf": [
        {
          "description": "Ingest document fixtures into a collection.",
          "type": "object",
          "required": [
            "ingest"
          ],
          "properties": {
            "ingest": {
              "$ref": "#/definitions/TestStepIngest"
            }
          },
          "additionalProperties": false
        },
        {
          "description": "Verify the contents of a collection match a set of document fixtures.",
          "type": "object",
          "required": [
            "verify"
          ],
          "properties": {
            "verify": {
              "$ref": "#/definitions/TestStepVerify"
            }
          },
          "additionalProperties": false
        }
      ]
    },
    "TestStepIngest": {
      "description": "An ingestion test step ingests document fixtures into the named collection.",
      "examples": [
        {
          "collection": "acmeCo/collection",
          "description": "Description of the ingestion.",
          "documents": [
            {
              "a": "document"
            },
            {
              "another": "document"
            }
          ]
        }
      ],
      "type": "object",
      "required": [
        "collection",
        "documents"
      ],
      "properties": {
        "collection": {
          "title": "Name of the collection into which the test will ingest.",
          "$ref": "#/definitions/Collection"
        },
        "description": {
          "title": "Description of this test ingestion.",
          "type": "string"
        },
        "documents": {
          "title": "Documents to ingest.",
          "description": "Each document must conform to the collection's schema.",
          "$ref": "#/definitions/TestDocuments"
        }
      },
      "additionalProperties": false
    },
    "TestStepVerify": {
      "description": "A verification test step verifies that the contents of the named collection match the expected fixtures, after fully processing all preceding ingestion test steps.",
      "examples": [
        {
          "collection": "acmeCo/collection",
          "description": "Description of the verification.",
          "documents": [
            {
              "a": "document"
            },
            {
              "another": "document"
            }
          ]
        }
      ],
      "type": "object",
      "required": [
        "collection",
        "documents"
      ],
      "properties": {
        "collection": {
          "title": "Collection from which the test will verify.",
          "$ref": "#/definitions/Source"
        },
        "description": {
          "title": "Description of this test verification.",
          "type": "string"
        },
        "documents": {
          "title": "Documents to verify.",
          "description": "Each document may contain only a portion of the matched document's properties, and any properties present in the actual document but not in this document fixture are ignored. All other values must match or the test will fail.",
          "$ref": "#/definitions/TestDocuments"
        }
      },
      "additionalProperties": false
    },
    "Transform": {
      "description": "Transform names are Unicode letters, numbers, '-', '_', or '.'.",
      "examples": [
        "myTransform"
      ],
      "type": "string",
      "pattern": "^[\\p{Letter}\\p{Number}\\-_\\.]+$"
    },
    "TransformDef": {
      "description": "A Transform reads and shuffles documents of a source collection, and processes each document through either one or both of a register \"update\" lambda and a derived document \"publish\" lambda.",
      "examples": [
        {
          "name": "my-transform",
          "shuffle": "any",
          "source": "some/source/collection"
        }
      ],
      "type": "object",
      "required": [
        "name",
        "shuffle",
        "source"
      ],
      "properties": {
        "backfill": {
          "title": "Backfill counter for this transform.",
          "description": "Every increment of this counter will result in a new backfill of this transform. Specifically, the transform's lambda will be re-invoked for every applicable document of its source collection.\n\nNote that a backfill does *not* truncate the derived collection, and documents published by a backfilled transform will coexist with (and be ordered after) any documents which were published as part of a preceding backfill.",
          "type": "integer",
          "format": "uint32",
          "minimum": 0.0
        },
        "disable": {
          "title": "Whether to disable this transform.",
          "description": "Disabled transforms are completely ignored at runtime and are not validated.",
          "type": "boolean"
        },
        "lambda": {
          "title": "Lambda applied to the sourced documents of this transform.",
          "description": "Lambdas may be provided inline, or as a relative URL to a file containing the lambda."
        },
        "name": {
          "title": "Name of this transformation.",
          "description": "The names of transforms within a derivation must be unique and stable.",
          "$ref": "#/definitions/Transform"
        },
        "priority": {
          "title": "Priority applied to documents processed by this transform.",
          "description": "When all transforms are of equal priority, Flow processes documents according to their associated publishing time, as encoded in the document UUID.\n\nHowever, when one transform has a higher priority than others, then *all* ready documents are processed through the transform before *any* documents of other transforms are processed.",
          "type": "integer",
          "format": "uint32",
          "minimum": 0.0
        },
        "readDelay": {
          "title": "Delay applied to documents processed by this transform.",
          "description": "Delays are applied as an adjustment to the UUID clock encoded within each document, which is then used to impose a relative ordering of all documents read by this derivation. This means that read delays are applied in a consistent way, even when back-filling over historical documents. When caught up and tailing the source collection, delays also \"gate\" documents such that they aren't processed until the current wall-time reflects the delay.",
          "type": [
            "string",
            "null"
          ],
          "pattern": "^\\d+(s|m|h)$"
        },
        "shuffle": {
          "title": "Shuffle by which source documents are mapped to processing shards.",
          "$ref": "#/definitions/Shuffle"
        },
        "source": {
          "title": "Source collection read by this transform.",
          "$ref": "#/definitions/Source"
        }
      },
      "additionalProperties": false
    }
  }
}