chore: confluent API spec evolution (#925)

* chore: allow definition of two new fields when schema is registered to keep compatibility with confluent clients

* test: validate_schema_request_body

* chore: lint

* chore: lint

* feat: define proper types for SchemaMetadata & SchemaRuleSet to facilitate future impl

karapace/schema_registry_apis.py

@@ -991,7 +991,7 @@ def _validate_schema_request_body(self, content_type: str, body: dict | Any) ->
                 status=HTTPStatus.BAD_REQUEST,
             )
         for field in body:
-            if field not in {"schema", "schemaType", "references"}:
+            if field not in {"schema", "schemaType", "references", "metadata", "ruleSet"}:
                 self.r(
                     body={
                         "error_code": SchemaErrorCodes.HTTP_UNPROCESSABLE_ENTITY.value,

karapace/typing.py

@@ -6,7 +6,7 @@
 
 from enum import Enum, unique
 from karapace.errors import InvalidVersion
-from typing import ClassVar, Dict, List, Mapping, NewType, Sequence, Union
+from typing import Any, ClassVar, Dict, List, Mapping, NewType, Sequence, Union
 from typing_extensions import TypeAlias
 
 import functools
@@ -23,6 +23,8 @@
 
 Subject = NewType("Subject", str)
 VersionTag = Union[str, int]
+SchemaMetadata = NewType("SchemaMetadata", Dict[str, Any])
+SchemaRuleSet = NewType("SchemaRuleSet", Dict[str, Any])
 
 # note: the SchemaID is a unique id among all the schemas (and each version should be assigned to a different id)
 # basically the same SchemaID refer always to the same TypedSchema.

tests/integration/schema_registry/test_jsonschema.py

@@ -6,6 +6,7 @@
 from karapace.client import Client
 from karapace.compatibility import CompatibilityModes
 from karapace.schema_reader import SchemaType
+from karapace.typing import SchemaMetadata, SchemaRuleSet
 from tests.schemas.json_schemas import (
     A_DINT_B_DINT_OBJECT_SCHEMA,
     A_DINT_B_INT_OBJECT_SCHEMA,
@@ -234,8 +235,14 @@ async def not_schemas_are_backward_compatible(
 
 @pytest.mark.parametrize("trail", ["", "/"])
 @pytest.mark.parametrize("compatibility", [CompatibilityModes.FORWARD, CompatibilityModes.BACKWARD, CompatibilityModes.FULL])
+@pytest.mark.parametrize("metadata", [None, {}])
+@pytest.mark.parametrize("rule_set", [None, {}])
 async def test_same_jsonschema_must_have_same_id(
-    registry_async_client: Client, compatibility: CompatibilityModes, trail: str
+    registry_async_client: Client,
+    compatibility: CompatibilityModes,
+    trail: str,
+    metadata: SchemaMetadata,
+    rule_set: SchemaRuleSet,
 ) -> None:
     for schema in ALL_SCHEMAS:
         subject = new_random_name("subject")
@@ -248,6 +255,8 @@ async def test_same_jsonschema_must_have_same_id(
             json={
                 "schema": json.dumps(schema.schema),
                 "schemaType": SchemaType.JSONSCHEMA.value,
+                "metadata": metadata,
+                "ruleSet": rule_set,
             },
         )
         assert first_res.status_code == 200
@@ -259,6 +268,8 @@ async def test_same_jsonschema_must_have_same_id(
             json={
                 "schema": json.dumps(schema.schema),
                 "schemaType": SchemaType.JSONSCHEMA.value,
+                "metadata": metadata,
+                "ruleSet": rule_set,
             },
         )
         assert second_res.status_code == 200

tests/integration/test_schema_protobuf.py

@@ -9,7 +9,7 @@
 from karapace.errors import InvalidTest
 from karapace.protobuf.kotlin_wrapper import trim_margin
 from karapace.schema_type import SchemaType
-from karapace.typing import JsonData
+from karapace.typing import JsonData, SchemaMetadata, SchemaRuleSet
 from tests.base_testcase import BaseTestCase
 from tests.utils import create_subject_name_factory
 from typing import List, Optional, Union
@@ -963,11 +963,20 @@ class ReferenceTestCase(BaseTestCase):
     ],
     ids=str,
 )
-async def test_references(testcase: ReferenceTestCase, registry_async_client: Client):
+@pytest.mark.parametrize("metadata", [None, {}])
+@pytest.mark.parametrize("rule_set", [None, {}])
+async def test_references(
+    testcase: ReferenceTestCase, registry_async_client: Client, metadata: SchemaMetadata, rule_set: SchemaRuleSet
+):
     for testdata in testcase.schemas:
         if isinstance(testdata, TestCaseSchema):
             print(f"Adding new schema, subject: '{testdata.subject}'\n{testdata.schema_str}")
-            body = {"schemaType": testdata.schema_type, "schema": testdata.schema_str}
+            body = {
+                "schemaType": testdata.schema_type,
+                "schema": testdata.schema_str,
+                "metadata": metadata,
+                "ruleSet": rule_set,
+            }
             if testdata.references:
                 body["references"] = testdata.references
             res = await registry_async_client.post(f"subjects/{testdata.subject}/versions", json=body)

tests/unit/test_schema_registry_api.py

@@ -11,6 +11,23 @@
 from unittest.mock import ANY, AsyncMock, Mock, patch, PropertyMock
 
 import asyncio
+import pytest
+
+
+async def test_validate_schema_request_body():
+    controller = KarapaceSchemaRegistryController(config=set_config_defaults(DEFAULTS))
+
+    controller._validate_schema_request_body(  # pylint: disable=W0212
+        "application/json", {"schema": "{}", "schemaType": "JSON", "references": [], "metadata": {}, "ruleSet": {}}
+    )
+
+    with pytest.raises(HTTPResponse) as exc_info:
+        controller._validate_schema_request_body(  # pylint: disable=W0212
+            "application/json",
+            {"schema": "{}", "schemaType": "JSON", "references": [], "unexpected_field_name": {}, "ruleSet": {}},
+        )
+    assert exc_info.type is HTTPResponse
+    assert str(exc_info.value) == "HTTPResponse 422"
 
 
 async def test_forward_when_not_ready():