schema_registry: standalone module for SR with basic dependency injection

.coveragerc

@@ -1,4 +1,4 @@
 [run]
 branch = true
 relative_files = true
-source = src/karapace
+source = src

.dockerignore

@@ -10,7 +10,6 @@
 !LICENSE
 !pyproject.toml
 !setup.py
-!container/start.sh
 !container/healthcheck.py
 
 # Ignore some files in source directories.

.github/workflows/tests.yml

@@ -14,13 +14,14 @@ env:
   FORCE_COLOR: 1
   PIP_PROGRESS_BAR: off
   PYTHONUNBUFFERED: 1
+  KARAPACE_DOTENV: ${{ github.workspace }}/karapace.config.env
 
 jobs:
   tests:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.9', '3.10', '3.11', '3.12' ]
+        python-version: [ '3.10', '3.11', '3.12' ]
     env:
       PYTEST_ADDOPTS: >-
         --log-dir=/tmp/ci-logs
@@ -44,11 +45,11 @@ jobs:
     - run: make unit-tests
       env:
         COVERAGE_FILE: ".coverage.${{ matrix.python-version }}"
-        PYTEST_ARGS: "--cov=karapace --cov-append --numprocesses 4"
+        PYTEST_ARGS: "--cov=src --cov-append --numprocesses 4"
     - run: make integration-tests
       env:
         COVERAGE_FILE: ".coverage.${{ matrix.python-version }}"
-        PYTEST_ARGS: "--cov=karapace --cov-append --random-order --numprocesses 4"
+        PYTEST_ARGS: "--cov=src --cov-append --random-order --numprocesses 4"
 
     - name: Archive logs
       uses: actions/upload-artifact@v4

GNUmakefile

@@ -3,13 +3,14 @@ SHELL := /usr/bin/env bash
 VENV_DIR ?= $(CURDIR)/venv
 PIP      ?= pip3 --disable-pip-version-check --no-input --require-virtualenv
 PYTHON   ?= python3
+CLI   ?= docker-compose -f container/compose.yml run karapace-cli
 PYTHON_VERSION ?= 3.9
 
 define PIN_VERSIONS_COMMAND
 pip install pip-tools && \
-	python -m piptools compile -o /karapace/requirements/requirements.txt /karapace/pyproject.toml && \
-	python -m piptools compile --extra dev -o /karapace/requirements/requirements-dev.txt /karapace/pyproject.toml && \
-	python -m piptools compile --extra typing -o /karapace/requirements/requirements-typing.txt /karapace/pyproject.toml
+	python -m piptools compile --upgrade -o /karapace/requirements/requirements.txt /karapace/pyproject.toml && \
+	python -m piptools compile --upgrade --extra dev -o /karapace/requirements/requirements-dev.txt /karapace/pyproject.toml && \
+	python -m piptools compile --upgrade --extra typing -o /karapace/requirements/requirements-typing.txt /karapace/pyproject.toml
 endef
 
 
@@ -102,3 +103,9 @@ schema:
 .PHONY: pin-requirements
 pin-requirements:
 	docker run -e CUSTOM_COMPILE_COMMAND='make pin-requirements' -it -v .:/karapace --security-opt label=disable python:$(PYTHON_VERSION)-bullseye /bin/bash -c "$(PIN_VERSIONS_COMMAND)"
+
+cli:
+	# $(CLI) python3 -m pytest -vvv tests/integration/test_client.py
+	# $(CLI) python3 -m pytest -vvv tests/integration/schema_registry/test_jsonschema.py
+	$(CLI) python3 -m pytest -vvv tests/integration/
+	# $(CLI) python3 -m pytest -vvv tests/unit

bin/smoke-test-registry.sh

@@ -6,6 +6,7 @@ for ((i = 0; i <= retries; i++)); do
     response=$(
         curl --silent --verbose --fail --request POST \
             --header 'Content-Type: application/vnd.schemaregistry.v1+json' \
+            --header 'Authorization: Basic YWRtaW46YWRtaW4=' \
             --data '{"schema": "{\"type\": \"record\", \"name\": \"Obj\", \"fields\":[{\"name\": \"age\", \"type\": \"int\"}]}"}' \
             http://localhost:8081/subjects/test-key/versions
     )

container/Dockerfile

@@ -55,10 +55,6 @@ RUN apt-get update \
 COPY --from=builder /venv /venv
 ENV PATH="/venv/bin:$PATH"
 
-COPY ./container/start.sh /opt/karapace
-RUN chmod 500 /opt/karapace/start.sh \
- && chown karapace:karapace /opt/karapace/start.sh
-
 COPY ./container/healthcheck.py /opt/karapace
 
 WORKDIR /opt/karapace

container/Dockerfile.dev

@@ -0,0 +1,62 @@
+# Current versions of avro and zstandard don't yet have wheels for 3.11.
+FROM python:3.10.11-bullseye AS builder
+
+ARG KARAPACE_VERSION
+
+# Create, activate, and enforce usage of virtualenv.
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+ENV PIP_REQUIRE_VIRTUALENV=true
+
+# Install golang needed by extensions
+ENV GO_VERSION=1.21.0
+ENV PATH="/usr/local/go/bin:${PATH}"
+RUN wget --progress=dot:giga "https://go.dev/dl/go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" \
+ && tar -C /usr/local -xzf "go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz" \
+ && rm "go${GO_VERSION}.linux-$(dpkg --print-architecture).tar.gz"
+
+# Copy the requirements.txt and install dependencies in venv. Using a separate
+# command to use layer caching.
+#
+# Note: the requirements.txt is pinned, if any of the dependencies is updated
+# the cache will be invalidated and the image regenerated, which is the
+# intended behavior.
+COPY ./requirements/requirements.txt /build/
+COPY ./requirements/requirements-dev.txt /build/
+RUN --mount=type=cache,target=/root/.cache/pip \
+    python3 -m pip install -r /build/requirements.txt -r /build/requirements-dev.txt
+
+COPY . /build/karapace-repo
+WORKDIR /build/karapace-repo
+RUN --mount=type=cache,target=/root/.cache/pip \
+    if [ -z "${KARAPACE_VERSION}" ]; then \
+        PRETEND_VERSION="$(python -c 'from src.karapace import version; print(version.__version__)')"; \
+    else \
+        PRETEND_VERSION=$KARAPACE_VERSION; \
+    fi; \
+    SETUPTOOLS_SCM_PRETEND_VERSION=$PRETEND_VERSION python3 -m pip install --no-deps .
+
+# Karapace image, i.e. production.
+FROM python:3.10.11-slim-bullseye AS karapace
+
+# Setup user and directories.
+RUN groupadd --system karapace \
+ && useradd --system --gid karapace karapace \
+ && mkdir /opt/karapace /opt/karapace/runtime /var/log/karapace \
+ && chown --recursive karapace:karapace /opt/karapace /var/log/karapace
+
+# Install protobuf compiler.
+ARG PROTOBUF_COMPILER_VERSION="3.12.4-1+deb11u1"
+RUN apt-get update \
+ && apt-get install --assume-yes --no-install-recommends \
+    protobuf-compiler=$PROTOBUF_COMPILER_VERSION \
+ && rm -rf /var/lib/apt/lists/*
+
+# Copy virtualenv from builder and activate it.
+COPY --from=builder /venv /venv
+ENV PATH="/venv/bin:$PATH"
+
+COPY ./container/healthcheck.py /opt/karapace
+
+WORKDIR /opt/karapace
+USER karapace

container/compose.yml

@@ -4,7 +4,7 @@ services:
   zookeeper:
     image: confluentinc/cp-zookeeper:latest
     ports:
-      - "2181:2181"
+      - 2181:2181
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_TICK_TIME: 2000
@@ -14,16 +14,16 @@ services:
     depends_on:
       - zookeeper
     ports:
-      - "9101:9101"  # JMX
-      - "9092:9092"  # Kafka
+      - 9101:9101  # JMX
+      - 9092:9092  # Kafka
     environment:
       # Listeners:
       # PLAINTEXT_HOST -> Expose kafka to the host network
       # PLAINTEXT -> Used by kafka for inter broker communication / containers
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
       KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
-      KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://karapace-registry:8081
+      KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://karapace-schema-registry:8081
       # Metrics:
       KAFKA_JMX_PORT: 9101
       KAFKA_JMX_HOSTNAME: localhost
@@ -54,62 +54,60 @@ services:
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 6000
       KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
 
-  karapace-registry:
+  karapace-schema-registry:
     image: ghcr.io/aiven-open/karapace:develop
     build:
       context: ..
       dockerfile: container/Dockerfile
     entrypoint:
-      - /bin/bash
-      - /opt/karapace/start.sh
-      - registry
+      - python3
+      - -m
+      - schema_registry
     depends_on:
       - kafka
     ports:
-      - "8081:8081"
+      - 8081:8081
+    volumes:
+    - ./karapace.registry.env:/opt/karapace/karapace.env
     environment:
-      KARAPACE_ADVERTISED_HOSTNAME: karapace-registry
-      KARAPACE_BOOTSTRAP_URI: kafka:29092
+      KARAPACE_DOTENV: /opt/karapace/karapace.env
       KARAPACE_PORT: 8081
-      KARAPACE_HOST: 0.0.0.0
-      KARAPACE_CLIENT_ID: karapace
-      KARAPACE_GROUP_ID: karapace-registry
-      KARAPACE_MASTER_ELIGIBILITY: "true"
-      KARAPACE_TOPIC_NAME: _schemas
-      KARAPACE_LOG_LEVEL: WARNING
-      KARAPACE_COMPATIBILITY: FULL
-      KARAPACE_STATSD_HOST: statsd-exporter
-      KARAPACE_STATSD_PORT: 8125
-      KARAPACE_KAFKA_SCHEMA_READER_STRICT_MODE: false
-      KARAPACE_KAFKA_RETRIABLE_ERRORS_SILENCED: true
 
-  karapace-rest:
+  karapace-rest-proxy:
     image: ghcr.io/aiven-open/karapace:develop
     build:
       context: ..
       dockerfile: container/Dockerfile
     entrypoint:
-      - /bin/bash
-      - /opt/karapace/start.sh
-      - rest
+      - python3
+      - -m
+      - karapace.karapace_all
     depends_on:
       - kafka
-      - karapace-registry
+      - karapace-schema-registry
     ports:
-      - "8082:8082"
+      - 8082:8082
+    volumes:
+    - ./karapace.rest.env:/opt/karapace/karapace.env
     environment:
+      KARAPACE_DOTENV: /opt/karapace/karapace.env
       KARAPACE_PORT: 8082
-      KARAPACE_HOST: 0.0.0.0
-      KARAPACE_ADVERTISED_HOSTNAME: karapace-rest
-      KARAPACE_BOOTSTRAP_URI: kafka:29092
-      KARAPACE_REGISTRY_HOST: karapace-registry
-      KARAPACE_REGISTRY_PORT: 8081
-      KARAPACE_ADMIN_METADATA_MAX_AGE: 0
-      KARAPACE_LOG_LEVEL: WARNING
-      KARAPACE_STATSD_HOST: statsd-exporter
-      KARAPACE_STATSD_PORT: 8125
-      KARAPACE_KAFKA_SCHEMA_READER_STRICT_MODE: false
-      KARAPACE_KAFKA_RETRIABLE_ERRORS_SILENCED: true
+
+  karapace-cli:
+    image: ghcr.io/aiven-open/karapace:cli
+    build:
+      context: ..
+      dockerfile: container/Dockerfile.dev
+    tty: true
+    depends_on:
+      - kafka
+      - karapace-schema-registry
+      - karapace-rest-proxy
+    volumes:
+    - ../tests:/opt/karapace/tests
+    - ../karapace.config.env:/opt/karapace/karapace.env
+    environment:
+      KARAPACE_DOTENV: /opt/karapace/karapace.env
 
   prometheus:
     image: prom/prometheus

container/karapace.registry.env

@@ -0,0 +1,47 @@
+KARAPACE_DOTENV=/opt/karapace/karapace.env
+ACCESS_LOGS_DEBUG=False
+ADVERTISED_HOSTNAME=karapace-schema-registry
+ADVERTISED_PORT=8081
+ADVERTISED_PROTOCOL=http
+BOOTSTRAP_URI=kafka:29092
+CLIENT_ID=karapace-schema-registry
+COMPATIBILITY=BACKWARD
+CONNECTIONS_MAX_IDLE_MS=15000
+CONSUMER_ENABLE_AUTO_COMMIT=True
+CONSUMER_REQUEST_TIMEOUT_MS=11000
+CONSUMER_REQUEST_MAX_BYTES=67108864
+CONSUMER_IDLE_DISCONNECT_TIMEOUT=0
+FETCH_MIN_BYTES=1
+GROUP_ID=karapace-schema-registry
+HOST=0.0.0.0
+PORT=8081
+REGISTRY_HOST=karapace-schema-registry
+REGISTRY_PORT=8081
+REST_AUTHORIZATION=False
+LOG_HANDLER=stdout
+LOG_LEVEL=WARNING
+LOG_FORMAT=%(asctime)s [%(threadName)s] %(filename)s:%(funcName)s:%(lineno)d %(message)s
+MASTER_ELIGIBILITY=True
+REPLICATION_FACTOR=1
+SECURITY_PROTOCOL=PLAINTEXT
+SSL_CHECK_HOSTNAME=True
+TOPIC_NAME=_schemas
+METADATA_MAX_AGE_MS=60000
+ADMIN_METADATA_MAX_AGE=5
+PRODUCER_ACKS=1
+PRODUCER_COUNT=5
+PRODUCER_LINGER_MS=100
+PRODUCER_MAX_REQUEST_SIZE=1048576
+SESSION_TIMEOUT_MS=10000
+KARAPACE_REST=False
+KARAPACE_REGISTRY=True
+KARAPACE_PORT=8081
+NAME_STRATEGY=topic_name
+NAME_STRATEGY_VALIDATION=True
+MASTER_ELECTION_STRATEGY=lowest
+PROTOBUF_RUNTIME_DIRECTORY=runtime
+STATSD_HOST=statsd-exporter
+STATSD_PORT=8125
+KAFKA_SCHEMA_READER_STRICT_MODE=False
+KAFKA_RETRIABLE_ERRORS_SILENCED=True
+USE_PROTOBUF_FORMATTER=False

container/karapace.rest.env

@@ -0,0 +1,51 @@
+KARAPACE_DOTENV=/opt/karapace/karapace.env
+ACCESS_LOGS_DEBUG=False
+# ACCESS_LOG_CLASS=karapace.utils.DebugAccessLogger
+ACCESS_LOG_CLASS=aiohttp.web_log.AccessLogger
+ADVERTISED_HOSTNAME=karapace-rest-proxy
+ADVERTISED_PORT=8082
+ADVERTISED_PROTOCOL=http
+BOOTSTRAP_URI=kafka:29092
+CLIENT_ID=karapace-rest-proxy
+COMPATIBILITY=BACKWARD
+CONNECTIONS_MAX_IDLE_MS=15000
+CONSUMER_ENABLE_AUTO_COMMIT=True
+CONSUMER_REQUEST_TIMEOUT_MS=11000
+CONSUMER_REQUEST_MAX_BYTES=67108864
+CONSUMER_IDLE_DISCONNECT_TIMEOUT=0
+FETCH_MIN_BYTES=1
+GROUP_ID=karapace-rest-proxy
+HOST=0.0.0.0
+PORT=8082
+REGISTRY_HOST=karapace-schema-registry
+REGISTRY_PORT=8081
+REST_AUTHORIZATION=False
+LOG_HANDLER=stdout
+LOG_LEVEL=WARNING
+LOG_FORMAT=%(asctime)s [%(threadName)s] %(filename)s:%(funcName)s:%(lineno)d %(message)s
+MASTER_ELIGIBILITY=True
+REPLICATION_FACTOR=1
+SECURITY_PROTOCOL=PLAINTEXT
+SSL_CHECK_HOSTNAME=True
+TOPIC_NAME=_schemas
+METADATA_MAX_AGE_MS=60000
+ADMIN_METADATA_MAX_AGE=5
+PRODUCER_ACKS=1
+PRODUCER_COUNT=5
+PRODUCER_LINGER_MS=100
+PRODUCER_MAX_REQUEST_SIZE=1048576
+SESSION_TIMEOUT_MS=10000
+KARAPACE_REST=True
+KARAPACE_REGISTRY=False
+KARAPACE_PORT=8082
+NAME_STRATEGY=topic_name
+NAME_STRATEGY_VALIDATION=True
+MASTER_ELECTION_STRATEGY=lowest
+PROTOBUF_RUNTIME_DIRECTORY=runtime
+STATSD_HOST=statsd-exporter
+STATSD_PORT=8125
+KAFKA_SCHEMA_READER_STRICT_MODE=False
+KAFKA_RETRIABLE_ERRORS_SILENCED=True
+USE_PROTOBUF_FORMATTER=False
+HTTP_REQUEST_MAX_SIZE=1048576
+TAGS='{ "app": "karapace-rest-proxy" }'