GEOMESA-3428 Update to Kafka 3.9 (#3250)

locationtech · Jan 3, 2025 · 56c339e · 56c339e
1 parent 4ce9b2e
commit 56c339e
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 12 deletions.
diff --git a/build/cqs.tsv b/build/cqs.tsv
@@ -284,8 +284,8 @@ ch.qos.reload4j:reload4j	1.2.25	provided
 com.amazonaws:aws-java-sdk-s3	1.12.735	provided
 com.datastax.cassandra:cassandra-driver-core	3.11.5	provided
 com.datastax.cassandra:cassandra-driver-mapping	3.11.5	provided
-io.confluent:kafka-avro-serializer	6.2.9	provided
-io.confluent:kafka-schema-registry-client	6.2.9	provided
+io.confluent:kafka-avro-serializer	7.8.0	provided
+io.confluent:kafka-schema-registry-client	7.8.0	provided
 io.micrometer:micrometer-registry-cloudwatch2	1.13.4	provided
 io.micrometer:micrometer-registry-prometheus	1.13.4	provided
 io.prometheus:prometheus-metrics-exporter-httpserver	1.3.1	provided
@@ -304,9 +304,9 @@ org.apache.hadoop:hadoop-yarn-api	3.4.1	provided
 org.apache.hadoop:hadoop-yarn-common	3.4.1	provided
 org.apache.hbase:hbase-server	2.6.1-hadoop3	provided
 org.apache.htrace:htrace-core4	4.1.0-incubating	provided
-org.apache.kafka:kafka-clients	3.7.0	provided
-org.apache.kafka:kafka-streams-scala_2.12	3.7.0	provided
-org.apache.kafka:kafka-streams	3.7.0	provided
+org.apache.kafka:kafka-clients	3.9.0	provided
+org.apache.kafka:kafka-streams-scala_2.12	3.9.0	provided
+org.apache.kafka:kafka-streams	3.9.0	provided
 org.apache.sedona:sedona-spark-3.4_2.12	1.5.0	provided
 org.apache.sedona:sedona-spark-common-3.4_2.12	1.5.0	provided
 org.apache.spark:spark-catalyst_2.12	3.5.0	provided
@@ -329,8 +329,8 @@ org.apache.cassandra:cassandra-all	3.11.14	test
 org.apache.cassandra:cassandra-thrift	3.11.14	test
 org.apache.curator:curator-test	5.6.0	test
 org.apache.hbase:hbase-testing-util	2.6.1-hadoop3	test
-org.apache.kafka:kafka-clients	test:3.7.0	test
-org.apache.kafka:kafka-streams-test-utils	3.7.0	test
+org.apache.kafka:kafka-clients	test:3.9.0	test
+org.apache.kafka:kafka-streams-test-utils	3.9.0	test
 org.apache.logging.log4j:log4j-api	2.22.1	test
 org.apache.logging.log4j:log4j-core	2.22.1	test
 org.apache.sedona:sedona-common	1.5.0	test

diff --git a/docs/user/upgrade.rst b/docs/user/upgrade.rst
@@ -99,9 +99,11 @@ Dependency Version Upgrades
 The following dependencies have been upgraded:
 
 * arrow ``17.0.0`` -> ``18.1.0``
+* confluent ``6.2.9`` -> ``7.8.0``
 * geotools ``32.0`` -> ``32.1``
 * hbase ``2.5.8-hadoop3`` -> ``2.6.1-hadoop3``
 * hadoop ``3.4.0`` -> ``3.4.1``
+* kafka ``3.7.0`` -> ``3.9.0``
 * orc ``1.9.4`` -> ``1.9.5``
 * spring-security ``5.8.14`` -> ``5.8.15``
 * zookeeper ``3.9.2`` -> ``3.9.3``

diff --git a/pom.xml b/pom.xml
@@ -142,9 +142,9 @@
         <hbase.protobuf.version>2.5.0</hbase.protobuf.version>
         <protobuf.version>3.25.3</protobuf.version> <!-- see also confluent.protobuf.version, hbase.protobuf.version -->
         <cassandra.version>3.11.5</cassandra.version>
-        <kafka.version>3.7.0</kafka.version> <!-- needs to align with confluent version -->
-        <confluent.version>6.2.9</confluent.version> <!-- corresponds to kafka 2.8 - newer versions have license issues, see https://gitlab.eclipse.org/eclipsefdn/emo-team/iplab/-/issues/13628 -->
-        <confluent.protobuf.version>3.19.6</confluent.protobuf.version>
+        <kafka.version>3.9.0</kafka.version> <!-- needs to align with confluent version -->
+        <confluent.version>7.8.0</confluent.version> <!-- confluent 7.8.x corresponds to kafka 3.8.x -->
+        <confluent.protobuf.version>3.25.5</confluent.protobuf.version>
         <sedona.version>1.5.0</sedona.version>
         <sedona.spark.version>3.4</sedona.spark.version>
         <sedona.h3.version>4.1.1</sedona.h3.version>
@@ -202,7 +202,7 @@
         <test.fork.count>2</test.fork.count>
         <test.enable.assertions>false</test.enable.assertions> <!-- arrow tests become extremely slow with assertions enabled-->
         <test.accumulo.docker.tag>2.1.3</test.accumulo.docker.tag>
-        <test.confluent.docker.tag>7.6.0</test.confluent.docker.tag> <!-- confluent 7.6.x corresponds to kafka 3.6.x -->
+        <test.confluent.docker.tag>7.8.0</test.confluent.docker.tag> <!-- confluent 7.8.x corresponds to kafka 3.8.x -->
         <test.minio.docker.tag>RELEASE.2024-10-29T16-01-48Z</test.minio.docker.tag>
         <test.postgis.docker.tag>15-3.4</test.postgis.docker.tag>
         <test.postgres.docker.tag>15.1</test.postgres.docker.tag>
@@ -223,7 +223,7 @@
         <cassandra.driver.version.recommended>${cassandra.version}</cassandra.driver.version.recommended>
         <hadoop.version.recommended>${hadoop.version}</hadoop.version.recommended>
         <zookeeper.version.recommended>${zookeeper.version}</zookeeper.version.recommended>
-        <confluent.version.recommended>7.6.0</confluent.version.recommended>
+        <confluent.version.recommended>7.8.0</confluent.version.recommended>
     </properties>
 
     <profiles>