Merge pull request #575 from bcgov/dev

Merging From DEV to MAIN

etl/debezium-jdbc/Dockerfile

@@ -5,9 +5,9 @@ WORKDIR /app/
 COPY ./ ./
 RUN mvn clean install
 
+RUN echo "DEBEZIUM_VERSION is $DEBEZIUM_VERSION"
 
-
-FROM debezium/connect:$DEBEZIUM_VERSION
+FROM debezium/connect:1.9
 
 ARG ORACLE_JDBC_VERSION
 ARG ORACLE_JDBC_MD5
@@ -34,6 +34,8 @@ COPY --from=builder /app/target/*.jar $KAFKA_CONNECT_JDBC_DIR/
 COPY --from=builder /app/lib/*.jar $KAFKA_CONNECT_JDBC_DIR/
 COPY InstallCert.class .
 COPY "InstallCert\$SavingTrustManager.class" .
+COPY connect-log4j.properties /kafka/config/
+COPY log4j.properties /kafka/config/
 
 RUN java InstallCert --quiet "${ORACLE_DB_HOST}:${ORACLE_DB_PORT}"
 RUN keytool -exportcert -alias "$ORACLE_DB_HOST-1" -keystore jssecacerts -storepass changeit -file oracle.cer

etl/debezium-jdbc/connect-log4j.properties

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=ERROR, stdout, connectAppender
+
+# Send the logs to the console.
+#
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+
+# Send the logs to a file, rolling the file at midnight local time. For example, the `File` option specifies the
+# location of the log files (e.g. ${kafka.logs.dir}/connect.log), and at midnight local time the file is closed
+# and copied in the same directory but with a filename that ends in the `DatePattern` option.
+#
+log4j.appender.connectAppender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.connectAppender.DatePattern='.'yyyy-MM-dd-HH
+log4j.appender.connectAppender.File=${kafka.logs.dir}/connect.log
+log4j.appender.connectAppender.layout=org.apache.log4j.PatternLayout
+
+# The `%X{connector.context}` parameter in the layout includes connector-specific and task-specific information
+# in the log messages, where appropriate. This makes it easier to identify those log messages that apply to a
+# specific connector.
+#
+connect.log.pattern=[%d] %p %X{connector.context}%m (%c:%L)%n
+
+log4j.appender.stdout.layout.ConversionPattern=${connect.log.pattern}
+log4j.appender.connectAppender.layout.ConversionPattern=${connect.log.pattern}
+
+log4j.logger.org.apache.zookeeper=ERROR
+log4j.logger.org.reflections=ERROR

etl/debezium-jdbc/log4j.properties

@@ -0,0 +1,23 @@
+kafka.logs.dir=logs
+
+log4j.rootLogger=INFO, stdout, appender
+
+# Disable excessive reflection warnings - KAFKA-5229
+log4j.logger.org.reflections=ERROR
+
+# Disable excessive logging by consumers and producers.
+log4j.logger.org.apache.kafka=ERROR
+log4j.logger.org.apache.kafka.clients.consumer=WARN
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.threshold=INFO
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p  %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext}  %m   [%c]%n
+
+
+log4j.appender.appender=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.appender.DatePattern='.'yyyy-MM-dd-HH
+log4j.appender.appender.File=${kafka.logs.dir}/connect-service.log
+log4j.appender.appender.layout=org.apache.log4j.PatternLayout
+log4j.appender.appender.layout.ConversionPattern=%d{ISO8601} %-5p  %X{dbz.connectorType}|%X{dbz.connectorName}|%X{dbz.connectorContext}  %m   [%c]%n
+

etl/debezium-postgis/15-3.3/Dockerfile

@@ -44,4 +44,6 @@ COPY --from=build /usr/share/postgresql/$PG_MAJOR/extension/decoderbufs.control
 COPY postgresql.conf.sample /usr/share/postgresql/postgresql.conf.sample
 
 # Copy the script which will initialize the replication permissions
-COPY /docker-entrypoint-initdb.d /docker-entrypoint-initdb.d
+COPY /docker-entrypoint-initdb.d/init-permissions.sh /docker-entrypoint-initdb.d/init-permissions.sh
+
+RUN chmod +x /docker-entrypoint-initdb.d/init-permissions.sh

etl/docker-compose.yaml

@@ -19,7 +19,7 @@ services:
     build:
       context: debezium-postgis/15-3.3
     ports:
-     - "5432:5432"
+     - "7432:5432"
     environment:
      - POSTGRES_USER=postgres
      - POSTGRES_PASSWORD=postgres