fix(HMS-2703): allow to configure TLS without CA cert

Signed-off-by: Lukas Zapletal <[email protected]> Signed-off-by: Lukáš Zapletal <[email protected]>
RHEnVision · Oct 3, 2023 · 97ba236 · 97ba236
1 parent 88f4954
commit 97ba236
Show file tree

Hide file tree

Showing 5 changed files with 62 additions and 41 deletions.
diff --git a/config/api.env.example b/config/api.env.example
@@ -111,7 +111,7 @@
 #   KAFKA_BROKERS slice
 #     	kafka hostname:port list of brokers (default "localhost:9092")
 #   KAFKA_CA_CERT string
-#     	kafka TLS CA certificate path (default "")
+#     	kafka TLS CA certificate path (use the OS cert store when blank) (default "")
 #   KAFKA_ENABLED bool
 #     	kafka service enabled (default "false")
 #   KAFKA_SASL_MECHANISM string
@@ -122,6 +122,10 @@
 #     	kafka SASL security protocol (default "")
 #   KAFKA_SASL_USERNAME string
 #     	kafka SASL username (default "")
+#   KAFKA_TLS_ENABLED bool
+#     	enable TLS or use plaintext when false (default "false")
+#   KAFKA_TLS_SKIP_VERIFY bool
+#     	do not verify TLS server certificate (default "false")
 #   LOGGING_LEVEL string
 #     	logger level (trace, debug, info, warn, error, fatal, panic) (default "info")
 #   LOGGING_MAX_FIELD int

diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml
@@ -103,6 +103,8 @@ objects:
                 value: ${APP_CACHE_TYPE}
               - name: WORKER_QUEUE
                 value: ${WORKER_QUEUE}
+              - name: KAFKA_TLS_ENABLED
+                value: ${KAFKA_TLS_ENABLED}
             resources:
               limits:
                 cpu: ${{CPU_LIMIT}}
@@ -190,6 +192,8 @@ objects:
                 value: ${APP_INSTANCE_PREFIX}
               - name: APP_CACHE_TYPE
                 value: ${APP_CACHE_TYPE}
+              - name: KAFKA_TLS_ENABLED
+                value: ${KAFKA_TLS_ENABLED}
             resources:
               limits:
                 cpu: ${{CPU_LIMIT}}
@@ -239,6 +243,8 @@ objects:
                 value: ${APP_INSTANCE_PREFIX}
               - name: APP_CACHE_TYPE
                 value: ${APP_CACHE_TYPE}
+              - name: KAFKA_TLS_ENABLED
+                value: ${KAFKA_TLS_ENABLED}
             resources:
               limits:
                 cpu: ${{CPU_LIMIT}}
@@ -365,6 +371,8 @@ objects:
                 value: ${APP_CACHE_TYPE}
               - name: WORKER_QUEUE
                 value: ${WORKER_QUEUE}
+              - name: KAFKA_TLS_ENABLED
+                value: ${KAFKA_TLS_ENABLED}
             resources:
               limits:
                 cpu: ${{CPU_LIMIT}}
@@ -572,3 +580,6 @@ parameters:
   - description: Notification service enabled
     name: APP_NOTIFICATIONS_ENABLED
     value: "true"
+  - description: Kafka TLS connection
+    name: KAFKA_TLS_ENABLED
+    value: "false"
diff --git a/docs/make.md b/docs/make.md
@@ -4,10 +4,10 @@
 Usage:
   make <target>
 
-Database migrations
-  migrate               Run database migration
-  purgedb               Delete database (dangerous!)
-  generate-migration    Generate new migration file, use MIGRATION_NAME=name
+HTTP Clients
+  update-clients        Update OpenAPI specs from upstream
+  generate-clients      Generate HTTP client stubs
+  validate-clients      Compare generated client code with git
 
 Code quality
   format                Format Go source code using `go fmt`
@@ -26,49 +26,49 @@ Building
   run                   Build and run backend API
   clean                 Clean build artifacts and cache
 
-Help
-  help                  Print out the help content
-  generate-help-doc     Generate 'make help' markdown in docs/
-  validate-help-doc     Compare example configuration
-  generate-example-config  Generate example configuration
-  validate-example-config  Compare example configuration
+Image building
+  build-podman          Build container image using Podman
 
 Dashboard
   generate-dashboard    Generate dashboard
   validate-dashboard    Compare dashboard configmaps with git
 
+Database migrations
+  migrate               Run database migration
+  purgedb               Delete database (dangerous!)
+  generate-migration    Generate new migration file, use MIGRATION_NAME=name
+
 Go modules
   tidy-deps             Cleanup Go modules
   download-deps         Download Go modules
   list-mods             List application modules
   list-deps             List dependencies and their versions
   update-deps           Update Go modules to latest versions
 
-Go commands
-  install-go            Install required Go version
-  install-tools         Install required Go commands into ./bin
-  update-tools          Update required Go commands
-  generate-changelog    Generate CHANGELOG.md from git history
+Help
+  help                  Print out the help content
+  generate-help-doc     Generate 'make help' markdown in docs/
+  validate-help-doc     Compare example configuration
+  generate-example-config  Generate example configuration
+  validate-example-config  Compare example configuration
+
+OpenAPI
+  generate-spec         Generate OpenAPI spec
+  validate-spec         Compare OpenAPI spec with git
 
 Testing
   test                  Run unit tests
   integration-test      Run integration tests (require database)
 
-OpenAPI
-  generate-spec         Generate OpenAPI spec
-  validate-spec         Compare OpenAPI spec with git
+Go commands
+  install-go            Install required Go version
+  install-tools         Install required Go commands into ./bin
+  update-tools          Update required Go commands
+  generate-changelog    Generate CHANGELOG.md from git history
 
 Instance types
   generate-azure-types  Generate instance types for Azure
   generate-ec2-types    Generate instance types for EC2
   generate-gcp-types    Generate instance types for GCP
   generate-types        Generate instance types for all providers
-
-HTTP Clients
-  update-clients        Update OpenAPI specs from upstream
-  generate-clients      Generate HTTP client stubs
-  validate-clients      Compare generated client code with git
-
-Image building
-  build-podman          Build container image using Podman
 ```
diff --git a/internal/config/config.go b/internal/config/config.go
@@ -155,11 +155,13 @@ var config struct {
 		Dsn string `env:"DSN" env-default:"" env-description:"data source name (empty value disables Sentry)"`
 	} `env-prefix:"SENTRY_"`
 	Kafka struct {
-		Enabled  bool     `env:"ENABLED" env-default:"false" env-description:"kafka service enabled"`
-		Brokers  []string `env:"BROKERS" env-default:"localhost:9092" env-description:"kafka hostname:port list of brokers"`
-		AuthType string   `env:"AUTH_TYPE" env-default:"" env-description:"kafka authentication type (mtls, sasl or empty)"`
-		CACert   string   `env:"CA_CERT" env-default:"" env-description:"kafka TLS CA certificate path"`
-		SASL     struct {
+		Enabled       bool     `env:"ENABLED" env-default:"false" env-description:"kafka service enabled"`
+		TlsEnabled    bool     `env:"TLS_ENABLED" env-default:"false" env-description:"enable TLS or use plaintext when false"`
+		TlsSkipVerify bool     `env:"TLS_SKIP_VERIFY" env-default:"false" env-description:"do not verify TLS server certificate"`
+		Brokers       []string `env:"BROKERS" env-default:"localhost:9092" env-description:"kafka hostname:port list of brokers"`
+		AuthType      string   `env:"AUTH_TYPE" env-default:"" env-description:"kafka authentication type (mtls, sasl or empty)"`
+		CACert        string   `env:"CA_CERT" env-default:"" env-description:"kafka TLS CA certificate path (use the OS cert store when blank)"`
+		SASL          struct {
 			Username         string `env:"USERNAME" env-default:"" env-description:"kafka SASL username"`
 			Password         string `env:"PASSWORD" env-default:"" env-description:"kafka SASL password"`
 			SaslMechanism    string `env:"MECHANISM" env-default:"" env-description:"kafka SASL mechanism (scram-sha-512, scram-sha-256 or plain)"`

diff --git a/internal/kafka/kafka.go b/internal/kafka/kafka.go
@@ -39,7 +39,7 @@ var (
 
 func createSASLMechanism(saslMechanismName string, username string, password string) (sasl.Mechanism, error) {
 	switch strings.ToLower(saslMechanismName) {
-	case "plain":
+	case "plain", "none":
 		return plain.Mechanism{
 			Username: username,
 			Password: password,
@@ -76,30 +76,34 @@ func InitializeKafkaBroker(ctx context.Context) error {
 }
 
 func NewKafkaBroker(ctx context.Context) (Broker, error) {
+	var pool *x509.CertPool
 	var tlsConfig *tls.Config
 	var saslMechanism sasl.Mechanism
 
 	logger := zerolog.Ctx(ctx)
-	logger.Debug().Msgf("Setting up Kafka transport: %v CA:%v SASL:%v", config.Kafka.Brokers,
-		config.Kafka.CACert != "", config.Kafka.SASL.SaslMechanism != "" && config.Kafka.SASL.SaslMechanism != "none")
+	logger.Debug().Msgf("Setting up Kafka transport: %v", config.Kafka.Brokers)
 
-	// configure TLS when CA certificate was provided
 	if config.Kafka.CACert != "" {
-		logger.Debug().Str("cert", config.Kafka.CACert).Msg("Adding CA certificates to the pool")
+		logger.Debug().Str("cert", config.Kafka.CACert).Msg("Configuring TLS CA pool for Kafka")
 
 		pemCerts := config.Kafka.CACert
-		pool := x509.NewCertPool()
+		pool = x509.NewCertPool()
 		if ok := pool.AppendCertsFromPEM([]byte(pemCerts)); !ok {
 			logger.Warn().Msg("Could not add an CA cert to the pool")
 		}
+	}
+
+	if config.Kafka.TlsEnabled && !config.InEphemeralClowder() {
+		logger.Debug().Msg("Configuring Kafka for TLS")
 
+		//nolint:gosec
 		tlsConfig = &tls.Config{
-			MinVersion: tls.VersionTLS13,
-			RootCAs:    pool,
+			MinVersion:         tls.VersionTLS12,
+			RootCAs:            pool,
+			InsecureSkipVerify: config.Kafka.TlsSkipVerify,
 		}
 	}
 
-	// configure SASL if mechanism was provided
 	if config.Kafka.SASL.SaslMechanism != "" {
 		var err error
 		saslMechanism, err = createSASLMechanism(config.Kafka.SASL.SaslMechanism, config.Kafka.SASL.Username, config.Kafka.SASL.Password)