Skip to content

Commit

Permalink
Enable envoy idleTimeout and TCP keep-alive for connections to kafka …
Browse files Browse the repository at this point in the history
…and clients

1/ Kafka broker defines connections.max.idle.ms=600s
To ensure envoy as a client for kafka broker is terminating
the connection first to avoid network disconnects
this patch is setting the idleTimeout to value slightly less
than that

2/ Enable tcp-keep alive for all TCP connections established by envoy to kafka
and to client (or fronting Load Balancer)
  • Loading branch information
amuraru committed Oct 7, 2021
1 parent 1c20cf6 commit 507a471
Show file tree
Hide file tree
Showing 2 changed files with 276 additions and 5 deletions.
210 changes: 210 additions & 0 deletions controllers/tests/kafkacluster_controller_envoy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,11 @@ staticResources:
portValue: 9094
name: broker-0
type: STRICT_DNS
upstreamConnectionOptions:
tcpKeepalive:
keepaliveInterval: 30
keepaliveProbes: 3
keepaliveTime: 30
- circuitBreakers:
thresholds:
- maxConnections: 1000000000
Expand All @@ -143,6 +148,11 @@ staticResources:
portValue: 9094
name: broker-1
type: STRICT_DNS
upstreamConnectionOptions:
tcpKeepalive:
keepaliveInterval: 30
keepaliveProbes: 3
keepaliveTime: 30
- circuitBreakers:
thresholds:
- maxConnections: 1000000000
Expand All @@ -166,6 +176,11 @@ staticResources:
portValue: 9094
name: broker-2
type: STRICT_DNS
upstreamConnectionOptions:
tcpKeepalive:
keepaliveInterval: 30
keepaliveProbes: 3
keepaliveTime: 30
- circuitBreakers:
thresholds:
- maxConnections: 1000000000
Expand Down Expand Up @@ -203,6 +218,11 @@ staticResources:
portValue: 9020
name: all-brokers
type: STRICT_DNS
upstreamConnectionOptions:
tcpKeepalive:
keepaliveInterval: 30
keepaliveProbes: 3
keepaliveTime: 30
listeners:
- address:
socketAddress:
Expand All @@ -214,8 +234,22 @@ staticResources:
typedConfig:
'@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
cluster: broker-0
idleTimeout: 560s
maxConnectAttempts: 2
statPrefix: broker_tcp-0
socketOptions:
- intValue: "1"
level: "1"
name: "9"
- intValue: "30"
level: "6"
name: "4"
- intValue: "30"
level: "6"
name: "5"
- intValue: "3"
level: "6"
name: "6"
- address:
socketAddress:
address: 0.0.0.0
Expand All @@ -226,8 +260,22 @@ staticResources:
typedConfig:
'@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
cluster: broker-1
idleTimeout: 560s
maxConnectAttempts: 2
statPrefix: broker_tcp-1
socketOptions:
- intValue: "1"
level: "1"
name: "9"
- intValue: "30"
level: "6"
name: "4"
- intValue: "30"
level: "6"
name: "5"
- intValue: "3"
level: "6"
name: "6"
- address:
socketAddress:
address: 0.0.0.0
Expand All @@ -238,8 +286,22 @@ staticResources:
typedConfig:
'@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
cluster: broker-2
idleTimeout: 560s
maxConnectAttempts: 2
statPrefix: broker_tcp-2
socketOptions:
- intValue: "1"
level: "1"
name: "9"
- intValue: "30"
level: "6"
name: "4"
- intValue: "30"
level: "6"
name: "5"
- intValue: "3"
level: "6"
name: "6"
- address:
socketAddress:
address: 0.0.0.0
Expand All @@ -250,8 +312,22 @@ staticResources:
typedConfig:
'@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
cluster: all-brokers
idleTimeout: 560s
maxConnectAttempts: 2
statPrefix: all-brokers
socketOptions:
- intValue: "1"
level: "1"
name: "9"
- intValue: "30"
level: "6"
name: "4"
- intValue: "30"
level: "6"
name: "5"
- intValue: "3"
level: "6"
name: "6"
- address:
socketAddress:
address: 0.0.0.0
Expand Down Expand Up @@ -289,6 +365,19 @@ staticResources:
redirect:
pathRedirect: /healthcheck
statPrefix: all-brokers-healthcheck
socketOptions:
- intValue: "1"
level: "1"
name: "9"
- intValue: "30"
level: "6"
name: "4"
- intValue: "30"
level: "6"
name: "5"
- intValue: "3"
level: "6"
name: "6"
`, fmt.Sprintf(svcTemplate, "0"), fmt.Sprintf(svcTemplate, "1"), fmt.Sprintf(svcTemplate, "2"), fmt.Sprintf(svcTemplate, "all-broker"))
Expect(configMap.Data["envoy.yaml"]).To(Equal(expected))
}
Expand Down Expand Up @@ -471,6 +560,11 @@ staticResources:
portValue: 9094
name: broker-0
type: STRICT_DNS
upstreamConnectionOptions:
tcpKeepalive:
keepaliveInterval: 30
keepaliveProbes: 3
keepaliveTime: 30
- circuitBreakers:
thresholds:
- maxConnections: 1000000000
Expand Down Expand Up @@ -508,6 +602,11 @@ staticResources:
portValue: 9020
name: all-brokers
type: STRICT_DNS
upstreamConnectionOptions:
tcpKeepalive:
keepaliveInterval: 30
keepaliveProbes: 3
keepaliveTime: 30
listeners:
- address:
socketAddress:
Expand All @@ -519,8 +618,22 @@ staticResources:
typedConfig:
'@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
cluster: broker-0
idleTimeout: 560s
maxConnectAttempts: 2
statPrefix: broker_tcp-0
socketOptions:
- intValue: "1"
level: "1"
name: "9"
- intValue: "30"
level: "6"
name: "4"
- intValue: "30"
level: "6"
name: "5"
- intValue: "3"
level: "6"
name: "6"
- address:
socketAddress:
address: 0.0.0.0
Expand All @@ -531,8 +644,22 @@ staticResources:
typedConfig:
'@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
cluster: all-brokers
idleTimeout: 560s
maxConnectAttempts: 2
statPrefix: all-brokers
socketOptions:
- intValue: "1"
level: "1"
name: "9"
- intValue: "30"
level: "6"
name: "4"
- intValue: "30"
level: "6"
name: "5"
- intValue: "3"
level: "6"
name: "6"
- address:
socketAddress:
address: 0.0.0.0
Expand Down Expand Up @@ -570,6 +697,19 @@ staticResources:
redirect:
pathRedirect: /healthcheck
statPrefix: all-brokers-healthcheck
socketOptions:
- intValue: "1"
level: "1"
name: "9"
- intValue: "30"
level: "6"
name: "4"
- intValue: "30"
level: "6"
name: "5"
- intValue: "3"
level: "6"
name: "6"
`, fmt.Sprintf(svcTemplate, "0"), fmt.Sprintf(svcTemplate, "all-broker"))
Expect(configMap.Data["envoy.yaml"]).To(Equal(expected))
}
Expand Down Expand Up @@ -682,6 +822,11 @@ staticResources:
portValue: 9094
name: broker-1
type: STRICT_DNS
upstreamConnectionOptions:
tcpKeepalive:
keepaliveInterval: 30
keepaliveProbes: 3
keepaliveTime: 30
- circuitBreakers:
thresholds:
- maxConnections: 1000000000
Expand All @@ -705,6 +850,11 @@ staticResources:
portValue: 9094
name: broker-2
type: STRICT_DNS
upstreamConnectionOptions:
tcpKeepalive:
keepaliveInterval: 30
keepaliveProbes: 3
keepaliveTime: 30
- circuitBreakers:
thresholds:
- maxConnections: 1000000000
Expand Down Expand Up @@ -742,6 +892,11 @@ staticResources:
portValue: 9020
name: all-brokers
type: STRICT_DNS
upstreamConnectionOptions:
tcpKeepalive:
keepaliveInterval: 30
keepaliveProbes: 3
keepaliveTime: 30
listeners:
- address:
socketAddress:
Expand All @@ -753,8 +908,22 @@ staticResources:
typedConfig:
'@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
cluster: broker-1
idleTimeout: 560s
maxConnectAttempts: 2
statPrefix: broker_tcp-1
socketOptions:
- intValue: "1"
level: "1"
name: "9"
- intValue: "30"
level: "6"
name: "4"
- intValue: "30"
level: "6"
name: "5"
- intValue: "3"
level: "6"
name: "6"
- address:
socketAddress:
address: 0.0.0.0
Expand All @@ -765,8 +934,22 @@ staticResources:
typedConfig:
'@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
cluster: broker-2
idleTimeout: 560s
maxConnectAttempts: 2
statPrefix: broker_tcp-2
socketOptions:
- intValue: "1"
level: "1"
name: "9"
- intValue: "30"
level: "6"
name: "4"
- intValue: "30"
level: "6"
name: "5"
- intValue: "3"
level: "6"
name: "6"
- address:
socketAddress:
address: 0.0.0.0
Expand All @@ -777,8 +960,22 @@ staticResources:
typedConfig:
'@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
cluster: all-brokers
idleTimeout: 560s
maxConnectAttempts: 2
statPrefix: all-brokers
socketOptions:
- intValue: "1"
level: "1"
name: "9"
- intValue: "30"
level: "6"
name: "4"
- intValue: "30"
level: "6"
name: "5"
- intValue: "3"
level: "6"
name: "6"
- address:
socketAddress:
address: 0.0.0.0
Expand Down Expand Up @@ -816,6 +1013,19 @@ staticResources:
redirect:
pathRedirect: /healthcheck
statPrefix: all-brokers-healthcheck
socketOptions:
- intValue: "1"
level: "1"
name: "9"
- intValue: "30"
level: "6"
name: "4"
- intValue: "30"
level: "6"
name: "5"
- intValue: "3"
level: "6"
name: "6"
`, fmt.Sprintf(svcTemplate, "1"), fmt.Sprintf(svcTemplate, "2"), fmt.Sprintf(svcTemplate, "all-broker"))
Expect(configMap.Data["envoy.yaml"]).To(Equal(expected))
}
Loading

0 comments on commit 507a471

Please sign in to comment.