diff --git a/api/v1beta1/kafkacluster_types.go b/api/v1beta1/kafkacluster_types.go index 7db9f8c4e..fb108d25d 100644 --- a/api/v1beta1/kafkacluster_types.go +++ b/api/v1beta1/kafkacluster_types.go @@ -242,12 +242,16 @@ type EnvoyConfig struct { // +kubebuilder:validation:Minimum=1 Concurrency int32 `json:"concurrency,omitempty"` // +kubebuilder:validation:Minimum=1 - Replicas int32 `json:"replicas,omitempty"` - ServiceAccountName string `json:"serviceAccountName,omitempty"` - ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` - Affinity *corev1.Affinity `json:"affinity,omitempty"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + Replicas int32 `json:"replicas,omitempty"` + // ServiceAccountName the name of service account + ServiceAccountName string `json:"serviceAccountName,omitempty"` + // ImagePullSecrets image pull secrets + ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` + // NodeSelector node selector for envoy pods + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` // Annotations defines the annotations placed on the envoy ingress controller deployment Annotations map[string]string `json:"annotations,omitempty"` LoadBalancerSourceRanges []string `json:"loadBalancerSourceRanges,omitempty"` @@ -663,6 +667,11 @@ func (eConfig *EnvoyConfig) GetAffinity() *corev1.Affinity { return eConfig.Affinity } +//GetTopologySpreadConstaints returns the Affinity config for envoy +func (eConfig *EnvoyConfig) GetTopologySpreadConstaints() []corev1.TopologySpreadConstraint { + return eConfig.TopologySpreadConstraints +} + //GetNodeSelector returns the node selector for the given broker func (bConfig *BrokerConfig) GetNodeSelector() map[string]string { return bConfig.NodeSelector diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index b4481a1e6..cd195de86 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -385,6 +385,13 @@ func (in *EnvoyConfig) DeepCopyInto(out *EnvoyConfig) { *out = new(v1.Affinity) (*in).DeepCopyInto(*out) } + if in.TopologySpreadConstraints != nil { + in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints + *out = make([]v1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = make(map[string]string, len(*in)) diff --git a/charts/kafka-operator/templates/crds.yaml b/charts/kafka-operator/templates/crds.yaml index 2bfdc2ba3..2c0a2f5e5 100644 --- a/charts/kafka-operator/templates/crds.yaml +++ b/charts/kafka-operator/templates/crds.yaml @@ -15812,6 +15812,7 @@ spec: image: type: string imagePullSecrets: + description: ImagePullSecrets image pull secrets items: description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. @@ -15833,6 +15834,7 @@ spec: nodeSelector: additionalProperties: type: string + description: NodeSelector node selector for envoy pods type: object replicas: format: int32 @@ -15866,6 +15868,7 @@ spec: type: object type: object serviceAccountName: + description: ServiceAccountName the name of service account type: string tolerations: items: @@ -15907,6 +15910,108 @@ spec: type: string type: object type: array + topologySpreadConstraints: + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + maxSkew: + description: 'MaxSkew describes the degree to which pods + may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global + minimum. For example, in a 3-zone cluster, MaxSkew is + set to 1, and pods with the same labelSelector spread + as 1/1/0: | zone1 | zone2 | zone3 | | P | P | | + - if MaxSkew is 1, incoming pod can only be scheduled + to zone3 to become 1/1/1; scheduling it onto zone1(zone2) + would make the ActualSkew(2-0) on zone1(zone2) violate + MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled + onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that + satisfy it. It''s a required field. Default value is 1 + and 0 is not allowed.' + format: int32 + type: integer + topologyKey: + description: TopologyKey is the key of node labels. Nodes + that have a label with this key and identical values are + considered to be in the same topology. We consider each + as a "bucket", and try to put balanced number + of pods into each bucket. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal with + a pod if it doesn''t satisfy the spread constraint. - + DoNotSchedule (default) tells the scheduler not to schedule + it. - ScheduleAnyway tells the scheduler to schedule the + pod in any location, but giving higher precedence to + topologies that would help reduce the skew. A constraint + is considered "Unsatisfiable" for an incoming pod if and + only if every possible node assigment for that pod would + violate "MaxSkew" on some topology. For example, in a + 3-zone cluster, MaxSkew is set to 1, and pods with the + same labelSelector spread as 3/1/1: | zone1 | zone2 | + zone3 | | P P P | P | P | If WhenUnsatisfiable + is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) + on zone2(zone3) satisfies MaxSkew(1). In other words, + the cluster can still be imbalanced, but scheduler won''t + make it *more* imbalanced. It''s a required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array type: object envs: description: Envs defines environment variables for Kafka broker Pods. @@ -17633,6 +17738,7 @@ spec: image: type: string imagePullSecrets: + description: ImagePullSecrets image pull secrets items: description: LocalObjectReference contains enough information to let you locate the @@ -17658,6 +17764,8 @@ spec: nodeSelector: additionalProperties: type: string + description: NodeSelector node selector for + envoy pods type: object replicas: format: int32 @@ -17694,6 +17802,8 @@ spec: type: object type: object serviceAccountName: + description: ServiceAccountName the name of + service account type: string tolerations: items: @@ -17746,6 +17856,144 @@ spec: type: string type: object type: array + topologySpreadConstraints: + items: + description: TopologySpreadConstraint specifies + how to spread matching pods among the + given topology. + properties: + labelSelector: + description: LabelSelector is used to + find matching pods. Pods that match + this label selector are counted to + determine the number of pods in their + corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector + requirement is a selector that + contains values, a key, and + an operator that relates the + key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to + a set of values. Valid operators + are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an + array of string values. + If the operator is In or + NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be + empty. This array is replaced + during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. A single + {key,value} in the matchLabels + map is equivalent to an element + of matchExpressions, whose key + field is "key", the operator is + "In", and the values array contains + only "value". The requirements + are ANDed. + type: object + type: object + maxSkew: + description: 'MaxSkew describes the + degree to which pods may be unevenly + distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference + between the number of matching pods + in the target topology and the global + minimum. For example, in a 3-zone + cluster, MaxSkew is set to 1, and + pods with the same labelSelector spread + as 1/1/0: | zone1 | zone2 | zone3 + | | P | P | | - if MaxSkew + is 1, incoming pod can only be scheduled + to zone3 to become 1/1/1; scheduling + it onto zone1(zone2) would make the + ActualSkew(2-0) on zone1(zone2) violate + MaxSkew(1). - if MaxSkew is 2, incoming + pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence + to topologies that satisfy it. It''s + a required field. Default value is + 1 and 0 is not allowed.' + format: int32 + type: integer + topologyKey: + description: TopologyKey is the key + of node labels. Nodes that have a + label with this key and identical + values are considered to be in the + same topology. We consider each as a "bucket", and try to put + balanced number of pods into each + bucket. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates + how to deal with a pod if it doesn''t + satisfy the spread constraint. - DoNotSchedule + (default) tells the scheduler not + to schedule it. - ScheduleAnyway tells + the scheduler to schedule the pod + in any location, but giving higher + precedence to topologies that would + help reduce the skew. A constraint + is considered "Unsatisfiable" for + an incoming pod if and only if every + possible node assigment for that pod + would violate "MaxSkew" on some topology. + For example, in a 3-zone cluster, + MaxSkew is set to 1, and pods with + the same labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | | P P P + | P | P | If WhenUnsatisfiable + is set to DoNotSchedule, incoming + pod can only be scheduled to zone2(zone3) + to become 3/2/1(3/1/2) as ActualSkew(2-1) + on zone2(zone3) satisfies MaxSkew(1). + In other words, the cluster can still + be imbalanced, but scheduler won''t + make it *more* imbalanced. It''s a + required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array type: object externalTrafficPolicy: description: externalTrafficPolicy denotes if diff --git a/config/base/crds/kafka.banzaicloud.io_kafkaclusters.yaml b/config/base/crds/kafka.banzaicloud.io_kafkaclusters.yaml index b834144fd..0b48c02b3 100644 --- a/config/base/crds/kafka.banzaicloud.io_kafkaclusters.yaml +++ b/config/base/crds/kafka.banzaicloud.io_kafkaclusters.yaml @@ -15811,6 +15811,7 @@ spec: image: type: string imagePullSecrets: + description: ImagePullSecrets image pull secrets items: description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. @@ -15832,6 +15833,7 @@ spec: nodeSelector: additionalProperties: type: string + description: NodeSelector node selector for envoy pods type: object replicas: format: int32 @@ -15865,6 +15867,7 @@ spec: type: object type: object serviceAccountName: + description: ServiceAccountName the name of service account type: string tolerations: items: @@ -15906,6 +15909,108 @@ spec: type: string type: object type: array + topologySpreadConstraints: + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + maxSkew: + description: 'MaxSkew describes the degree to which pods + may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global + minimum. For example, in a 3-zone cluster, MaxSkew is + set to 1, and pods with the same labelSelector spread + as 1/1/0: | zone1 | zone2 | zone3 | | P | P | | + - if MaxSkew is 1, incoming pod can only be scheduled + to zone3 to become 1/1/1; scheduling it onto zone1(zone2) + would make the ActualSkew(2-0) on zone1(zone2) violate + MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled + onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that + satisfy it. It''s a required field. Default value is 1 + and 0 is not allowed.' + format: int32 + type: integer + topologyKey: + description: TopologyKey is the key of node labels. Nodes + that have a label with this key and identical values are + considered to be in the same topology. We consider each + as a "bucket", and try to put balanced number + of pods into each bucket. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal with + a pod if it doesn''t satisfy the spread constraint. - + DoNotSchedule (default) tells the scheduler not to schedule + it. - ScheduleAnyway tells the scheduler to schedule the + pod in any location, but giving higher precedence to + topologies that would help reduce the skew. A constraint + is considered "Unsatisfiable" for an incoming pod if and + only if every possible node assigment for that pod would + violate "MaxSkew" on some topology. For example, in a + 3-zone cluster, MaxSkew is set to 1, and pods with the + same labelSelector spread as 3/1/1: | zone1 | zone2 | + zone3 | | P P P | P | P | If WhenUnsatisfiable + is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) + on zone2(zone3) satisfies MaxSkew(1). In other words, + the cluster can still be imbalanced, but scheduler won''t + make it *more* imbalanced. It''s a required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array type: object envs: description: Envs defines environment variables for Kafka broker Pods. @@ -17632,6 +17737,7 @@ spec: image: type: string imagePullSecrets: + description: ImagePullSecrets image pull secrets items: description: LocalObjectReference contains enough information to let you locate the @@ -17657,6 +17763,8 @@ spec: nodeSelector: additionalProperties: type: string + description: NodeSelector node selector for + envoy pods type: object replicas: format: int32 @@ -17693,6 +17801,8 @@ spec: type: object type: object serviceAccountName: + description: ServiceAccountName the name of + service account type: string tolerations: items: @@ -17745,6 +17855,144 @@ spec: type: string type: object type: array + topologySpreadConstraints: + items: + description: TopologySpreadConstraint specifies + how to spread matching pods among the + given topology. + properties: + labelSelector: + description: LabelSelector is used to + find matching pods. Pods that match + this label selector are counted to + determine the number of pods in their + corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector + requirement is a selector that + contains values, a key, and + an operator that relates the + key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to + a set of values. Valid operators + are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an + array of string values. + If the operator is In or + NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be + empty. This array is replaced + during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. A single + {key,value} in the matchLabels + map is equivalent to an element + of matchExpressions, whose key + field is "key", the operator is + "In", and the values array contains + only "value". The requirements + are ANDed. + type: object + type: object + maxSkew: + description: 'MaxSkew describes the + degree to which pods may be unevenly + distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference + between the number of matching pods + in the target topology and the global + minimum. For example, in a 3-zone + cluster, MaxSkew is set to 1, and + pods with the same labelSelector spread + as 1/1/0: | zone1 | zone2 | zone3 + | | P | P | | - if MaxSkew + is 1, incoming pod can only be scheduled + to zone3 to become 1/1/1; scheduling + it onto zone1(zone2) would make the + ActualSkew(2-0) on zone1(zone2) violate + MaxSkew(1). - if MaxSkew is 2, incoming + pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence + to topologies that satisfy it. It''s + a required field. Default value is + 1 and 0 is not allowed.' + format: int32 + type: integer + topologyKey: + description: TopologyKey is the key + of node labels. Nodes that have a + label with this key and identical + values are considered to be in the + same topology. We consider each as a "bucket", and try to put + balanced number of pods into each + bucket. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates + how to deal with a pod if it doesn''t + satisfy the spread constraint. - DoNotSchedule + (default) tells the scheduler not + to schedule it. - ScheduleAnyway tells + the scheduler to schedule the pod + in any location, but giving higher + precedence to topologies that would + help reduce the skew. A constraint + is considered "Unsatisfiable" for + an incoming pod if and only if every + possible node assigment for that pod + would violate "MaxSkew" on some topology. + For example, in a 3-zone cluster, + MaxSkew is set to 1, and pods with + the same labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | | P P P + | P | P | If WhenUnsatisfiable + is set to DoNotSchedule, incoming + pod can only be scheduled to zone2(zone3) + to become 3/2/1(3/1/2) as ActualSkew(2-1) + on zone2(zone3) satisfies MaxSkew(1). + In other words, the cluster can still + be imbalanced, but scheduler won''t + make it *more* imbalanced. It''s a + required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array type: object externalTrafficPolicy: description: externalTrafficPolicy denotes if diff --git a/pkg/resources/envoy/deployment.go b/pkg/resources/envoy/deployment.go index fea84d1d2..4519fa75b 100644 --- a/pkg/resources/envoy/deployment.go +++ b/pkg/resources/envoy/deployment.go @@ -89,11 +89,12 @@ func (r *Reconciler) deployment(log logr.Logger, extListener v1beta1.ExternalLis defaultIngressConfigName, log), }, Spec: corev1.PodSpec{ - ServiceAccountName: ingressConfig.EnvoyConfig.GetServiceAccount(), - ImagePullSecrets: ingressConfig.EnvoyConfig.GetImagePullSecrets(), - Tolerations: ingressConfig.EnvoyConfig.GetTolerations(), - NodeSelector: ingressConfig.EnvoyConfig.GetNodeSelector(), - Affinity: ingressConfig.EnvoyConfig.GetAffinity(), + ServiceAccountName: ingressConfig.EnvoyConfig.GetServiceAccount(), + ImagePullSecrets: ingressConfig.EnvoyConfig.GetImagePullSecrets(), + Tolerations: ingressConfig.EnvoyConfig.GetTolerations(), + NodeSelector: ingressConfig.EnvoyConfig.GetNodeSelector(), + Affinity: ingressConfig.EnvoyConfig.GetAffinity(), + TopologySpreadConstraints: ingressConfig.EnvoyConfig.GetTopologySpreadConstaints(), Containers: []corev1.Container{ { Name: "envoy",