Use separate Secret for csi-driver-controller

```improvement user An issue causing kube-controller-manager to panic when upgrading an OpenStack cluster from `v1.18` to `v1.19` is now fixed. ``` Signed-off-by: ialidzhikov <[email protected]>
gardener · Sep 15, 2020 · 2f9be52 · 2f9be52
1 parent ae74d2a
commit 2f9be52
Show file tree

Hide file tree

Showing 7 changed files with 59 additions and 34 deletions.
diff --git a/charts/internal/cloud-provider-config/templates/cloud-provider-disk-config-csi.yaml b/charts/internal/cloud-provider-config/templates/cloud-provider-disk-config-csi.yaml
@@ -0,0 +1,19 @@
+{{- define "cloud-provider-disk-config-csi" -}}
+[Global]
+{{ include "cloud-provider-config-credentials" . }}
+{{ include "cloud-provider-config-meta" . }}
+
+[BlockStorage]
+rescan-on-resize={{ .Values.rescanBlockStorageOnResize }}
+{{- end -}}
+{{- if semverCompare ">= 1.19-0" .Values.kubernetesVersion }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cloud-provider-disk-config-csi
+  namespace: {{ .Release.Namespace }}
+type: Opaque
+data:
+  cloudprovider.conf: {{ include "cloud-provider-disk-config-csi" . | b64enc }}
+{{- end }}
diff --git a/charts/internal/cloud-provider-config/templates/cloud-provider-disk-config.yaml b/charts/internal/cloud-provider-config/templates/cloud-provider-disk-config.yaml
@@ -2,10 +2,6 @@
 [Global]
 {{ include "cloud-provider-config-credentials" . }}
 {{ include "cloud-provider-config-meta" . }}
-{{- if semverCompare ">= 1.19-0" .Values.kubernetesVersion }}
-[BlockStorage]
-rescan-on-resize={{ .Values.rescanBlockStorageOnResize }}
-{{- end }}
 {{- end -}}
 ---
 apiVersion: v1

diff --git a/...controlplane/charts/csi-driver-controller/templates/deployment-csi-driver-controller.yaml b/...controlplane/charts/csi-driver-controller/templates/deployment-csi-driver-controller.yaml
@@ -200,7 +200,7 @@ spec:
           path: /usr/share/ca-certificates
       - name: cloud-provider-config
         secret:
-          secretName: cloud-provider-disk-config
+          secretName: cloud-provider-disk-config-csi
       # Host certificates are mounted to accommodate OpenStack endpoints that might be served with a certificate
       # signed by a CA that is not globally trusted.
       - name: etc-ssl

diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go
@@ -327,11 +327,18 @@ func (vp *valuesProvider) GetControlPlaneChartValues(
 	}
 	checksums[openstack.CloudProviderConfigName] = gutil.ComputeChecksum(cpConfigSecret.Data)
 
-	cpDiskConfigSecret := &corev1.Secret{}
-	if err := vp.Client().Get(ctx, kutil.Key(cp.Namespace, openstack.CloudProviderDiskConfigName), cpDiskConfigSecret); err != nil {
+	k8sVersionLessThan119, err := version.CompareVersions(cluster.Shoot.Spec.Kubernetes.Version, "<", "1.19")
+	if err != nil {
 		return nil, err
 	}
-	checksums[openstack.CloudProviderDiskConfigName] = gutil.ComputeChecksum(cpDiskConfigSecret.Data)
+
+	if !k8sVersionLessThan119 {
+		cpDiskConfigSecret := &corev1.Secret{}
+		if err := vp.Client().Get(ctx, kutil.Key(cp.Namespace, openstack.CloudProviderCSIDiskConfigName), cpDiskConfigSecret); err != nil {
+			return nil, err
+		}
+		checksums[openstack.CloudProviderCSIDiskConfigName] = gutil.ComputeChecksum(cpDiskConfigSecret.Data)
+	}
 
 	// TODO: Remove this code in a future version again.
 	if err := vp.deleteLegacyCloudProviderConfigMaps(ctx, cp.Namespace); err != nil {
@@ -355,13 +362,13 @@ func (vp *valuesProvider) GetControlPlaneShootChartValues(
 
 	var cloudProviderDiskConfig []byte
 	if !k8sVersionLessThan119 {
-		cm := &corev1.Secret{}
-		if err := vp.Client().Get(ctx, kutil.Key(cp.Namespace, openstack.CloudProviderDiskConfigName), cm); err != nil {
+		secret := &corev1.Secret{}
+		if err := vp.Client().Get(ctx, kutil.Key(cp.Namespace, openstack.CloudProviderCSIDiskConfigName), secret); err != nil {
 			return nil, err
 		}
 
-		cloudProviderDiskConfig = cm.Data[openstack.CloudProviderConfigDataKey]
-		checksums[openstack.CloudProviderDiskConfigName] = gutil.ComputeChecksum(cm.Data)
+		cloudProviderDiskConfig = secret.Data[openstack.CloudProviderConfigDataKey]
+		checksums[openstack.CloudProviderCSIDiskConfigName] = gutil.ComputeChecksum(secret.Data)
 	}
 
 	return getControlPlaneShootChartValues(cluster, checksums, k8sVersionLessThan119, cloudProviderDiskConfig)
@@ -555,11 +562,11 @@ func getCSIControllerChartValues(
 		"enabled":  true,
 		"replicas": extensionscontroller.GetControlPlaneReplicas(cluster, scaledDown, 1),
 		"podAnnotations": map[string]interface{}{
-			"checksum/secret-" + openstack.CSIProvisionerName:          checksums[openstack.CSIProvisionerName],
-			"checksum/secret-" + openstack.CSIAttacherName:             checksums[openstack.CSIAttacherName],
-			"checksum/secret-" + openstack.CSISnapshotterName:          checksums[openstack.CSISnapshotterName],
-			"checksum/secret-" + openstack.CSIResizerName:              checksums[openstack.CSIResizerName],
-			"checksum/secret-" + openstack.CloudProviderDiskConfigName: checksums[openstack.CloudProviderDiskConfigName],
+			"checksum/secret-" + openstack.CSIProvisionerName:             checksums[openstack.CSIProvisionerName],
+			"checksum/secret-" + openstack.CSIAttacherName:                checksums[openstack.CSIAttacherName],
+			"checksum/secret-" + openstack.CSISnapshotterName:             checksums[openstack.CSISnapshotterName],
+			"checksum/secret-" + openstack.CSIResizerName:                 checksums[openstack.CSIResizerName],
+			"checksum/secret-" + openstack.CloudProviderCSIDiskConfigName: checksums[openstack.CloudProviderCSIDiskConfigName],
 		},
 		"csiSnapshotController": map[string]interface{}{
 			"replicas": extensionscontroller.GetControlPlaneReplicas(cluster, scaledDown, 1),
@@ -583,7 +590,7 @@ func getControlPlaneShootChartValues(
 			"enabled":    !k8sVersionLessThan119,
 			"vpaEnabled": gardencorev1beta1helper.ShootWantsVerticalPodAutoscaler(cluster.Shoot),
 			"podAnnotations": map[string]interface{}{
-				"checksum/secret-" + openstack.CloudProviderDiskConfigName: checksums[openstack.CloudProviderDiskConfigName],
+				"checksum/secret-" + openstack.CloudProviderCSIDiskConfigName: checksums[openstack.CloudProviderCSIDiskConfigName],
 			},
 			"cloudProviderConfig": cloudProviderDiskConfig,
 		},

diff --git a/pkg/controller/controlplane/valuesprovider_test.go b/pkg/controller/controlplane/valuesprovider_test.go
@@ -197,10 +197,10 @@ var _ = Describe("ValuesProvider", func() {
 		}
 
 		cloudProviderDiskConfig = []byte("foo")
-		cpDiskConfigKey         = client.ObjectKey{Namespace: namespace, Name: openstack.CloudProviderDiskConfigName}
-		cpDiskConfig            = &corev1.Secret{
+		cpCSIDiskConfigKey      = client.ObjectKey{Namespace: namespace, Name: openstack.CloudProviderCSIDiskConfigName}
+		cpCSIDiskConfig         = &corev1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
-				Name:      openstack.CloudProviderDiskConfigName,
+				Name:      openstack.CloudProviderCSIDiskConfigName,
 				Namespace: namespace,
 			},
 			Type: corev1.SecretTypeOpaque,
@@ -366,7 +366,6 @@ var _ = Describe("ValuesProvider", func() {
 
 		BeforeEach(func() {
 			c.EXPECT().Get(ctx, cpConfigKey, &corev1.Secret{}).DoAndReturn(clientGet(cpConfig))
-			c.EXPECT().Get(ctx, cpDiskConfigKey, &corev1.Secret{}).DoAndReturn(clientGet(cpDiskConfig))
 			c.EXPECT().Delete(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cloud-provider-config-cloud-controller-manager", Namespace: namespace}})
 			c.EXPECT().Delete(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cloud-provider-config-kube-controller-manager", Namespace: namespace}})
 		})
@@ -383,6 +382,8 @@ var _ = Describe("ValuesProvider", func() {
 		})
 
 		It("should return correct control plane chart values (k8s >= 1.19)", func() {
+			c.EXPECT().Get(ctx, cpCSIDiskConfigKey, &corev1.Secret{}).DoAndReturn(clientGet(cpCSIDiskConfig))
+
 			values, err := vp.GetControlPlaneChartValues(ctx, cp, clusterK8sAtLeast119, checksums, false)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(values).To(Equal(map[string]interface{}{
@@ -392,11 +393,11 @@ var _ = Describe("ValuesProvider", func() {
 				openstack.CSIControllerName: utils.MergeMaps(enabledTrue, map[string]interface{}{
 					"replicas": 1,
 					"podAnnotations": map[string]interface{}{
-						"checksum/secret-" + openstack.CSIProvisionerName:          checksums[openstack.CSIProvisionerName],
-						"checksum/secret-" + openstack.CSIAttacherName:             checksums[openstack.CSIAttacherName],
-						"checksum/secret-" + openstack.CSISnapshotterName:          checksums[openstack.CSISnapshotterName],
-						"checksum/secret-" + openstack.CSIResizerName:              checksums[openstack.CSIResizerName],
-						"checksum/secret-" + openstack.CloudProviderDiskConfigName: checksums[openstack.CloudProviderDiskConfigName],
+						"checksum/secret-" + openstack.CSIProvisionerName:             checksums[openstack.CSIProvisionerName],
+						"checksum/secret-" + openstack.CSIAttacherName:                checksums[openstack.CSIAttacherName],
+						"checksum/secret-" + openstack.CSISnapshotterName:             checksums[openstack.CSISnapshotterName],
+						"checksum/secret-" + openstack.CSIResizerName:                 checksums[openstack.CSIResizerName],
+						"checksum/secret-" + openstack.CloudProviderCSIDiskConfigName: checksums[openstack.CloudProviderCSIDiskConfigName],
 					},
 					"csiSnapshotController": map[string]interface{}{
 						"replicas": 1,
@@ -419,15 +420,15 @@ var _ = Describe("ValuesProvider", func() {
 				openstack.CSINodeName: utils.MergeMaps(enabledFalse, map[string]interface{}{
 					"vpaEnabled": false,
 					"podAnnotations": map[string]interface{}{
-						"checksum/secret-" + openstack.CloudProviderDiskConfigName: "",
+						"checksum/secret-" + openstack.CloudProviderCSIDiskConfigName: "",
 					},
 					"cloudProviderConfig": b,
 				}),
 			}))
 		})
 
 		It("should return correct shoot control plane chart values (k8s >= 1.19)", func() {
-			c.EXPECT().Get(ctx, cpDiskConfigKey, &corev1.Secret{}).DoAndReturn(clientGet(cpDiskConfig))
+			c.EXPECT().Get(ctx, cpCSIDiskConfigKey, &corev1.Secret{}).DoAndReturn(clientGet(cpCSIDiskConfig))
 
 			values, err := vp.GetControlPlaneShootChartValues(ctx, cp, clusterK8sAtLeast119, map[string]string{})
 			Expect(err).NotTo(HaveOccurred())
@@ -436,7 +437,7 @@ var _ = Describe("ValuesProvider", func() {
 				openstack.CSINodeName: utils.MergeMaps(enabledTrue, map[string]interface{}{
 					"vpaEnabled": true,
 					"podAnnotations": map[string]interface{}{
-						"checksum/secret-" + openstack.CloudProviderDiskConfigName: checksums[openstack.CloudProviderDiskConfigName],
+						"checksum/secret-" + openstack.CloudProviderCSIDiskConfigName: checksums[openstack.CloudProviderCSIDiskConfigName],
 					},
 					"cloudProviderConfig": cloudProviderDiskConfig,
 				}),

diff --git a/pkg/openstack/types.go b/pkg/openstack/types.go
@@ -58,11 +58,13 @@ const (
 	// Region is a constant for the key in a backup secret that holds the Openstack region.
 	Region = "region"
 
-	// CloudProviderConfigName is the name of the configmap containing the cloud provider config.
+	// CloudProviderConfigName is the name of the secret containing the cloud provider config.
 	CloudProviderConfigName = "cloud-provider-config"
-	// CloudProviderDiskConfigName is the name of the configmap containing the cloud provider config for disk/volume handling.
+	// CloudProviderDiskConfigName is the name of the secret containing the cloud provider config for disk/volume handling. It is used by kube-controller-manager.
 	CloudProviderDiskConfigName = "cloud-provider-disk-config"
-	// CloudProviderConfigDataKey is the key storing the cloud provider config as value in the cloud provider configmap.
+	// CloudProviderCSIDiskConfigName is the name of the secret containing the cloud provider config for disk/volume handling. It is used by csi-driver-controller.
+	CloudProviderCSIDiskConfigName = "cloud-provider-disk-config-csi"
+	// CloudProviderConfigDataKey is the key storing the cloud provider config as value in the cloud provider secret.
 	CloudProviderConfigDataKey = "cloudprovider.conf"
 	// CloudControllerManagerName is a constant for the name of the CloudController deployed by the worker controller.
 	CloudControllerManagerName = "cloud-controller-manager"

diff --git a/pkg/webhook/controlplane/ensurer.go b/pkg/webhook/controlplane/ensurer.go
@@ -375,7 +375,7 @@ func (e *ensurer) EnsureKubeletCloudProviderConfig(ctx context.Context, ectx gen
 	secret := corev1.Secret{}
 	if err := e.client.Get(ctx, kutil.Key(namespace, openstack.CloudProviderDiskConfigName), &secret); err != nil {
 		if apierrors.IsNotFound(err) {
-			e.logger.Info("configmap not found", "name", openstack.CloudProviderDiskConfigName, "namespace", namespace)
+			e.logger.Info("secret not found", "name", openstack.CloudProviderDiskConfigName, "namespace", namespace)
 			return nil
 		}
 		return errors.Wrapf(err, "could not get secret '%s/%s'", namespace, openstack.CloudProviderDiskConfigName)