Reapply "helm: manage CoreDNS addon as Helm chart (#3236)"

This reverts commit 4a9422d.

bootstrapper/internal/kubernetes/k8sapi/k8sutil.go

@@ -48,7 +48,6 @@ type Client interface {
 	AddNodeSelectorsToDeployment(ctx context.Context, selectors map[string]string, name string, namespace string) error
 	ListAllNamespaces(ctx context.Context) (*corev1.NamespaceList, error)
 	AnnotateNode(ctx context.Context, nodeName, annotationKey, annotationValue string) error
-	EnforceCoreDNSSpread(ctx context.Context) error
 	PatchFirstNodePodCIDR(ctx context.Context, firstNodePodCIDR string) error
 }
 
@@ -150,7 +149,7 @@ func (k *KubernetesUtil) InitCluster(
 
 	// initialize the cluster
 	log.Info("Initializing the cluster using kubeadm init")
-	skipPhases := "--skip-phases=preflight,certs"
+	skipPhases := "--skip-phases=preflight,certs,addon/coredns"
 	if !conformanceMode {
 		skipPhases += ",addon/kube-proxy"
 	}

bootstrapper/internal/kubernetes/kubernetes.go

@@ -165,10 +165,6 @@ func (k *KubeWrapper) InitCluster(
 		return nil, fmt.Errorf("waiting for Kubernetes API to be available: %w", err)
 	}
 
-	if err := k.client.EnforceCoreDNSSpread(ctx); err != nil {
-		return nil, fmt.Errorf("configuring CoreDNS deployment: %w", err)
-	}
-
 	// Setup the K8s components ConfigMap.
 	k8sComponentsConfigMap, err := k.setupK8sComponentsConfigMap(ctx, kubernetesComponents, versionString)
 	if err != nil {

cli/internal/cmd/applyhelm.go

@@ -42,6 +42,7 @@ func (a *applyCmd) runHelmApply(cmd *cobra.Command, conf *config.Config, stateFi
 		HelmWaitMode:        a.flags.helmWaitMode,
 		ApplyTimeout:        a.flags.helmTimeout,
 		AllowDestructive:    helm.DenyDestructive,
+		ServiceCIDR:         conf.ServiceCIDR,
 	}
 	if conf.Provider.OpenStack != nil {
 		var deployYawolLoadBalancer bool

go.mod

@@ -192,6 +192,8 @@ require (
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/containerd/containerd v1.7.12 // indirect
 	github.com/containerd/log v0.1.0 // indirect
+	github.com/coredns/caddy v1.1.1 // indirect
+	github.com/coredns/corefile-migration v1.0.21 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7 // indirect

internal/constellation/helm/BUILD.bazel

@@ -465,6 +465,14 @@ go_library(
         "charts/cert-manager/templates/cainjector-config.yaml",
         "charts/cert-manager/templates/extras-objects.yaml",
         "charts/cert-manager/templates/podmonitor.yaml",
+        "charts/coredns/Chart.yaml",
+        "charts/coredns/values.yaml",
+        "charts/coredns/templates/clusterrole.yaml",
+        "charts/coredns/templates/clusterrolebinding.yaml",
+        "charts/coredns/templates/configmap.yaml",
+        "charts/coredns/templates/deployment.yaml",
+        "charts/coredns/templates/service.yaml",
+        "charts/coredns/templates/serviceaccount.yaml",
     ],
     importpath = "github.com/edgelesssys/constellation/v2/internal/constellation/helm",
     visibility = ["//:__subpackages__"],
@@ -492,6 +500,7 @@ go_library(
         "@io_k8s_client_go//restmapper",
         "@io_k8s_client_go//tools/clientcmd",
         "@io_k8s_client_go//util/retry",
+        "@io_k8s_kubernetes//cmd/kubeadm/app/constants",
         "@sh_helm_helm_v3//pkg/action",
         "@sh_helm_helm_v3//pkg/chart",
         "@sh_helm_helm_v3//pkg/chart/loader",

internal/constellation/helm/charts/coredns/Chart.yaml

@@ -0,0 +1,3 @@
+apiVersion: v2
+name: kube-dns
+version: 0.0.0

internal/constellation/helm/charts/coredns/templates/clusterrole.yaml

@@ -0,0 +1,23 @@
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:coredns
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - services
+  - pods
+  - namespaces
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - list
+  - watch

internal/constellation/helm/charts/coredns/templates/clusterrolebinding.yaml

@@ -0,0 +1,13 @@
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:coredns
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:coredns
+subjects:
+- kind: ServiceAccount
+  name: coredns
+  namespace: kube-system

internal/constellation/helm/charts/coredns/templates/configmap.yaml

@@ -0,0 +1,28 @@
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health {
+           lameduck 5s
+        }
+        ready
+        kubernetes {{ .Values.dnsDomain }} in-addr.arpa ip6.arpa {
+           pods insecure
+           fallthrough in-addr.arpa ip6.arpa
+           ttl 30
+        }
+        prometheus :9153
+        forward . /etc/resolv.conf {
+           max_concurrent 1000
+        }
+        cache 30
+        loop
+        reload
+        loadbalance
+    }

internal/constellation/helm/charts/coredns/templates/deployment.yaml

@@ -0,0 +1,109 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  creationTimestamp: null
+  labels:
+    k8s-app: kube-dns
+  name: coredns
+  namespace: kube-system
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        k8s-app: kube-dns
+    spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: k8s-app
+                  operator: In
+                  values:
+                  - kube-dns
+              topologyKey: kubernetes.io/hostname
+            weight: 100
+      containers:
+      - args:
+        - -conf
+        - /etc/coredns/Corefile
+        image: '{{ .Values.image }}'
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 5
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          successThreshold: 1
+          timeoutSeconds: 5
+        name: coredns
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: 9153
+          name: metrics
+          protocol: TCP
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: 8181
+            scheme: HTTP
+        resources:
+          limits:
+            memory: 170Mi
+          requests:
+            cpu: 100m
+            memory: 70Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /etc/coredns
+          name: config-volume
+          readOnly: true
+      dnsPolicy: Default
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      serviceAccountName: coredns
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
+      - effect: NoSchedule
+        key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+      - effect: NoExecute
+        key: node.kubernetes.io/unreachable
+        operator: Exists
+        tolerationSeconds: 10
+      volumes:
+      - configMap:
+          items:
+          - key: Corefile
+            path: Corefile
+          name: coredns
+        name: config-volume
+status: {}

internal/constellation/helm/charts/coredns/templates/service.yaml

@@ -0,0 +1,33 @@
+
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "CoreDNS"
+  name: kube-dns
+  namespace: kube-system
+  annotations:
+    prometheus.io/port: "9153"
+    prometheus.io/scrape: "true"
+  # Without this resourceVersion value, an update of the Service between versions will yield:
+  #   Service "kube-dns" is invalid: metadata.resourceVersion: Invalid value: "": must be specified for an update
+  resourceVersion: "0"
+spec:
+  clusterIP: "{{ .Values.clusterIP }}"
+  ports:
+  - name: dns
+    port: 53
+    protocol: UDP
+    targetPort: 53
+  - name: dns-tcp
+    port: 53
+    protocol: TCP
+    targetPort: 53
+  - name: metrics
+    port: 9153
+    protocol: TCP
+    targetPort: 9153
+  selector:
+    k8s-app: kube-dns

internal/constellation/helm/charts/coredns/templates/serviceaccount.yaml

@@ -0,0 +1,6 @@
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: coredns
+  namespace: kube-system

internal/constellation/helm/charts/coredns/values.yaml

@@ -0,0 +1,3 @@
+clusterIP: 10.96.0.10
+dnsDomain: cluster.local
+image: registry.k8s.io/coredns/coredns:v1.11.1@sha256:1eeb4c7316bacb1d4c8ead65571cd92dd21e27359f0d4917f1a5822a73b75db1

internal/constellation/helm/corednsgen/BUILD.bazel

@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "corednsgen_lib",
+    srcs = ["corednsgen.go"],
+    importpath = "github.com/edgelesssys/constellation/v2/internal/constellation/helm/corednsgen",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//internal/versions",
+        "@com_github_regclient_regclient//:regclient",
+        "@com_github_regclient_regclient//types/ref",
+        "@io_k8s_api//apps/v1:apps",
+        "@io_k8s_api//core/v1:core",
+        "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm",
+        "@io_k8s_kubernetes//cmd/kubeadm/app/images",
+        "@io_k8s_kubernetes//cmd/kubeadm/app/phases/addons/dns",
+        "@io_k8s_kubernetes//cmd/kubeadm/app/util",
+        "@io_k8s_sigs_yaml//:yaml",
+    ],
+)
+
+go_binary(
+    name = "corednsgen",
+    embed = [":corednsgen_lib"],
+    visibility = ["//:__subpackages__"],
+)