From 3a0c4ce30f4ad7c402a17e79d09d890da0d8a57d Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Wed, 16 Aug 2023 14:05:49 +0300 Subject: [PATCH] Use kustomize for deployment supported overlays: - no-webhook - deploy without webhook - certmanager - deploy with webhook to k8s cluster where certmanager is available - openshift - deploy with webhook to the Openshift cluster Signed-off-by: Yury Kulazhenkov --- README.md | 75 +++-- deploy/crds/kustomization.yaml | 2 +- deploy/manifests/certmanager/certificate.yaml | 25 ++ .../manifests/certmanager/kustomization.yaml | 5 + .../certmanager/kustomizeconfig.yaml | 16 + deploy/manifests/controller/deployment.yaml | 99 ++++++ .../manifests/controller/kustomization.yaml | 5 + deploy/manifests/controller/role.yaml | 60 ++++ deploy/manifests/controller/role_binding.yaml | 12 + .../manifests/controller/service_account.yaml | 5 + deploy/manifests/node/daemonset.yaml | 90 ++++++ deploy/manifests/node/kustomization.yaml | 5 + deploy/manifests/node/role.yaml | 21 ++ deploy/manifests/node/role_binding.yaml | 12 + deploy/manifests/node/service_account.yaml | 5 + .../openshift-privileged/kustomization.yaml | 3 + .../manifests/openshift-privileged/role.yaml | 14 + .../openshift-privileged/role_binding.yaml | 13 + deploy/manifests/webhook/kustomization.yaml | 6 + deploy/manifests/webhook/kustomizeconfig.yaml | 25 ++ deploy/manifests/webhook/manifests.yaml | 25 ++ deploy/manifests/webhook/service.yaml | 11 + deploy/nv-ipam.yaml | 301 ------------------ .../certmanager/controller_webhook_patch.yaml | 23 ++ .../overlays/certmanager/kustomization.yaml | 59 ++++ .../certmanager/webhook_config_patch.yaml | 6 + deploy/overlays/no-webhook/kustomization.yaml | 19 ++ .../node_openshift_cni_path_patch.yaml | 13 + .../openshilft/controller_webhook_patch.yaml | 23 ++ deploy/overlays/openshilft/kustomization.yaml | 33 ++ .../node_openshift_cni_path_patch.yaml | 13 + .../openshilft/webhook_config_patch.yaml | 6 + .../openshilft/webhook_service_patch.yaml | 7 + 33 files changed, 712 insertions(+), 325 deletions(-) create mode 100644 deploy/manifests/certmanager/certificate.yaml create mode 100644 deploy/manifests/certmanager/kustomization.yaml create mode 100644 deploy/manifests/certmanager/kustomizeconfig.yaml create mode 100644 deploy/manifests/controller/deployment.yaml create mode 100644 deploy/manifests/controller/kustomization.yaml create mode 100644 deploy/manifests/controller/role.yaml create mode 100644 deploy/manifests/controller/role_binding.yaml create mode 100644 deploy/manifests/controller/service_account.yaml create mode 100644 deploy/manifests/node/daemonset.yaml create mode 100644 deploy/manifests/node/kustomization.yaml create mode 100644 deploy/manifests/node/role.yaml create mode 100644 deploy/manifests/node/role_binding.yaml create mode 100644 deploy/manifests/node/service_account.yaml create mode 100644 deploy/manifests/openshift-privileged/kustomization.yaml create mode 100644 deploy/manifests/openshift-privileged/role.yaml create mode 100644 deploy/manifests/openshift-privileged/role_binding.yaml create mode 100644 deploy/manifests/webhook/kustomization.yaml create mode 100644 deploy/manifests/webhook/kustomizeconfig.yaml create mode 100644 deploy/manifests/webhook/manifests.yaml create mode 100644 deploy/manifests/webhook/service.yaml delete mode 100644 deploy/nv-ipam.yaml create mode 100644 deploy/overlays/certmanager/controller_webhook_patch.yaml create mode 100644 deploy/overlays/certmanager/kustomization.yaml create mode 100644 deploy/overlays/certmanager/webhook_config_patch.yaml create mode 100644 deploy/overlays/no-webhook/kustomization.yaml create mode 100644 deploy/overlays/no-webhook/node_openshift_cni_path_patch.yaml create mode 100644 deploy/overlays/openshilft/controller_webhook_patch.yaml create mode 100644 deploy/overlays/openshilft/kustomization.yaml create mode 100644 deploy/overlays/openshilft/node_openshift_cni_path_patch.yaml create mode 100644 deploy/overlays/openshilft/webhook_config_patch.yaml create mode 100644 deploy/overlays/openshilft/webhook_service_patch.yaml diff --git a/README.md b/README.md index 0b0da21..f047f67 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,16 @@ NVIDIA IPAM plugin consists of 3 main components: A Kubernetes(K8s) controller that Watches on IPPools CRs in a predefined Namespace. It then proceeds by assiging each node via IPPools Status a cluster unique range of IPs of the defined IP Pools. +#### Validation webhook + +ipam-controller implements validation webhook for IPPool resource. +The webhook can prevent the creation of IPPool resources with invalid configurations. +Supported X.509 certificate management system should be available in the cluster to enable the webhook. +Currently supported systems are [certmanager](https://cert-manager.io/) and +[Openshift certificate management](https://docs.openshift.com/container-platform/4.13/security/certificates/service-serving-certificate.html) + +Activation of the validation webhook is optional. Check the [Deployment](#deployment) section for details. + ### ipam-node The daemon is responsible for: @@ -144,48 +154,50 @@ ipam-controller accepts configuration using command line flags and IPPools CRs. ```text Logging flags: - --log-flush-frequency duration + --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) - --log-json-info-buffer-size quantity - [Alpha] In JSON format with split output streams, the info messages can be buffered for a while to increase performance. The default value of zero bytes disables buffering. The - size can be specified as number of bytes (512), multiples of 1000 (1K), multiples of 1024 (2Ki), or powers of those (3M, 4G, 5Mi, 6Gi). Enable the LoggingAlphaOptions feature - gate to use this. - --log-json-split-stream - [Alpha] In JSON format, write error messages to stderr and info messages to stdout. The default is to write a single stream to stdout. Enable the LoggingAlphaOptions feature gate - to use this. - --logging-format string + --log-json-info-buffer-size quantity + [Alpha] In JSON format with split output streams, the info messages can be buffered for a while to increase performance. The default value of zero bytes disables buffering. The size can + be specified as number of bytes (512), multiples of 1000 (1K), multiples of 1024 (2Ki), or powers of those (3M, 4G, 5Mi, 6Gi). Enable the LoggingAlphaOptions feature gate to use this. + --log-json-split-stream + [Alpha] In JSON format, write error messages to stderr and info messages to stdout. The default is to write a single stream to stdout. Enable the LoggingAlphaOptions feature gate to use + this. + --logging-format string Sets the log format. Permitted formats: "json" (gated by LoggingBetaOptions), "text". (default "text") - -v, --v Level + -v, --v Level number for the log level verbosity - --vmodule pattern=N,... + --vmodule pattern=N,... comma-separated list of pattern=N settings for file-filtered logging (only works for text log format) Common flags: - --feature-gates mapStringBool + --feature-gates mapStringBool A set of key=value pairs that describe feature gates for alpha/experimental features. Options are: AllAlpha=true|false (ALPHA - default=false) AllBeta=true|false (BETA - default=false) ContextualLogging=true|false (ALPHA - default=false) LoggingAlphaOptions=true|false (ALPHA - default=false) LoggingBetaOptions=true|false (BETA - default=true) - --version + --version print binary version and exit Controller flags: - --health-probe-bind-address string + --health-probe-bind-address string The address the probe endpoint binds to. (default ":8081") - --kubeconfig string + --ippools-namespace string + The name of the namespace to watch for IPPools CRs (default "kube-system") + --kubeconfig string Paths to a kubeconfig. Only required if out-of-cluster. - --leader-elect + --leader-elect Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager. - --leader-elect-namespace string + --leader-elect-namespace string Determines the namespace in which the leader election resource will be created. (default "kube-system") - --metrics-bind-address string + --metrics-bind-address string The address the metric endpoint binds to. (default ":8080") - --ippools-namespace string - The name of the namespace to watch for IPPools CRs. (default "kube-system") + --webhook + Enable validating webhook server as a part of the controller + ``` #### IPPool CR @@ -331,11 +343,28 @@ interface should have two IP addresses: one IPv4 and one IPv6. (default: network ### Deploy IPAM plugin -> _NOTE:_ This command will deploy latest dev build with default configuration +> _NOTE:_ These commands will deploy latest dev build with default configuration + +The plugin can be deployed with kustomize. + +Supported overlays are: + +`no-webhook` - deploy without webhook + +```shell +kubectl kustomize https://github.com/mellanox/nvidia-k8s-ipam/deploy/overlays/no-webhook?ref=main | kubectl apply -f - +``` + +`certmanager` - deploy with webhook to the Kubernetes cluster where certmanager is available + +```shell +kubectl kustomize https://github.com/mellanox/nvidia-k8s-ipam/deploy/overlays/certmanager?ref=main | kubectl apply -f - +``` + +`openshift` - deploy with webhook to the Openshift cluster ```shell -kubectl apply -f https://raw.githubusercontent.com/Mellanox/nvidia-k8s-ipam/main/deploy/crds/nv-ipam.nvidia.com_ippools.yaml -kubectl apply -f https://raw.githubusercontent.com/Mellanox/nvidia-k8s-ipam/main/deploy/nv-ipam.yaml +kubectl kustomize https://github.com/mellanox/nvidia-k8s-ipam/deploy/overlays/openshift?ref=main | kubectl apply -f - ``` ### Create IPPool CR diff --git a/deploy/crds/kustomization.yaml b/deploy/crds/kustomization.yaml index 764ffa4..a9c6a2f 100644 --- a/deploy/crds/kustomization.yaml +++ b/deploy/crds/kustomization.yaml @@ -1,2 +1,2 @@ resources: - - nv-ipam.nvidia.com_ippools.yaml \ No newline at end of file + - nv-ipam.nvidia.com_ippools.yaml diff --git a/deploy/manifests/certmanager/certificate.yaml b/deploy/manifests/certmanager/certificate.yaml new file mode 100644 index 0000000..91713a2 --- /dev/null +++ b/deploy/manifests/certmanager/certificate.yaml @@ -0,0 +1,25 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize + dnsNames: + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: nv-ipam-webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/deploy/manifests/certmanager/kustomization.yaml b/deploy/manifests/certmanager/kustomization.yaml new file mode 100644 index 0000000..bebea5a --- /dev/null +++ b/deploy/manifests/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/deploy/manifests/certmanager/kustomizeconfig.yaml b/deploy/manifests/certmanager/kustomizeconfig.yaml new file mode 100644 index 0000000..e631f77 --- /dev/null +++ b/deploy/manifests/certmanager/kustomizeconfig.yaml @@ -0,0 +1,16 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name + +varReference: +- kind: Certificate + group: cert-manager.io + path: spec/commonName +- kind: Certificate + group: cert-manager.io + path: spec/dnsNames diff --git a/deploy/manifests/controller/deployment.yaml b/deploy/manifests/controller/deployment.yaml new file mode 100644 index 0000000..bc86924 --- /dev/null +++ b/deploy/manifests/controller/deployment.yaml @@ -0,0 +1,99 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: controller + namespace: system + annotations: + kubernetes.io/description: | + This deployment launches the nv-ipam controller for nv-ipam. +spec: + strategy: + type: RollingUpdate + replicas: 1 + selector: + matchLabels: + name: controller + template: + metadata: + labels: + name: controller + spec: + priorityClassName: system-cluster-critical + serviceAccountName: controller + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: name + operator: In + values: + - controller + topologyKey: "kubernetes.io/hostname" + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" + - weight: 1 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "" + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + containers: + - name: controller + image: ghcr.io/mellanox/nvidia-k8s-ipam:latest + imagePullPolicy: IfNotPresent + command: [ "/ipam-controller" ] + args: + - --leader-elect=true + - --leader-elect-namespace=$(POD_NAMESPACE) + - --ippools-namespace=$(POD_NAMESPACE) + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 300Mi + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP diff --git a/deploy/manifests/controller/kustomization.yaml b/deploy/manifests/controller/kustomization.yaml new file mode 100644 index 0000000..3246145 --- /dev/null +++ b/deploy/manifests/controller/kustomization.yaml @@ -0,0 +1,5 @@ +resources: + - deployment.yaml + - role.yaml + - role_binding.yaml + - service_account.yaml diff --git a/deploy/manifests/controller/role.yaml b/deploy/manifests/controller/role.yaml new file mode 100644 index 0000000..13a9b6b --- /dev/null +++ b/deploy/manifests/controller/role.yaml @@ -0,0 +1,60 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: controller +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - delete + - apiGroups: + - nv-ipam.nvidia.com + resources: + - ippools + verbs: + - get + - list + - watch + - create + - apiGroups: + - nv-ipam.nvidia.com + resources: + - ippools/status + verbs: + - get + - update + - patch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/deploy/manifests/controller/role_binding.yaml b/deploy/manifests/controller/role_binding.yaml new file mode 100644 index 0000000..fcd3a36 --- /dev/null +++ b/deploy/manifests/controller/role_binding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: controller +subjects: + - kind: ServiceAccount + name: controller + namespace: system diff --git a/deploy/manifests/controller/service_account.yaml b/deploy/manifests/controller/service_account.yaml new file mode 100644 index 0000000..130eeec --- /dev/null +++ b/deploy/manifests/controller/service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: controller + namespace: system diff --git a/deploy/manifests/node/daemonset.yaml b/deploy/manifests/node/daemonset.yaml new file mode 100644 index 0000000..7d4cd07 --- /dev/null +++ b/deploy/manifests/node/daemonset.yaml @@ -0,0 +1,90 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-ds + namespace: system + labels: + tier: node + app: nv-ipam-node + name: nv-ipam-node +spec: + selector: + matchLabels: + name: nv-ipam-node + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + tier: node + app: nv-ipam-node + name: nv-ipam-node + spec: + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + serviceAccountName: node + containers: + - name: node + image: ghcr.io/mellanox/nvidia-k8s-ipam:latest + imagePullPolicy: IfNotPresent + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: true + command: [ "/ipam-node" ] + args: + - --node-name=$(NODE_NAME) + - --v=1 # log level for ipam-node + - --logging-format=json + - --bind-address=unix:///var/lib/cni/nv-ipam/daemon.sock + - --store-file=/var/lib/cni/nv-ipam/store + - --cni-daemon-socket=unix:///var/lib/cni/nv-ipam/daemon.sock + - --cni-daemon-call-timeout=5 # 5 seconds + - --cni-bin-dir=/opt/cni/bin + - --cni-conf-dir=/etc/cni/net.d/nv-ipam.d + - --cni-log-file=/var/log/nv-ipam-cni.log + - --cni-log-level=info # log level for shim CNI + - --ippools-namespace=$(POD_NAMESPACE) + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "300m" + memory: "300Mi" + volumeMounts: + - name: cnibin + mountPath: /opt/cni/bin + - name: cniconf + mountPath: /etc/cni/net.d/nv-ipam.d + - name: daemonstate + mountPath: /var/lib/cni/nv-ipam/ + terminationGracePeriodSeconds: 10 + volumes: + - name: cnibin + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + - name: cniconf + hostPath: + path: /etc/cni/net.d/nv-ipam.d + type: DirectoryOrCreate + - name: daemonstate + hostPath: + path: /var/lib/cni/nv-ipam/ + type: DirectoryOrCreate diff --git a/deploy/manifests/node/kustomization.yaml b/deploy/manifests/node/kustomization.yaml new file mode 100644 index 0000000..799ffbc --- /dev/null +++ b/deploy/manifests/node/kustomization.yaml @@ -0,0 +1,5 @@ +resources: + - daemonset.yaml + - role.yaml + - role_binding.yaml + - service_account.yaml diff --git a/deploy/manifests/node/role.yaml b/deploy/manifests/node/role.yaml new file mode 100644 index 0000000..58519d2 --- /dev/null +++ b/deploy/manifests/node/role.yaml @@ -0,0 +1,21 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: node +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: + - nv-ipam.nvidia.com + resources: + - ippools + verbs: + - get + - list + - watch diff --git a/deploy/manifests/node/role_binding.yaml b/deploy/manifests/node/role_binding.yaml new file mode 100644 index 0000000..c3db13c --- /dev/null +++ b/deploy/manifests/node/role_binding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: node +subjects: + - kind: ServiceAccount + name: node + namespace: system diff --git a/deploy/manifests/node/service_account.yaml b/deploy/manifests/node/service_account.yaml new file mode 100644 index 0000000..d55c1d4 --- /dev/null +++ b/deploy/manifests/node/service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node + namespace: system diff --git a/deploy/manifests/openshift-privileged/kustomization.yaml b/deploy/manifests/openshift-privileged/kustomization.yaml new file mode 100644 index 0000000..1af844a --- /dev/null +++ b/deploy/manifests/openshift-privileged/kustomization.yaml @@ -0,0 +1,3 @@ +resources: + - role_binding.yaml + - role.yaml diff --git a/deploy/manifests/openshift-privileged/role.yaml b/deploy/manifests/openshift-privileged/role.yaml new file mode 100644 index 0000000..b9cc539 --- /dev/null +++ b/deploy/manifests/openshift-privileged/role.yaml @@ -0,0 +1,14 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: node + namespace: system +rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - use + resourceNames: + - privileged diff --git a/deploy/manifests/openshift-privileged/role_binding.yaml b/deploy/manifests/openshift-privileged/role_binding.yaml new file mode 100644 index 0000000..cf60f70 --- /dev/null +++ b/deploy/manifests/openshift-privileged/role_binding.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: node + namespace: system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: node +subjects: + - kind: ServiceAccount + name: node + namespace: system diff --git a/deploy/manifests/webhook/kustomization.yaml b/deploy/manifests/webhook/kustomization.yaml new file mode 100644 index 0000000..9cf2613 --- /dev/null +++ b/deploy/manifests/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/deploy/manifests/webhook/kustomizeconfig.yaml b/deploy/manifests/webhook/kustomizeconfig.yaml new file mode 100644 index 0000000..25e21e3 --- /dev/null +++ b/deploy/manifests/webhook/kustomizeconfig.yaml @@ -0,0 +1,25 @@ +# the following config is for teaching kustomize where to look at when substituting vars. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true + +varReference: +- path: metadata/annotations diff --git a/deploy/manifests/webhook/manifests.yaml b/deploy/manifests/webhook/manifests.yaml new file mode 100644 index 0000000..d06f2b7 --- /dev/null +++ b/deploy/manifests/webhook/manifests.yaml @@ -0,0 +1,25 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration +webhooks: + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-nv-ipam-nvidia-com-v1alpha1-ippool + failurePolicy: Fail + name: validate-ippool.nv-ipam.nvidia.com + rules: + - apiGroups: + - nv-ipam.nvidia.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - ippools + sideEffects: None diff --git a/deploy/manifests/webhook/service.yaml b/deploy/manifests/webhook/service.yaml new file mode 100644 index 0000000..50a79d1 --- /dev/null +++ b/deploy/manifests/webhook/service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + name: controller diff --git a/deploy/nv-ipam.yaml b/deploy/nv-ipam.yaml deleted file mode 100644 index 95c0f34..0000000 --- a/deploy/nv-ipam.yaml +++ /dev/null @@ -1,301 +0,0 @@ ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nv-ipam-node -rules: - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch - - apiGroups: - - nv-ipam.nvidia.com - resources: - - ippools - verbs: - - get - - list - - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nv-ipam-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nv-ipam-node -subjects: - - kind: ServiceAccount - name: nv-ipam-node - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nv-ipam-node - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: nv-ipam-node-ds - namespace: kube-system - labels: - tier: node - app: nv-ipam-node - name: nv-ipam-node -spec: - selector: - matchLabels: - name: nv-ipam-node - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - tier: node - app: nv-ipam-node - name: nv-ipam-node - spec: - tolerations: - - operator: Exists - effect: NoSchedule - - operator: Exists - effect: NoExecute - serviceAccountName: nv-ipam-node - containers: - - name: nv-ipam-node - image: ghcr.io/mellanox/nvidia-k8s-ipam:latest - imagePullPolicy: IfNotPresent - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: IPPOOLS_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: [ "/ipam-node" ] - args: - - --node-name=$(NODE_NAME) - - --v=1 # log level for ipam-node - - --logging-format=json - - --bind-address=unix:///var/lib/cni/nv-ipam/daemon.sock - - --store-file=/var/lib/cni/nv-ipam/store - - --cni-daemon-socket=unix:///var/lib/cni/nv-ipam/daemon.sock - - --cni-daemon-call-timeout=5 # 5 seconds - - --cni-bin-dir=/opt/cni/bin - - --cni-conf-dir=/etc/cni/net.d/nv-ipam.d - - --cni-log-file=/var/log/nv-ipam-cni.log - - --cni-log-level=info # log level for shim CNI - - --ippools-namespace=$(IPPOOLS_NAMESPACE) - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "300m" - memory: "300Mi" - volumeMounts: - - name: cnibin - mountPath: /opt/cni/bin - - name: cniconf - mountPath: /etc/cni/net.d/nv-ipam.d - - name: daemonstate - mountPath: /var/lib/cni/nv-ipam/ - terminationGracePeriodSeconds: 10 - volumes: - - name: cnibin - hostPath: - path: /opt/cni/bin - type: DirectoryOrCreate - - name: cniconf - hostPath: - path: /etc/cni/net.d/nv-ipam.d - type: DirectoryOrCreate - - name: daemonstate - hostPath: - path: /var/lib/cni/nv-ipam/ - type: DirectoryOrCreate ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nv-ipam-controller -rules: - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - delete - - apiGroups: - - nv-ipam.nvidia.com - resources: - - ippools - verbs: - - get - - list - - watch - - create - - apiGroups: - - nv-ipam.nvidia.com - resources: - - ippools/status - verbs: - - get - - update - - patch - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nv-ipam-controller -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nv-ipam-controller -subjects: - - kind: ServiceAccount - name: nv-ipam-controller - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nv-ipam-controller - namespace: kube-system ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: nv-ipam-controller - namespace: kube-system - annotations: - kubernetes.io/description: | - This deployment launches the nv-ipam controller for nv-ipam. -spec: - strategy: - type: RollingUpdate - replicas: 1 - selector: - matchLabels: - name: nv-ipam-controller - template: - metadata: - labels: - name: nv-ipam-controller - spec: - priorityClassName: system-cluster-critical - serviceAccountName: nv-ipam-controller - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: name - operator: In - values: - - nv-ipam-controller - topologyKey: "kubernetes.io/hostname" - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: node-role.kubernetes.io/master - operator: In - values: - - "" - - weight: 1 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: In - values: - - "" - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - containers: - - name: nv-ipam-controller - image: ghcr.io/mellanox/nvidia-k8s-ipam:latest - imagePullPolicy: IfNotPresent - command: [ "/ipam-controller" ] - args: - - --leader-elect=true - - --leader-elect-namespace=$(POD_NAMESPACE) - - --ippools-namespace=$(POD_NAMESPACE) - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - livenessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - resources: - requests: - cpu: 100m - memory: 300Mi diff --git a/deploy/overlays/certmanager/controller_webhook_patch.yaml b/deploy/overlays/certmanager/controller_webhook_patch.yaml new file mode 100644 index 0000000..ca7011d --- /dev/null +++ b/deploy/overlays/certmanager/controller_webhook_patch.yaml @@ -0,0 +1,23 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: controller + namespace: system +spec: + template: + spec: + containers: + - name: controller + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: nv-ipam-webhook-server-cert diff --git a/deploy/overlays/certmanager/kustomization.yaml b/deploy/overlays/certmanager/kustomization.yaml new file mode 100644 index 0000000..b23adea --- /dev/null +++ b/deploy/overlays/certmanager/kustomization.yaml @@ -0,0 +1,59 @@ +# Adds namespace to all resources. +namespace: kube-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +namePrefix: nv-ipam- + +# Labels to add to all resources and selectors. +commonLabels: + app: nvidia-k8s-ipam + +resources: + - ../../crds + - ../../manifests/controller + - ../../manifests/node + - ../../manifests/webhook + - ../../manifests/certmanager + +patches: + - patch: | + - op: add + path: /spec/template/spec/containers/0/args/- + value: --webhook=true + target: + group: apps + version: v1 + kind: Deployment + name: controller + namespace: system + - path: controller_webhook_patch.yaml + - path: webhook_config_patch.yaml + +vars: + - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldref: + fieldpath: metadata.namespace + - name: CERTIFICATE_NAME + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + - name: SERVICE_NAMESPACE # namespace of the service + objref: + kind: Service + version: v1 + name: webhook-service + fieldref: + fieldpath: metadata.namespace + - name: SERVICE_NAME + objref: + kind: Service + version: v1 + name: webhook-service diff --git a/deploy/overlays/certmanager/webhook_config_patch.yaml b/deploy/overlays/certmanager/webhook_config_patch.yaml new file mode 100644 index 0000000..6b2c0f3 --- /dev/null +++ b/deploy/overlays/certmanager/webhook_config_patch.yaml @@ -0,0 +1,6 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/deploy/overlays/no-webhook/kustomization.yaml b/deploy/overlays/no-webhook/kustomization.yaml new file mode 100644 index 0000000..1454e23 --- /dev/null +++ b/deploy/overlays/no-webhook/kustomization.yaml @@ -0,0 +1,19 @@ +# Adds namespace to all resources. +namespace: kube-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +namePrefix: nv-ipam- + +# Labels to add to all resources and selectors. +commonLabels: + app: nvidia-k8s-ipam + +resources: + - ../../crds + - ../../manifests/controller + - ../../manifests/node + +# Uncomment this to deploy to Openshift cluster +#patches: +# - path: node_openshift_cni_path_patch.yaml diff --git a/deploy/overlays/no-webhook/node_openshift_cni_path_patch.yaml b/deploy/overlays/no-webhook/node_openshift_cni_path_patch.yaml new file mode 100644 index 0000000..8c5438c --- /dev/null +++ b/deploy/overlays/no-webhook/node_openshift_cni_path_patch.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-ds + namespace: system +spec: + template: + spec: + volumes: + - name: cnibin + hostPath: + path: /var/lib/cni/bin + type: DirectoryOrCreate diff --git a/deploy/overlays/openshilft/controller_webhook_patch.yaml b/deploy/overlays/openshilft/controller_webhook_patch.yaml new file mode 100644 index 0000000..ca7011d --- /dev/null +++ b/deploy/overlays/openshilft/controller_webhook_patch.yaml @@ -0,0 +1,23 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: controller + namespace: system +spec: + template: + spec: + containers: + - name: controller + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: nv-ipam-webhook-server-cert diff --git a/deploy/overlays/openshilft/kustomization.yaml b/deploy/overlays/openshilft/kustomization.yaml new file mode 100644 index 0000000..abce16d --- /dev/null +++ b/deploy/overlays/openshilft/kustomization.yaml @@ -0,0 +1,33 @@ +# Adds namespace to all resources. +namespace: kube-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +namePrefix: nv-ipam- + +# Labels to add to all resources and selectors. +commonLabels: + app: nvidia-k8s-ipam + +resources: + - ../../crds + - ../../manifests/controller + - ../../manifests/node + - ../../manifests/openshift-privileged + - ../../manifests/webhook + +patches: + - path: node_openshift_cni_path_patch.yaml + - path: controller_webhook_patch.yaml + - patch: | + - op: add + path: /spec/template/spec/containers/0/args/- + value: --webhook=true + target: + group: apps + version: v1 + kind: Deployment + name: controller + namespace: system + - path: webhook_service_patch.yaml + - path: webhook_config_patch.yaml diff --git a/deploy/overlays/openshilft/node_openshift_cni_path_patch.yaml b/deploy/overlays/openshilft/node_openshift_cni_path_patch.yaml new file mode 100644 index 0000000..8c5438c --- /dev/null +++ b/deploy/overlays/openshilft/node_openshift_cni_path_patch.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-ds + namespace: system +spec: + template: + spec: + volumes: + - name: cnibin + hostPath: + path: /var/lib/cni/bin + type: DirectoryOrCreate diff --git a/deploy/overlays/openshilft/webhook_config_patch.yaml b/deploy/overlays/openshilft/webhook_config_patch.yaml new file mode 100644 index 0000000..e5a1054 --- /dev/null +++ b/deploy/overlays/openshilft/webhook_config_patch.yaml @@ -0,0 +1,6 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration + annotations: + service.beta.openshift.io/inject-cabundle: "true" diff --git a/deploy/overlays/openshilft/webhook_service_patch.yaml b/deploy/overlays/openshilft/webhook_service_patch.yaml new file mode 100644 index 0000000..50084b6 --- /dev/null +++ b/deploy/overlays/openshilft/webhook_service_patch.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: system + annotations: + service.alpha.openshift.io/serving-cert-secret-name: nv-ipam-webhook-server-cert