From e70804030e8b262d2de883ad8ebf46849db2df45 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Fri, 16 Jul 2021 14:14:02 +0800 Subject: [PATCH] Apply endpointslice crd when kubernetes version < 1.17 --- Makefile | 2 +- controllers/checker/handler.go | 33 +++++++++++++++ controllers/checker/version.go | 41 +++++++++++------- controllers/deploy_broker.go | 38 +++++------------ .../common => }/embeddedyamls/.gitignore | 0 .../common => }/embeddedyamls/generate.go | 2 +- .../embeddedyamls/generators/yamls2go.go | 1 + .../common => }/embeddedyamls/unmarshal.go | 0 controllers/ensures/broker/ensure.go | 2 +- controllers/ensures/gateway/crds.go | 4 +- controllers/ensures/lighthouse/crds.go | 4 +- .../operator/common/serviceaccount/ensure.go | 2 +- .../operator/lighthouse/crds/ensure.go | 4 +- .../ensures/operator/lighthouse/scc/ensure.go | 2 +- .../lighthouse/serviceaccount/ensure.go | 2 +- .../operator/submarinerop/crds/ensure.go | 4 +- .../operator/submarinerop/scc/ensure.go | 2 +- .../submarinerop/serviceaccount/ensure.go | 2 +- controllers/join_broker.go | 42 ++++++++++++------- .../{ensures => }/utils/createorupdate.go | 2 +- .../crds/discovery.k8s.io_endpointslices.yaml | 13 ++++++ 21 files changed, 128 insertions(+), 74 deletions(-) create mode 100644 controllers/checker/handler.go rename controllers/{ensures/operator/common => }/embeddedyamls/.gitignore (100%) rename controllers/{ensures/operator/common => }/embeddedyamls/generate.go (90%) rename controllers/{ensures/operator/common => }/embeddedyamls/generators/yamls2go.go (98%) rename controllers/{ensures/operator/common => }/embeddedyamls/unmarshal.go (100%) rename controllers/{ensures => }/utils/createorupdate.go (96%) create mode 100644 manifests/fix/crds/discovery.k8s.io_endpointslices.yaml diff --git a/Makefile b/Makefile index bfc398b..a30c4f4 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." generate-embeddedyamls: - go generate controllers/ensures/operator/common/embeddedyamls/generate.go + go generate controllers/embeddedyamls/generate.go fmt: ## Run go fmt against code. go fmt ./... diff --git a/controllers/checker/handler.go b/controllers/checker/handler.go new file mode 100644 index 0000000..3814e90 --- /dev/null +++ b/controllers/checker/handler.go @@ -0,0 +1,33 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package checker + +import ( + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tkestack/knitnet-operator/controllers/embeddedyamls" + "github.com/tkestack/knitnet-operator/controllers/utils" +) + +func CreateOrUpdateEndpointslicesCRD(c client.Client) error { + if err := utils.CreateOrUpdateEmbeddedCRD(c, embeddedyamls.Manifests_fix_crds_discovery_k8s_io_endpointslices_yaml); err != nil { + klog.Errorf("Error creating the EndpointSlice CRD: %v", err) + return err + } + return nil +} diff --git a/controllers/checker/version.go b/controllers/checker/version.go index 16a5c5b..7420a7e 100644 --- a/controllers/checker/version.go +++ b/controllers/checker/version.go @@ -14,37 +14,40 @@ See the License for the specific language governing permissions and limitations under the License. */ -package version +package checker import ( "fmt" "strconv" "strings" - "github.com/pkg/errors" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/klog/v2" ) const ( - minK8sMajor = 1 // We need K8s 1.15 for endpoint slices - minK8sMinor = 15 + minK8sMajor = 1 // We need K8s 1.15 for endpoint slices + minK8sMinor = 15 // Need patch if k8s minor version < 17 and >= 15 + goodK8sMinor = 17 // Don't need patch if k8s minor version >= 17 ) -func CheckRequirements(config *rest.Config) (string, []string, error) { - failedRequirements := []string{} +func CheckKubernetesVersion(config *rest.Config) (bool, error) { + needPatch := false clientset, err := kubernetes.NewForConfig(config) if err != nil { - return "", failedRequirements, errors.WithMessage(err, "error creating API server client") + klog.Errorf("Error creating API server client: %v", err) + return needPatch, err } serverVersion, err := clientset.Discovery().ServerVersion() if err != nil { - return "", failedRequirements, errors.WithMessage(err, "error obtaining API server version") + klog.Errorf("Error obtaining API server version: %v", err) + return needPatch, err } major, err := strconv.Atoi(serverVersion.Major) if err != nil { - return serverVersion.String(), failedRequirements, - errors.WithMessagef(err, "error parsing API server major version %v", serverVersion.Major) + klog.Errorf("Error parsing API server major version %v", err) + return needPatch, err } var minor int if strings.HasSuffix(serverVersion.Minor, "+") { @@ -53,13 +56,19 @@ func CheckRequirements(config *rest.Config) (string, []string, error) { minor, err = strconv.Atoi(serverVersion.Minor) } if err != nil { - return serverVersion.String(), failedRequirements, - errors.WithMessagef(err, "error parsing API server minor version %v", serverVersion.Minor) + klog.Errorf("Error parsing API server minor version %v", err) + return needPatch, err } + if major < minK8sMajor || (major == minK8sMajor && minor < minK8sMinor) { - failedRequirements = append(failedRequirements, - fmt.Sprintf("Submariner requires Kubernetes %d.%d; your cluster is running %s.%s", - minK8sMajor, minK8sMinor, serverVersion.Major, serverVersion.Minor)) + klog.Errorf("Submariner requires Kubernetes %d.%d; your cluster is running %s.%s", + minK8sMajor, minK8sMinor, serverVersion.Major, serverVersion.Minor) + return needPatch, fmt.Errorf("submariner requires Kubernetes %d.%d; your cluster is running %s.%s", + minK8sMajor, minK8sMinor, serverVersion.Major, serverVersion.Minor) + } else { + if minor < goodK8sMinor { + needPatch = true + } + return needPatch, nil } - return serverVersion.String(), failedRequirements, nil } diff --git a/controllers/deploy_broker.go b/controllers/deploy_broker.go index be10186..28602ae 100644 --- a/controllers/deploy_broker.go +++ b/controllers/deploy_broker.go @@ -20,6 +20,7 @@ import ( submarinerv1a1 "github.com/submariner-io/submariner-operator/apis/submariner/v1alpha1" "k8s.io/klog/v2" + "github.com/tkestack/knitnet-operator/controllers/checker" "github.com/tkestack/knitnet-operator/controllers/components" consts "github.com/tkestack/knitnet-operator/controllers/ensures" @@ -31,17 +32,18 @@ import ( "github.com/tkestack/knitnet-operator/controllers/ensures/operator/submarinerop" ) -// var defaultComponents = []string{components.ServiceDiscovery, components.Connectivity} -// var validComponents = []string{components.ServiceDiscovery, components.Connectivity, components.Globalnet, components.Broker} - func (r *KnitnetReconciler) DeploySubmerinerBroker(instance *operatorv1alpha1.Knitnet) error { - brokerConfig := &instance.Spec.BrokerConfig - - // if err := isValidComponents(instance); err != nil { - // klog.Errorf("Invalid components parameter: %v", err) - // return err - // } + needPatch, err := checker.CheckKubernetesVersion(r.Config) + if err != nil { + return err + } + if needPatch { + if err := checker.CreateOrUpdateEndpointslicesCRD(r.Client); err != nil { + return err + } + } + brokerConfig := &instance.Spec.BrokerConfig if valid, err := isValidGlobalnetConfig(instance); !valid { klog.Errorf("Invalid GlobalCIDR configuration: %v", err) return err @@ -83,24 +85,6 @@ func (r *KnitnetReconciler) DeploySubmerinerBroker(instance *operatorv1alpha1.Kn return nil } -// func isValidComponents(instance *operatorv1alpha1.Knitnet) error { -// componentSet := stringset.New(instance.Spec.BrokerConfig.ComponentArr...) -// validComponentSet := stringset.New(validComponents...) - -// if componentSet.Size() < 1 { -// klog.Info("Use default components") -// instance.Spec.BrokerConfig.ComponentArr = defaultComponents -// return nil -// } - -// for _, component := range componentSet.Elements() { -// if !validComponentSet.Contains(component) { -// return fmt.Errorf("unknown component: %s", component) -// } -// } -// return nil -// } - func isValidGlobalnetConfig(instance *operatorv1alpha1.Knitnet) (bool, error) { brokerConfig := &instance.Spec.BrokerConfig var err error diff --git a/controllers/ensures/operator/common/embeddedyamls/.gitignore b/controllers/embeddedyamls/.gitignore similarity index 100% rename from controllers/ensures/operator/common/embeddedyamls/.gitignore rename to controllers/embeddedyamls/.gitignore diff --git a/controllers/ensures/operator/common/embeddedyamls/generate.go b/controllers/embeddedyamls/generate.go similarity index 90% rename from controllers/ensures/operator/common/embeddedyamls/generate.go rename to controllers/embeddedyamls/generate.go index 304fba1..c40e95e 100644 --- a/controllers/ensures/operator/common/embeddedyamls/generate.go +++ b/controllers/embeddedyamls/generate.go @@ -15,4 +15,4 @@ limitations under the License. */ package embeddedyamls -//go:generate go run generators/yamls2go.go ../../../../../ . +//go:generate go run generators/yamls2go.go ../../ . diff --git a/controllers/ensures/operator/common/embeddedyamls/generators/yamls2go.go b/controllers/embeddedyamls/generators/yamls2go.go similarity index 98% rename from controllers/ensures/operator/common/embeddedyamls/generators/yamls2go.go rename to controllers/embeddedyamls/generators/yamls2go.go index eb597c6..7a14c84 100644 --- a/controllers/ensures/operator/common/embeddedyamls/generators/yamls2go.go +++ b/controllers/embeddedyamls/generators/yamls2go.go @@ -69,6 +69,7 @@ var files = []string{ "manifests/config/rbac/networkplugin_syncer/service_account.yaml", "manifests/config/rbac/networkplugin_syncer/cluster_role.yaml", "manifests/config/rbac/networkplugin_syncer/cluster_role_binding.yaml", + "manifests/fix/crds/discovery.k8s.io_endpointslices.yaml", } // Reads all .yaml files in the crdDirectory diff --git a/controllers/ensures/operator/common/embeddedyamls/unmarshal.go b/controllers/embeddedyamls/unmarshal.go similarity index 100% rename from controllers/ensures/operator/common/embeddedyamls/unmarshal.go rename to controllers/embeddedyamls/unmarshal.go diff --git a/controllers/ensures/broker/ensure.go b/controllers/ensures/broker/ensure.go index 8d73399..965d234 100644 --- a/controllers/ensures/broker/ensure.go +++ b/controllers/ensures/broker/ensure.go @@ -34,7 +34,7 @@ import ( consts "github.com/tkestack/knitnet-operator/controllers/ensures" "github.com/tkestack/knitnet-operator/controllers/ensures/gateway" "github.com/tkestack/knitnet-operator/controllers/ensures/lighthouse" - crdutils "github.com/tkestack/knitnet-operator/controllers/ensures/utils" + crdutils "github.com/tkestack/knitnet-operator/controllers/utils" ) func Ensure(c client.Client, config *rest.Config, serviceDiscoveryEnabled, globalnetEnabled, crds bool) error { diff --git a/controllers/ensures/gateway/crds.go b/controllers/ensures/gateway/crds.go index 400b363..ce32fba 100644 --- a/controllers/ensures/gateway/crds.go +++ b/controllers/ensures/gateway/crds.go @@ -20,8 +20,8 @@ import ( "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/tkestack/knitnet-operator/controllers/ensures/operator/common/embeddedyamls" - "github.com/tkestack/knitnet-operator/controllers/ensures/utils" + "github.com/tkestack/knitnet-operator/controllers/embeddedyamls" + "github.com/tkestack/knitnet-operator/controllers/utils" ) // Ensure ensures that the required resources are deployed on the target system diff --git a/controllers/ensures/lighthouse/crds.go b/controllers/ensures/lighthouse/crds.go index bebc1cd..da5f151 100644 --- a/controllers/ensures/lighthouse/crds.go +++ b/controllers/ensures/lighthouse/crds.go @@ -23,8 +23,8 @@ import ( "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/tkestack/knitnet-operator/controllers/ensures/operator/common/embeddedyamls" - "github.com/tkestack/knitnet-operator/controllers/ensures/utils" + "github.com/tkestack/knitnet-operator/controllers/embeddedyamls" + "github.com/tkestack/knitnet-operator/controllers/utils" ) const ( diff --git a/controllers/ensures/operator/common/serviceaccount/ensure.go b/controllers/ensures/operator/common/serviceaccount/ensure.go index 78309e6..c1a3e95 100644 --- a/controllers/ensures/operator/common/serviceaccount/ensure.go +++ b/controllers/ensures/operator/common/serviceaccount/ensure.go @@ -26,7 +26,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/tkestack/knitnet-operator/controllers/ensures/operator/common/embeddedyamls" + "github.com/tkestack/knitnet-operator/controllers/embeddedyamls" ) // Ensure creates the given service account diff --git a/controllers/ensures/operator/lighthouse/crds/ensure.go b/controllers/ensures/operator/lighthouse/crds/ensure.go index aa31b1d..2232d77 100644 --- a/controllers/ensures/operator/lighthouse/crds/ensure.go +++ b/controllers/ensures/operator/lighthouse/crds/ensure.go @@ -19,8 +19,8 @@ package crds import ( "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/tkestack/knitnet-operator/controllers/ensures/operator/common/embeddedyamls" - "github.com/tkestack/knitnet-operator/controllers/ensures/utils" + "github.com/tkestack/knitnet-operator/controllers/embeddedyamls" + "github.com/tkestack/knitnet-operator/controllers/utils" ) func Ensure(c client.Client) error { diff --git a/controllers/ensures/operator/lighthouse/scc/ensure.go b/controllers/ensures/operator/lighthouse/scc/ensure.go index dba2dee..7d6817c 100644 --- a/controllers/ensures/operator/lighthouse/scc/ensure.go +++ b/controllers/ensures/operator/lighthouse/scc/ensure.go @@ -19,7 +19,7 @@ package scc import ( "k8s.io/client-go/rest" - "github.com/tkestack/knitnet-operator/controllers/ensures/operator/common/embeddedyamls" + "github.com/tkestack/knitnet-operator/controllers/embeddedyamls" "github.com/tkestack/knitnet-operator/controllers/ensures/operator/common/scc" ) diff --git a/controllers/ensures/operator/lighthouse/serviceaccount/ensure.go b/controllers/ensures/operator/lighthouse/serviceaccount/ensure.go index a12cc6a..4ea47b9 100644 --- a/controllers/ensures/operator/lighthouse/serviceaccount/ensure.go +++ b/controllers/ensures/operator/lighthouse/serviceaccount/ensure.go @@ -21,7 +21,7 @@ import ( "github.com/tkestack/knitnet-operator/controllers/ensures/operator/common/serviceaccount" - "github.com/tkestack/knitnet-operator/controllers/ensures/operator/common/embeddedyamls" + "github.com/tkestack/knitnet-operator/controllers/embeddedyamls" ) // Ensure functions updates or installs the operator CRDs in the cluster diff --git a/controllers/ensures/operator/submarinerop/crds/ensure.go b/controllers/ensures/operator/submarinerop/crds/ensure.go index a0bd730..5b60ca3 100644 --- a/controllers/ensures/operator/submarinerop/crds/ensure.go +++ b/controllers/ensures/operator/submarinerop/crds/ensure.go @@ -19,8 +19,8 @@ package crds import ( "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/tkestack/knitnet-operator/controllers/ensures/operator/common/embeddedyamls" - "github.com/tkestack/knitnet-operator/controllers/ensures/utils" + "github.com/tkestack/knitnet-operator/controllers/embeddedyamls" + "github.com/tkestack/knitnet-operator/controllers/utils" ) // Ensure functions updates or installs the operator CRDs in the cluster diff --git a/controllers/ensures/operator/submarinerop/scc/ensure.go b/controllers/ensures/operator/submarinerop/scc/ensure.go index 4ad4594..ffce9cf 100644 --- a/controllers/ensures/operator/submarinerop/scc/ensure.go +++ b/controllers/ensures/operator/submarinerop/scc/ensure.go @@ -19,7 +19,7 @@ package scc import ( "k8s.io/client-go/rest" - "github.com/tkestack/knitnet-operator/controllers/ensures/operator/common/embeddedyamls" + "github.com/tkestack/knitnet-operator/controllers/embeddedyamls" "github.com/tkestack/knitnet-operator/controllers/ensures/operator/common/scc" ) diff --git a/controllers/ensures/operator/submarinerop/serviceaccount/ensure.go b/controllers/ensures/operator/submarinerop/serviceaccount/ensure.go index 22d4290..265913d 100644 --- a/controllers/ensures/operator/submarinerop/serviceaccount/ensure.go +++ b/controllers/ensures/operator/submarinerop/serviceaccount/ensure.go @@ -21,7 +21,7 @@ import ( "github.com/tkestack/knitnet-operator/controllers/ensures/operator/common/serviceaccount" - "github.com/tkestack/knitnet-operator/controllers/ensures/operator/common/embeddedyamls" + "github.com/tkestack/knitnet-operator/controllers/embeddedyamls" ) // Ensure functions updates or installs the operator CRDs in the cluster diff --git a/controllers/join_broker.go b/controllers/join_broker.go index f1774f8..1be2bbc 100644 --- a/controllers/join_broker.go +++ b/controllers/join_broker.go @@ -27,7 +27,7 @@ import ( submariner "github.com/submariner-io/submariner-operator/apis/submariner/v1alpha1" operatorv1alpha1 "github.com/tkestack/knitnet-operator/api/v1alpha1" - cmdVersion "github.com/tkestack/knitnet-operator/controllers/checker" + "github.com/tkestack/knitnet-operator/controllers/checker" "github.com/tkestack/knitnet-operator/controllers/discovery/globalnet" "github.com/tkestack/knitnet-operator/controllers/discovery/network" consts "github.com/tkestack/knitnet-operator/controllers/ensures" @@ -60,6 +60,16 @@ var nodeLabelBackoff wait.Backoff = wait.Backoff{ } func (r *KnitnetReconciler) JoinSubmarinerCluster(instance *operatorv1alpha1.Knitnet) error { + needPatch, err := checker.CheckKubernetesVersion(r.Config) + if err != nil { + return err + } + if needPatch { + if err := checker.CreateOrUpdateEndpointslicesCRD(r.Client); err != nil { + return err + } + } + brokerInfo, err := SyncBrokerInfo(r.Client, r.Reader) if err != nil { return err @@ -89,19 +99,19 @@ func (r *KnitnetReconciler) JoinSubmarinerCluster(instance *operatorv1alpha1.Kni return err } - _, failedRequirements, err := cmdVersion.CheckRequirements(r.Config) - // We display failed requirements even if an error occurred - if len(failedRequirements) > 0 { - klog.Info("The target cluster fails to meet Submariner's requirements:") - for i := range failedRequirements { - klog.Infof("* %s", (failedRequirements)[i]) - } - return fmt.Errorf("the target cluster fails to meet Submariner's requirements") - } - if err != nil { - klog.Errorf("Unable to check all requirements: %v", err) - return err - } + // _, failedRequirements, err := cmdVersion.CheckRequirements(r.Config) + // // We display failed requirements even if an error occurred + // if len(failedRequirements) > 0 { + // klog.Info("The target cluster fails to meet Submariner's requirements:") + // for i := range failedRequirements { + // klog.Infof("* %s", (failedRequirements)[i]) + // } + // return fmt.Errorf("the target cluster fails to meet Submariner's requirements") + // } + // if err != nil { + // klog.Errorf("Unable to check all requirements: %v", err) + // return err + // } if brokerInfo.IsConnectivityEnabled() && joinConfig.LabelGateway { if err := r.HandleNodeLabels(); err != nil { klog.Errorf("Unable to set the gateway node up: %v", err) @@ -227,18 +237,22 @@ func (r *KnitnetReconciler) AllocateAndUpdateGlobalCIDRConfigMap(c client.Client func SyncBrokerInfo(c client.Client, reader client.Reader) (*broker.BrokerInfo, error) { localConfigmap, err := broker.GetBrokerInfoConfigMap(reader) if err != nil { + klog.Errorf("Get local cluster broker info configmap failed: %v", err) return nil, err } brokerInfo, err := broker.NewFromString(localConfigmap.Data["brokerInfo"]) if err != nil { + klog.Errorf("New broker info configmap from string failed: %v", err) return nil, err } brokerCluster, err := brokerInfo.GetBrokerAdministratorCluster() if err != nil { + klog.Errorf("Get broker cluster administrator failed: %v", err) return nil, err } brokerClusterConfigmap, err := broker.GetBrokerInfoConfigMap(brokerCluster.GetAPIReader()) if err != nil { + klog.Errorf("Get broker cluster broker info configmap failed: %v", err) return nil, err } diff --git a/controllers/ensures/utils/createorupdate.go b/controllers/utils/createorupdate.go similarity index 96% rename from controllers/ensures/utils/createorupdate.go rename to controllers/utils/createorupdate.go index 0c78f76..4a9f731 100644 --- a/controllers/ensures/utils/createorupdate.go +++ b/controllers/utils/createorupdate.go @@ -28,7 +28,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/tkestack/knitnet-operator/controllers/ensures/operator/common/embeddedyamls" + "github.com/tkestack/knitnet-operator/controllers/embeddedyamls" ) type CRDUpdater interface { diff --git a/manifests/fix/crds/discovery.k8s.io_endpointslices.yaml b/manifests/fix/crds/discovery.k8s.io_endpointslices.yaml new file mode 100644 index 0000000..61dd829 --- /dev/null +++ b/manifests/fix/crds/discovery.k8s.io_endpointslices.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: endpointslices.discovery.k8s.io +spec: + group: discovery.k8s.io + version: v1beta1 + scope: Namespaced + names: + kind: EndpointSlice + plural: endpointslices + singular: endpointslice