From 630ba82b07d22743dc6b2883080123e49e67eb66 Mon Sep 17 00:00:00 2001 From: suhouzhen Date: Tue, 12 Oct 2021 16:53:03 +0800 Subject: [PATCH] operator create/delete zookeeper --- .../clickhouse.radondb.com/v1/type_chi.go | 5 + .../v1/type_zookeeper.go | 11 + pkg/apis/clickhouse.radondb.com/v1/types.go | 3 + pkg/controller/chi/controller.go | 1 + pkg/controller/chi/creator.go | 49 +++ pkg/controller/chi/deleter.go | 58 +++ pkg/controller/chi/getter.go | 48 +++ pkg/controller/chi/poller.go | 29 ++ pkg/controller/chi/types.go | 3 + pkg/controller/chi/worker.go | 227 +++++++++- pkg/model/ch_config_const.go | 22 +- pkg/model/ch_config_generator.go | 21 +- pkg/model/creator.go | 391 ++++++++++++++++++ pkg/model/labeler.go | 75 +++- pkg/model/namer.go | 148 ++++++- pkg/model/normalizer.go | 18 +- 16 files changed, 1094 insertions(+), 15 deletions(-) diff --git a/pkg/apis/clickhouse.radondb.com/v1/type_chi.go b/pkg/apis/clickhouse.radondb.com/v1/type_chi.go index ea62f426..55c2550a 100644 --- a/pkg/apis/clickhouse.radondb.com/v1/type_chi.go +++ b/pkg/apis/clickhouse.radondb.com/v1/type_chi.go @@ -404,6 +404,7 @@ func (chi *ClickHouseInstallation) WalkHostsTillError( // WalkTillError func (chi *ClickHouseInstallation) WalkTillError( + fZooKeeper func(chi *ClickHouseInstallation) error, fCHIPreliminary func(chi *ClickHouseInstallation) error, fCluster func(cluster *ChiCluster) error, fShard func(shard *ChiShard) error, @@ -411,6 +412,10 @@ func (chi *ClickHouseInstallation) WalkTillError( fCHI func(chi *ClickHouseInstallation) error, ) error { + if err := fZooKeeper(chi); err != nil { + return err + } + if err := fCHIPreliminary(chi); err != nil { return err } diff --git a/pkg/apis/clickhouse.radondb.com/v1/type_zookeeper.go b/pkg/apis/clickhouse.radondb.com/v1/type_zookeeper.go index 47ba25cc..b5b5743c 100644 --- a/pkg/apis/clickhouse.radondb.com/v1/type_zookeeper.go +++ b/pkg/apis/clickhouse.radondb.com/v1/type_zookeeper.go @@ -18,6 +18,10 @@ func (zkc *ChiZookeeperConfig) IsEmpty() bool { return len(zkc.Nodes) == 0 } +func (zkc *ChiZookeeperConfig) GetStatefulSetReplicasNum() int32 { + return zkc.Replica +} + func (zkc *ChiZookeeperConfig) MergeFrom(from *ChiZookeeperConfig, _type MergeType) { if from == nil { return @@ -61,4 +65,11 @@ func (zkc *ChiZookeeperConfig) MergeFrom(from *ChiZookeeperConfig, _type MergeTy if from.Identity != "" { zkc.Identity = from.Identity } + if from.Replica > 0 { + zkc.Replica = from.Replica + } + if from.Port > 0 { + zkc.Port = from.Port + } + zkc.Install = from.Install } diff --git a/pkg/apis/clickhouse.radondb.com/v1/types.go b/pkg/apis/clickhouse.radondb.com/v1/types.go index 90f9f0f8..7d8b3487 100644 --- a/pkg/apis/clickhouse.radondb.com/v1/types.go +++ b/pkg/apis/clickhouse.radondb.com/v1/types.go @@ -348,6 +348,9 @@ type ChiZookeeperConfig struct { OperationTimeoutMs int `json:"operation_timeout_ms,omitempty" yaml:"operation_timeout_ms"` Root string `json:"root,omitempty" yaml:"root"` Identity string `json:"identity,omitempty" yaml:"identity"` + Install bool `json:"install,omitempty" yaml:"install"` + Replica int32 `json:"replica,omitempty" yaml:"replica"` + Port int32 `json:"port,omitempty" yaml:"port"` } // ChiZookeeperNode defines item of nodes section of .spec.configuration.zookeeper diff --git a/pkg/controller/chi/controller.go b/pkg/controller/chi/controller.go index 890b2676..08365349 100644 --- a/pkg/controller/chi/controller.go +++ b/pkg/controller/chi/controller.go @@ -80,6 +80,7 @@ func NewController( chiListerSynced: chopInformerFactory.Clickhouse().V1().ClickHouseInstallations().Informer().HasSynced, chitLister: chopInformerFactory.Clickhouse().V1().ClickHouseInstallationTemplates().Lister(), chitListerSynced: chopInformerFactory.Clickhouse().V1().ClickHouseInstallationTemplates().Informer().HasSynced, + pdbLister: kubeInformerFactory.Policy().V1beta1().PodDisruptionBudgets().Lister(), serviceLister: kubeInformerFactory.Core().V1().Services().Lister(), serviceListerSynced: kubeInformerFactory.Core().V1().Services().Informer().HasSynced, endpointsLister: kubeInformerFactory.Core().V1().Endpoints().Lister(), diff --git a/pkg/controller/chi/creator.go b/pkg/controller/chi/creator.go index b81b6dc2..cd0dc645 100644 --- a/pkg/controller/chi/creator.go +++ b/pkg/controller/chi/creator.go @@ -27,6 +27,26 @@ import ( chop "github.com/TCeason/clickhouse-operator/pkg/apis/clickhouse.radondb.com/v1" ) +// createStatefulSetZooKeeper is an internal function, used in reconcileStatefulSet only +func (c *Controller) createStatefulSetZooKeeper(statefulSet *apps.StatefulSet, chi *chop.ClickHouseInstallation) error { + log.V(1).M(chi).F().P() + + if _, err := c.kubeClient.AppsV1().StatefulSets(statefulSet.Namespace).Create(statefulSet); err != nil { + // Unable to create StatefulSet at all + return err + } + + // StatefulSet created, wait until it is ready + + if err := c.waitZooKeeperReady(statefulSet); err == nil { + // Target generation reached, StatefulSet created successfully + return nil + } + + // Unable to run StatefulSet, StatefulSet create failed, time to rollback? + return c.onStatefulSetZooKeeperCreateFailed(statefulSet, chi) +} + // createStatefulSet is an internal function, used in reconcileStatefulSet only func (c *Controller) createStatefulSet(statefulSet *apps.StatefulSet, host *chop.ChiHost) error { log.V(1).M(host).F().P() @@ -126,6 +146,35 @@ func (c *Controller) onStatefulSetCreateFailed(failedStatefulSet *apps.StatefulS return fmt.Errorf("unexpected flow") } +// onStatefulSetZooKeeperCreateFailed handles situation when StatefulSet create failed +// It can just delete failed StatefulSet or do nothing +func (c *Controller) onStatefulSetZooKeeperCreateFailed(failedStatefulSet *apps.StatefulSet, chi *chop.ClickHouseInstallation) error { + // What to do with StatefulSet - look into chop configuration settings + switch c.chop.Config().OnStatefulSetCreateFailureAction { + case chop.OnStatefulSetCreateFailureActionAbort: + // Report appropriate error, it will break reconcile loop + log.V(1).M(chi).F().Info("abort") + return errors.New(fmt.Sprintf("Create failed on %s", util.NamespaceNameString(failedStatefulSet.ObjectMeta))) + + case chop.OnStatefulSetCreateFailureActionDelete: + // Delete gracefully failed StatefulSet + log.V(1).M(chi).F().Info("going to DELETE FAILED StatefulSet %s", util.NamespaceNameString(failedStatefulSet.ObjectMeta)) + _ = c.deleteZooKeeper(chi) + return c.shouldContinueOnCreateFailed() + + case chop.OnStatefulSetCreateFailureActionIgnore: + // Ignore error, continue reconcile loop + log.V(1).M(chi).F().Info("going to ignore error %s", util.NamespaceNameString(failedStatefulSet.ObjectMeta)) + return nil + + default: + log.V(1).M(chi).A().Error("Unknown c.chop.Config().OnStatefulSetCreateFailureAction=%s", c.chop.Config().OnStatefulSetCreateFailureAction) + return nil + } + + return fmt.Errorf("unexpected flow") +} + // onStatefulSetUpdateFailed handles situation when StatefulSet update failed // It can try to revert StatefulSet to its previous version, specified in rollbackStatefulSet func (c *Controller) onStatefulSetUpdateFailed(rollbackStatefulSet *apps.StatefulSet, host *chop.ChiHost) error { diff --git a/pkg/controller/chi/deleter.go b/pkg/controller/chi/deleter.go index d2f8cdcf..bf154c09 100644 --- a/pkg/controller/chi/deleter.go +++ b/pkg/controller/chi/deleter.go @@ -46,6 +46,64 @@ func (c *Controller) deleteHost(host *chop.ChiHost) error { return nil } +// deleteZooKeeper deletes all kubernetes resources related to ZooKeeper +func (c *Controller) deleteZooKeeper(chi *chop.ClickHouseInstallation) error { + // Each ZooKeeper consists of + // 1. StatefulSet + // 2. PersistentVolumeClaim + // 3. Service + // Need to delete all these item + + log.V(1).M(chi).S().Info(chi.Name) + defer log.V(1).M(chi).E().Info(chi.Name) + + var err error + + // Namespaced name + zooKeeperName := chopmodel.CreateStatefulSetZooKeeperName(chi) + namespace := chi.Namespace + + // delete StatefulSet + log.V(1).M(chi).F().Info("%s/%s", namespace, zooKeeperName) + if err = c.kubeClient.AppsV1().StatefulSets(namespace).Delete(zooKeeperName, newDeleteOptions()); err == nil { + log.V(1).M(chi).Info("OK delete StatefulSet %s/%s", namespace, zooKeeperName) + } else if apierrors.IsNotFound(err) { + log.V(1).M(chi).Info("NEUTRAL not found StatefulSet %s/%s", namespace, zooKeeperName) + err = nil + } else { + log.V(1).M(chi).A().Error("FAIL delete StatefulSet %s/%s err: %v", namespace, zooKeeperName, err) + } + + // delete PDB + pdbName := chopmodel.CreatePodDisruptionBudgetZooKeeperName(chi) + log.V(1).M(chi).F().Info("%s/%s", namespace, pdbName) + if err = c.kubeClient.PolicyV1beta1().PodDisruptionBudgets(namespace).Delete(pdbName, newDeleteOptions()); err == nil { + log.V(1).M(chi).Info("OK delete PodDisruptionBudget %s/%s", namespace, pdbName) + } else if apierrors.IsNotFound(err) { + log.V(1).M(chi).Info("NEUTRAL not found PodDisruptionBudget %s/%s", namespace, pdbName) + err = nil + } else { + log.V(1).M(chi).A().Error("FAIL delete PodDisruptionBudget %s/%s err: %v", namespace, pdbName, err) + } + + // delete PVC? + + // delete Service + clientName := chopmodel.CreateStatefulSetServiceZooKeeperClientName(chi) + log.V(1).M(chi).F().Info("%s/%s", namespace, clientName) + if err = c.deleteServiceIfExists(namespace, clientName); err != nil { + log.V(1).M(chi).A().Error("FAIL delete Service %s/%s err: %v", namespace, clientName, err) + } + + serverName := chopmodel.CreateStatefulSetServiceZooKeeperServerName(chi) + log.V(1).M(chi).F().Info("%s/%s", namespace, serverName) + if err = c.deleteServiceIfExists(namespace, serverName); err != nil { + log.V(1).M(chi).A().Error("FAIL delete Service %s/%s err: %v", namespace, serverName, err) + } + + return err +} + // deleteConfigMapsCHI func (c *Controller) deleteConfigMapsCHI(chi *chop.ClickHouseInstallation) error { // Delete common ConfigMap's diff --git a/pkg/controller/chi/getter.go b/pkg/controller/chi/getter.go index 4b63c1f4..42e39130 100644 --- a/pkg/controller/chi/getter.go +++ b/pkg/controller/chi/getter.go @@ -19,6 +19,7 @@ import ( apps "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" + policy "k8s.io/api/policy/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/apis/meta/v1" kublabels "k8s.io/apimachinery/pkg/labels" @@ -124,6 +125,53 @@ func (c *Controller) getService(objMeta *meta.ObjectMeta, byNameOnly bool) (*cor return nil, fmt.Errorf("too much objects found %d expecting 1", len(objects)) } +// getPodDisruptionBudget gets PodDisruptionBudget either by namespaced name or by labels +func (c *Controller) getPodDisruptionBudget(objMeta *meta.ObjectMeta, byNameOnly bool) (*policy.PodDisruptionBudget, error) { + get := c.pdbLister.PodDisruptionBudgets(objMeta.Namespace).Get + list := c.pdbLister.PodDisruptionBudgets(objMeta.Namespace).List + var objects []*policy.PodDisruptionBudget + + // Check whether object with such name already exists + obj, err := get(objMeta.Name) + + if (obj != nil) && (err == nil) { + // Object found by name + return obj, nil + } + + if !apierrors.IsNotFound(err) { + // Error, which is not related to "Object not found" + return nil, err + } + + // Object not found by name. Try to find by labels + + if byNameOnly { + return nil, fmt.Errorf("object not found by name %s/%s and no label search allowed ", objMeta.Namespace, objMeta.Name) + } + + var selector kublabels.Selector + if selector, err = chopmodel.MakeSelectorFromObjectMeta(objMeta); err != nil { + return nil, err + } + + if objects, err = list(selector); err != nil { + return nil, err + } + + if len(objects) == 0 { + return nil, apierrors.NewNotFound(apps.Resource("PodDisruptionBudget"), objMeta.Name) + } + + if len(objects) == 1 { + // Exactly one object found by labels + return objects[0], nil + } + + // Too much objects found by labels + return nil, fmt.Errorf("too much objects found %d expecting 1", len(objects)) +} + // getStatefulSet gets StatefulSet. Accepted types: // 1. *meta.ObjectMeta // 2. *chop.ChiHost diff --git a/pkg/controller/chi/poller.go b/pkg/controller/chi/poller.go index 1f9daa8b..228ea108 100644 --- a/pkg/controller/chi/poller.go +++ b/pkg/controller/chi/poller.go @@ -78,6 +78,35 @@ func (c *Controller) waitHostReady(host *chop.ChiHost) error { ) } +// waitZooKeeperReady polls zookeeper's StatefulSet until it is ready +func (c *Controller) waitZooKeeperReady(statefulSet *apps.StatefulSet) error { + // Wait for StatefulSet to reach generation + err := c.pollStatefulSet( + statefulSet, + nil, + func(sts *apps.StatefulSet) bool { + if sts == nil { + return false + } + return model.IsStatefulSetGeneration(sts, sts.Generation) + }, + func() {}, + ) + if err != nil { + return err + } + + // Wait StatefulSet to reach ready status + return c.pollStatefulSet( + statefulSet, + nil, + func(sts *apps.StatefulSet) bool { + return model.IsStatefulSetReady(sts) + }, + func() {}, + ) +} + // waitHostDeleted polls host's StatefulSet until it is not available func (c *Controller) waitHostDeleted(host *chop.ChiHost) { for { diff --git a/pkg/controller/chi/types.go b/pkg/controller/chi/types.go index b864e10c..21b26b82 100644 --- a/pkg/controller/chi/types.go +++ b/pkg/controller/chi/types.go @@ -21,6 +21,7 @@ import ( kube "k8s.io/client-go/kubernetes" appslisters "k8s.io/client-go/listers/apps/v1" corelisters "k8s.io/client-go/listers/core/v1" + policylisters "k8s.io/client-go/listers/policy/v1beta1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" @@ -49,6 +50,8 @@ type Controller struct { chitLister choplisters.ClickHouseInstallationTemplateLister chitListerSynced cache.InformerSynced + // pdbLister used as pdbLister.Services(namespace).Get(name) + pdbLister policylisters.PodDisruptionBudgetLister // serviceLister used as serviceLister.Services(namespace).Get(name) serviceLister corelisters.ServiceLister // serviceListerSynced used in waitForCacheSync() diff --git a/pkg/controller/chi/worker.go b/pkg/controller/chi/worker.go index 5a31840a..88d7db98 100644 --- a/pkg/controller/chi/worker.go +++ b/pkg/controller/chi/worker.go @@ -24,6 +24,7 @@ import ( "gopkg.in/d4l3k/messagediff.v1" apps "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" + policy "k8s.io/api/policy/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -438,6 +439,7 @@ func (w *worker) reconcile(chi *chop.ClickHouseInstallation) error { w.creator = chopmodel.NewCreator(w.c.chop, chi) return chi.WalkTillError( + w.reconcileZooKeeper, w.reconcileCHIAuxObjectsPreliminary, w.reconcileCluster, w.reconcileShard, @@ -463,9 +465,9 @@ func (w *worker) reconcileCHIAuxObjectsPreliminary(chi *chop.ClickHouseInstallat } // 2. CHI common ConfigMap without update - create only - w.reconcileCHIConfigMapCommon(chi, nil, false) + _ = w.reconcileCHIConfigMapCommon(chi, nil, false) // 3. CHI users ConfigMap - w.reconcileCHIConfigMapUsers(chi, nil, true) + _ = w.reconcileCHIConfigMapUsers(chi, nil, true) return nil } @@ -498,6 +500,78 @@ func (w *worker) reconcileCHIConfigMapUsers(chi *chop.ClickHouseInstallation, op return nil } +// reconcileZooKeeper reconciles zookeeper +func (w *worker) reconcileZooKeeper(chi *chop.ClickHouseInstallation) error { + w.a.V(2).M(chi).S().P() + defer w.a.V(2).M(chi).E().P() + + // Only handle create. + // That is, once the number of zookeeper nodes is specified, modifications are no longer supported. + // Same as the other zookeeper info, such as port. + zookeeperConfig := chi.Spec.Configuration.Zookeeper + if !zookeeperConfig.Install || len(zookeeperConfig.Nodes) > 0 { + w.a.V(1). + WithEvent(chi, eventActionReconcile, eventReasonReconcileStarted). + WithStatusAction(chi). + M(chi).F(). + Info("no need to reconcile zookeeper") + + return nil + } + + w.a.V(1). + WithEvent(chi, eventActionReconcile, eventReasonReconcileStarted). + WithStatusAction(chi). + M(chi).F(). + Info("reconcile zookeeper started") + + // Create zookeeper artifacts + statefulSetZooKeeper := w.creator.CreateStatefulSetZooKeeper(chi) + + // Check if zookeeper has been installed. + curStatefulSet, _ := w.c.getStatefulSet(&statefulSetZooKeeper.ObjectMeta, false) + if curStatefulSet != nil { + w.a.V(1). + WithEvent(chi, eventActionReconcile, eventReasonReconcileStarted). + WithStatusAction(chi). + M(chi).F(). + Info("zookeeper has installed") + + // Updates are not processed, return. + return nil + } + + serviceZooKeeperServer := w.creator.CreateServiceZooKeeperServer(chi) + serviceZooKeeperClient := w.creator.CreateServiceZooKeeperClient(chi) + podDistruptionBudgetZooKeeper := w.creator.CreatePodDisruptionBudgetZooKeeper(chi) + + // Reconclie zookeeper's Service first. Because the successful operation of ZooKeeper + // depends on successful communication with other nodes. + if err := w.reconcileService(chi, serviceZooKeeperServer); err != nil { + return err + } + if err := w.reconcileService(chi, serviceZooKeeperClient); err != nil { + return err + } + + // Reconcile zookeeper's PodDisruptionBudget + if err := w.reconcilePodDisruptionBudgetZooKeeper(chi, podDistruptionBudgetZooKeeper); err != nil { + return err + } + + // Reconcile zookeeper's StatefulSet + if err := w.reconcileStatefulSetZooKeeper(chi, statefulSetZooKeeper); err != nil { + return err + } + + w.a.V(1). + WithEvent(chi, eventActionReconcile, eventReasonReconcileCompleted). + WithStatusAction(chi). + M(chi).F(). + Info("reconcile zookeeper completed") + return nil +} + // reconcileCluster reconciles Cluster, excluding nested shards func (w *worker) reconcileCluster(cluster *chop.ChiCluster) error { w.a.V(2).M(cluster).S().P() @@ -578,7 +652,8 @@ func (w *worker) reconcileHost(host *chop.ChiHost) error { Info("Adding tables on shard/host:%d/%d cluster:%s", host.Address.ShardIndex, host.Address.ReplicaIndex, host.Address.ClusterName) if err := w.schemer.HostCreateTables(host); err != nil { w.a.M(host).A().Error("ERROR create tables on host %s. err: %v", host.Name, err) - }*/ + } + */ // Wait ClickHouse run if err := w.schemer.HostPing(host); err != nil { w.a.Error("ERROR ping on host %s. err: %v", host.Name, err) @@ -648,11 +723,11 @@ func (w *worker) includeHost(host *chop.ChiHost) error { } func (w *worker) excludeHostFromService(host *chop.ChiHost) { - w.c.deleteLabelReady(host) + _ = w.c.deleteLabelReady(host) } func (w *worker) includeHostIntoService(host *chop.ChiHost) { - w.c.appendLabelReady(host) + _ = w.c.appendLabelReady(host) } // excludeHostFromClickHouseCluster excludes host from ClickHouse configuration @@ -818,6 +893,9 @@ func (w *worker) deleteCHI(chi *chop.ClickHouseInstallation) error { return w.deleteCluster(cluster) }) + // Delete zookeeper + err = w.c.deleteZooKeeper(chi) + // Delete ConfigMap(s) err = w.c.deleteConfigMapsCHI(chi) @@ -1243,6 +1321,145 @@ func (w *worker) getStatefulSetStatus(statefulSet *apps.StatefulSet, host *chop. return chop.StatefulSetStatusUnknown } +// reconcilePodDisruptionBudgetZooKeeper reconciles core.PodDisruptionBudget +func (w *worker) reconcilePodDisruptionBudgetZooKeeper( + chi *chop.ClickHouseInstallation, + pdb *policy.PodDisruptionBudget, +) error { + w.a.V(2).M(chi).S().Info(pdb.Name) + defer w.a.V(2).M(chi).E().Info(pdb.Name) + + // Check whether this object already exists + curPodDisruptionBudget, err := w.c.getPodDisruptionBudget(&pdb.ObjectMeta, false) + + // Only handle create + if curPodDisruptionBudget != nil { + return nil + } + + // PodDisruptionBudget not found, try to create it + if apierrors.IsNotFound(err) { + err = w.createPodDisruptionBudgetZooKeeper(chi, pdb) + } + + if err != nil { + w.a.WithEvent(chi, eventActionReconcile, eventReasonReconcileFailed). + WithStatusAction(chi). + WithStatusError(chi). + M(chi).A(). + Error("FAILED to reconcile PodDisruptionBudget: %s CHI: %s ", pdb.Name, chi.Name) + } + + return err +} + +// createPodDisruptionBudgetZooKeeper create core.PodDisruptionBudget +func (w *worker) createPodDisruptionBudgetZooKeeper( + chi *chop.ClickHouseInstallation, + pdb *policy.PodDisruptionBudget, +) error { + _, err := w.c.kubeClient.PolicyV1beta1().PodDisruptionBudgets(pdb.Namespace).Create(pdb) + + if err == nil { + w.a.V(1). + WithEvent(chi, eventActionCreate, eventReasonCreateCompleted). + WithStatusAction(chi). + M(chi).F(). + Info("Create PodDisruptionBudget %s/%s", pdb.Namespace, pdb.Name) + } else { + w.a.WithEvent(chi, eventActionCreate, eventReasonCreateFailed). + WithStatusAction(chi). + WithStatusError(chi). + M(chi).A(). + Error("Create PodDisruptionBudget %s/%s failed with error %v", pdb.Namespace, pdb.Name, err) + } + + return err +} + +// reconcileStatefulSetZooKeeper reconciles apps.StatefulSet +func (w *worker) reconcileStatefulSetZooKeeper(chi *chop.ClickHouseInstallation, newStatefulSet *apps.StatefulSet) error { + w.a.V(2).M(chi).S().Info(util.NamespaceNameString(newStatefulSet.ObjectMeta)) + defer w.a.V(2).M(chi).E().Info(util.NamespaceNameString(newStatefulSet.ObjectMeta)) + + // Check whether this object already exists in k8s + curStatefulSet, err := w.c.getStatefulSet(&newStatefulSet.ObjectMeta, false) + + // Only handle create + if curStatefulSet != nil { + return nil + } + + // StatefulSet not found, try to create it + if apierrors.IsNotFound(err) { + err = w.createStatefulSetZooKeeper(newStatefulSet, chi) + + // Write zookeeper node info to status + if err == nil { + var zooKeeperNode []chop.ChiZookeeperNode + zookeeperConfig := chi.Spec.Configuration.Zookeeper + + // Create zookeeper node info + replica := int(zookeeperConfig.Replica) + + for i := 0; i < replica; i++ { + host := chopmodel.CreatePodFQDNOfZooKeeper(chi, i) + zooKeeperNode = append(zooKeeperNode, chop.ChiZookeeperNode{ + Host: host, + Port: zookeeperConfig.Port, + }) + } + + normalizedZKConfig := &chi.Status.NormalizedCHI.Configuration.Zookeeper + // Write zookeeper node info to status.chi + normalizedZKConfig.Nodes = make([]chop.ChiZookeeperNode, replica) + copy(normalizedZKConfig.Nodes, zooKeeperNode) + + // Update change to kubernetes + if err = w.c.updateCHIObjectStatus(chi, false); err != nil { + w.a.V(1).M(chi).A().Error("UNABLE to update CHI ZooKeeper. Err: %q", err) + } + } + } + + if err != nil { + w.a.WithEvent(chi, eventActionReconcile, eventReasonReconcileFailed). + WithStatusAction(chi). + WithStatusError(chi). + M(chi).A(). + Error("FAILED to reconcile ZooKeeper StatefulSet: %s CHI: %s ", newStatefulSet.Name, chi.Name) + } + + return err +} + +// createStatefulSetZooKeeper +func (w *worker) createStatefulSetZooKeeper(statefulSet *apps.StatefulSet, chi *chop.ClickHouseInstallation) error { + w.a.V(1). + WithEvent(chi, eventActionCreate, eventReasonCreateStarted). + WithStatusAction(chi). + M(chi).F(). + Info("Create StatefulSet ZooKeeper %s/%s - started", statefulSet.Namespace, statefulSet.Name) + + err := w.c.createStatefulSetZooKeeper(statefulSet, chi) + + if err == nil { + w.a.V(1). + WithEvent(chi, eventActionCreate, eventReasonCreateCompleted). + WithStatusAction(chi). + M(chi).F(). + Info("Create StatefulSet ZooKeeper %s/%s - completed", statefulSet.Namespace, statefulSet.Name) + } else { + w.a.WithEvent(chi, eventActionCreate, eventReasonCreateFailed). + WithStatusAction(chi). + WithStatusError(chi). + M(chi).A(). + Error("Create StatefulSet ZooKeeper %s/%s - failed with error %v", statefulSet.Namespace, statefulSet.Name, err) + } + + return err +} + // reconcileStatefulSet reconciles apps.StatefulSet func (w *worker) reconcileStatefulSet(newStatefulSet *apps.StatefulSet, host *chop.ChiHost) error { w.a.V(2).M(host).S().Info(util.NamespaceNameString(newStatefulSet.ObjectMeta)) diff --git a/pkg/model/ch_config_const.go b/pkg/model/ch_config_const.go index d75ac1ac..70838fac 100644 --- a/pkg/model/ch_config_const.go +++ b/pkg/model/ch_config_const.go @@ -74,6 +74,19 @@ const ( ClickHouseLogContainerName = "clickhouse-log" ) +const ( + // Default ZooKeeper docker image to be used + defaultZooKeeperDockerImage = "docker.io/zookeeper:3.6.3" + + // zooKeeperContainerName Name of container within Pod with ZooKeeper instance. + zooKeeperContainerName = "zookeeper" +) + +const ( + defaultPrometheusPortName = "prometheus" + defaultPrometheusPortNumber = 7000 +) + const ( chPortNumberMustBeAssignedLater = 0 @@ -87,7 +100,14 @@ const ( ) const ( - zkDefaultPort = 2181 + // ZooKeeper open ports + zkDefaultClientPortName = "client" + zkDefaultClientPortNumber = int32(2181) + zkDefaultServerPortName = "server" + zkDefaultServerPortNumber = int32(2888) + zkDefaultLeaderElectionPortName = "leader-election" + zkDefaultLeaderElectionPortNumber = int32(3888) + // zkDefaultRootTemplate specifies default ZK root - /clickhouse/{namespace}/{chi name} zkDefaultRootTemplate = "/clickhouse/%s/%s" ) diff --git a/pkg/model/ch_config_generator.go b/pkg/model/ch_config_generator.go index 0609ca82..b67c92a5 100644 --- a/pkg/model/ch_config_generator.go +++ b/pkg/model/ch_config_generator.go @@ -92,10 +92,25 @@ func (c *ClickHouseConfigGenerator) GetFiles(section chiv1.SettingsSection, incl // GetHostZookeeper creates data for "zookeeper.xml" func (c *ClickHouseConfigGenerator) GetHostZookeeper(host *chiv1.ChiHost) string { zk := host.GetZookeeper() + nodes := zk.Nodes if zk.IsEmpty() { // No Zookeeper nodes provided - return "" + if zk.Install { + var zooKeeperNode []chiv1.ChiZookeeperNode + replica := int(zk.Replica) + + for i := 0; i < replica; i++ { + zkHost := CreatePodFQDNOfZooKeeper(host.CHI, i) + zooKeeperNode = append(zooKeeperNode, chiv1.ChiZookeeperNode{ + Host: zkHost, + Port: zk.Port, + }) + } + nodes = zooKeeperNode + } else { + return "" + } } b := &bytes.Buffer{} @@ -105,9 +120,9 @@ func (c *ClickHouseConfigGenerator) GetHostZookeeper(host *chiv1.ChiHost) string util.Iline(b, 4, "") // Append Zookeeper nodes - for i := range zk.Nodes { + for i := range nodes { // Convenience wrapper - node := &zk.Nodes[i] + node := &nodes[i] // // HOST // PORT diff --git a/pkg/model/creator.go b/pkg/model/creator.go index e877e651..e01e9201 100644 --- a/pkg/model/creator.go +++ b/pkg/model/creator.go @@ -15,11 +15,15 @@ package model import ( + "bytes" "fmt" + "strconv" // "net/url" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + policyv1beta1 "k8s.io/api/policy/v1beta1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -93,6 +97,80 @@ func (c *Creator) CreateServiceCHI() *corev1.Service { } } +// CreateServiceZooKeeperServer creates new corev1.Service for zookeeper +func (c *Creator) CreateServiceZooKeeperServer(chi *chiv1.ClickHouseInstallation) *corev1.Service { + zooKeeperStatefulSetName := CreateStatefulSetZooKeeperName(chi) + zooKeeperServiceName := CreateStatefulSetServiceZooKeeperServerName(chi) + + c.a.V(1).F().Info("%s/%s for ZooKeeper Set %s", chi.Namespace, zooKeeperServiceName, zooKeeperStatefulSetName) + + // TODO: use template if exists. + // if template, ok := host.GetServiceZooKeeperServerTemplate(); ok { + // return c.createServiceZooKeeperServerFromTemplate() + // } + + // Create default Service + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: zooKeeperServiceName, + Namespace: chi.Namespace, + Labels: c.labeler.getLabelsServiceZooKeeperServer(), + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: zkDefaultServerPortName, + Protocol: corev1.ProtocolTCP, + Port: zkDefaultServerPortNumber, + TargetPort: intstr.FromInt(int(zkDefaultServerPortNumber)), + }, + { + Name: zkDefaultLeaderElectionPortName, + Protocol: corev1.ProtocolTCP, + Port: zkDefaultLeaderElectionPortNumber, + TargetPort: intstr.FromInt(int(zkDefaultLeaderElectionPortNumber)), + }, + }, + Selector: c.labeler.getSelectorZooKeeperScope(), + ClusterIP: templateDefaultsServiceClusterIP, + }, + } +} + +// CreateServiceZooKeeperClient creates new corev1.Service for zookeeper +func (c *Creator) CreateServiceZooKeeperClient(chi *chiv1.ClickHouseInstallation) *corev1.Service { + zooKeeperStatefulSetName := CreateStatefulSetZooKeeperName(chi) + zooKeeperServiceName := CreateStatefulSetServiceZooKeeperClientName(chi) + + c.a.V(1).F().Info("%s/%s for ZooKeeper Set %s", chi.Namespace, zooKeeperServiceName, zooKeeperStatefulSetName) + + // TODO: use template if exists. + // if template, ok := host.GetServiceZooKeeperClientTemplate(); ok { + // return c.createServiceZooKeeperClientFromTemplate() + // } + + // Create default Service + zookeeperConfig := chi.Spec.Configuration.Zookeeper + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: zooKeeperServiceName, + Namespace: chi.Namespace, + Labels: c.labeler.getLabelsServiceZooKeeperClient(), + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: zkDefaultClientPortName, + Protocol: corev1.ProtocolTCP, + Port: zookeeperConfig.Port, + TargetPort: intstr.FromInt(int(zookeeperConfig.Port)), + }, + }, + Selector: c.labeler.getSelectorZooKeeperScope(), + }, + } +} + // CreateServiceCluster creates new corev1.Service for specified Cluster func (c *Creator) CreateServiceCluster(cluster *chiv1.ChiCluster) *corev1.Service { serviceName := CreateClusterServiceName(cluster) @@ -269,6 +347,30 @@ func (c *Creator) CreateConfigMapHost(host *chiv1.ChiHost) *corev1.ConfigMap { } } +// CreatePodDisruptionBudgetZooKeeper creates new policy.PodDisruptionBudget for zookeeper +func (c *Creator) CreatePodDisruptionBudgetZooKeeper(chi *chiv1.ClickHouseInstallation) *policyv1beta1.PodDisruptionBudget { + zooKeeperPodDisruptionBudgetName := CreatePodDisruptionBudgetZooKeeperName(chi) + + // Create default PodDisruptionBudget + maxUnavailable := intstr.IntOrString{ + Type: intstr.Int, + IntVal: (chi.Spec.Configuration.Zookeeper.Replica - 1) / 2, + } + return &policyv1beta1.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: zooKeeperPodDisruptionBudgetName, + Namespace: chi.Namespace, + Labels: c.labeler.getLabelsPodDisruptionBudgetZooKeeper(), + }, + Spec: policyv1beta1.PodDisruptionBudgetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: c.labeler.getSelectorZooKeeperScope(), + }, + MaxUnavailable: &maxUnavailable, + }, + } +} + // CreateStatefulSet creates new apps.StatefulSet func (c *Creator) CreateStatefulSet(host *chiv1.ChiHost) *apps.StatefulSet { statefulSetName := CreateStatefulSetName(host) @@ -338,6 +440,53 @@ func (c *Creator) GetStatefulSetVersion(statefulSet *apps.StatefulSet) (string, return label, ok } +// CreateStatefulSetZooKeeper creates new apps.StatefulSet +func (c *Creator) CreateStatefulSetZooKeeper(chi *chiv1.ClickHouseInstallation) *apps.StatefulSet { + zooKeeperStatefulSetName := CreateStatefulSetZooKeeperName(chi) + zooKeeperServiceName := CreateStatefulSetServiceZooKeeperServerName(chi) + + c.a.V(1).F().Info("Create StatefulSet %s/%s", chi.Namespace, zooKeeperServiceName) + + // Create apps.StatefulSet object + replicasNum := chi.Spec.Configuration.Zookeeper.GetStatefulSetReplicasNum() + revisionHistoryLimit := int32(10) + + // StatefulSet has additional label - ZK config fingerprint + statefulSet := &apps.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: zooKeeperStatefulSetName, + Namespace: chi.Namespace, + Labels: c.labeler.getLabelsZooKeeperScope(), + }, + Spec: apps.StatefulSetSpec{ + Replicas: &replicasNum, + ServiceName: zooKeeperServiceName, + Selector: &metav1.LabelSelector{ + MatchLabels: c.labeler.getSelectorZooKeeperScope(), + }, + + // IMPORTANT + // Template is to be setup later + Template: corev1.PodTemplateSpec{}, + + // IMPORTANT + // VolumeClaimTemplates are to be setup later + VolumeClaimTemplates: nil, + + PodManagementPolicy: apps.ParallelPodManagement, + UpdateStrategy: apps.StatefulSetUpdateStrategy{ + Type: apps.RollingUpdateStatefulSetStrategyType, + }, + RevisionHistoryLimit: &revisionHistoryLimit, + }, + } + + c.setupStatefulSetZooKeeperPodTemplate(statefulSet, chi) + c.setupStatefulSetZooKeeperVolumeClaimTemplates(statefulSet) + + return statefulSet +} + // PreparePersistentVolume func (c *Creator) PreparePersistentVolume(pv *corev1.PersistentVolume, host *chiv1.ChiHost) *corev1.PersistentVolume { pv.Labels = util.MergeStringMapsOverwrite(pv.Labels, c.labeler.getLabelsHostScope(host, false)) @@ -355,6 +504,12 @@ func (c *Creator) setupStatefulSetPodTemplate(statefulSet *apps.StatefulSet, hos c.personalizeStatefulSetTemplate(statefulSet, host) } +// setupStatefulSetZooKeeperPodTemplate performs PodTemplate setup of StatefulSet +func (c *Creator) setupStatefulSetZooKeeperPodTemplate(statefulSet *apps.StatefulSet, chi *chiv1.ClickHouseInstallation) { + podTemplate := c.getPodTemplateZooKeeper(chi) + c.statefulSetApplyPodTemplateZooKeeper(statefulSet, podTemplate) +} + func (c *Creator) ensureStatefulSetTemplateIntegrity(statefulSet *apps.StatefulSet, host *chiv1.ChiHost) { c.ensureClickHouseContainerSpecified(statefulSet, host) c.ensureProbesSpecified(statefulSet) @@ -433,6 +588,23 @@ func (c *Creator) getPodTemplate(host *chiv1.ChiHost) *chiv1.ChiPodTemplate { return podTemplate } +// getPodTemplateZooKeeper gets ZooKeeper Pod Template to be used to create StatefulSet +func (c *Creator) getPodTemplateZooKeeper(chi *chiv1.ClickHouseInstallation) *chiv1.ChiPodTemplate { + statefulSetName := CreateStatefulSetZooKeeperName(chi) + zooKeeperConfig := chi.Spec.Configuration.Zookeeper + + // TODO: use template if exists. + // if template, ok := host.GetPodZooKeeperTemplate(); ok { + // return c.createPodZooKeeperFromTemplate() + // } + + // Create default one + podTemplate := c.newDefaultZooKeeperPodTemplate(zooKeeperConfig, statefulSetName) + c.a.V(1).F().Info("statefulSet %s use default generated template", statefulSetName) + + return podTemplate +} + // setupConfigMapVolumes adds to each container in the Pod VolumeMount objects with func (c *Creator) setupConfigMapVolumes(statefulSetObject *apps.StatefulSet, host *chiv1.ChiHost) { configMapPodName := CreateConfigMapPodName(host) @@ -499,6 +671,43 @@ func (c *Creator) setupStatefulSetVolumeClaimTemplates(statefulSet *apps.Statefu c.setupStatefulSetApplyVolumeClaimTemplates(statefulSet, host) } +// setupStatefulSetZooKeeperVolumeClaimTemplates performs VolumeClaimTemplate setup for Containers in PodTemplate of a StatefulSet +func (c *Creator) setupStatefulSetZooKeeperVolumeClaimTemplates(statefulSet *apps.StatefulSet) { + // applies `volumeMounts` of a `container` + // c.setupStatefulSetApplyVolumeMounts(statefulSet, host) + for i := range statefulSet.Spec.Template.Spec.Containers { + // Convenience wrapper + container := &statefulSet.Spec.Template.Spec.Containers[i] + + container.VolumeMounts = []corev1.VolumeMount{ + { + Name: "data", + MountPath: "/var/lib/zookeeper", + }, + } + } + + // applies Data VolumeClaimTemplates on all containers + // c.setupStatefulSetApplyVolumeClaimTemplates(statefulSet, host) + statefulSet.Spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: *resource.NewScaledQuantity(10, resource.Giga), + }, + }, + }, + }, + } +} + // statefulSetApplyPodTemplate fills StatefulSet.Spec.Template with data from provided ChiPodTemplate func (c *Creator) statefulSetApplyPodTemplate( statefulSet *apps.StatefulSet, @@ -523,6 +732,29 @@ func (c *Creator) statefulSetApplyPodTemplate( } } +// statefulSetApplyPodTemplateZooKeeper fills StatefulSet.Spec.Template with data from provided ChiPodTemplate +func (c *Creator) statefulSetApplyPodTemplateZooKeeper( + statefulSet *apps.StatefulSet, + template *chiv1.ChiPodTemplate, +) { + // StatefulSet's pod template is not directly compatible with ChiPodTemplate, + // we need to extract some fields from ChiPodTemplate and apply on StatefulSet + statefulSet.Spec.Template = corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: template.Name, + Labels: util.MergeStringMapsOverwrite( + c.labeler.getLabelsZooKeeperScope(), + template.ObjectMeta.Labels, + ), + Annotations: util.MergeStringMapsOverwrite( + c.labeler.getAnnotationsZooKeeperScope(), + template.ObjectMeta.Annotations, + ), + }, + Spec: *template.Spec.DeepCopy(), + } +} + func getClickHouseContainer(statefulSet *apps.StatefulSet) (*corev1.Container, bool) { // Find by name for i := range statefulSet.Spec.Template.Spec.Containers { @@ -842,6 +1074,52 @@ func (c *Creator) newDefaultPodTemplate(name string) *chiv1.ChiPodTemplate { return podTemplate } +// newDefaultZooKeeperPodTemplate returns default ZooKeeper Pod Template to be used with StatefulSet +func (c *Creator) newDefaultZooKeeperPodTemplate(zooKeeperConfig chiv1.ChiZookeeperConfig, name string) *chiv1.ChiPodTemplate { + runAsUser := int64(1000) + fSGroup := int64(1000) + podTemplate := &chiv1.ChiPodTemplate{ + Name: name, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ + { + Weight: 1, + PodAffinityTerm: corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "clickhouse.radondb.com/zookeeper", + Operator: "In", + Values: []string{ + name, + }, + }, + }, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, + }, + Containers: []corev1.Container{}, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: &runAsUser, + FSGroup: &fSGroup, + }, + }, + } + + addContainer( + &podTemplate.Spec, + c.newDefaultZooKeeperContainer(zooKeeperConfig), + ) + + return podTemplate +} + // newDefaultLivenessProbe func newDefaultLivenessProbe() *corev1.Probe { return &corev1.Probe{ @@ -896,6 +1174,119 @@ func (c *Creator) newDefaultClickHouseContainer() corev1.Container { } } +// newDefaultZooKeeperContainer returns default ZooKeeper Container +func (c *Creator) newDefaultZooKeeperContainer(zookeeperConfig chiv1.ChiZookeeperConfig) corev1.Container { + b := &bytes.Buffer{} + util.Iline(b, 0, "HOST=$(hostname -s) &&") + util.Iline(b, 0, "DOMAIN=$(hostname -d) &&") + util.Iline(b, 0, "ZOO_DATA_DIR=/var/lib/zookeeper/data &&") + util.Iline(b, 0, "ZOO_DATA_LOG_DIR=/var/lib/zookeeper/datalog &&") + util.Iline(b, 0, "SERVERS="+fmt.Sprintf("%d", zookeeperConfig.Replica)+" &&") + util.Iline(b, 0, "CLIENT_PORT="+fmt.Sprintf("%d", zookeeperConfig.Port)+" &&") + util.Iline(b, 0, "SERVER_PORT="+fmt.Sprintf("%d", zkDefaultServerPortNumber)+" &&") + util.Iline(b, 0, "ELECTION_PORT="+fmt.Sprintf("%d", zkDefaultLeaderElectionPortNumber)+" &&") + util.Iline(b, 0, "{") + util.Iline(b, 0, " echo clientPort=${CLIENT_PORT}") + util.Iline(b, 0, " echo tickTime=2000") + util.Iline(b, 0, " echo initLimit=300") + util.Iline(b, 0, " echo syncLimit=10") + util.Iline(b, 0, " echo maxClientCnxns=2000") + util.Iline(b, 0, " echo maxSessionTimeout=60000000") + util.Iline(b, 0, " echo dataDir=${ZOO_DATA_DIR}") + util.Iline(b, 0, " echo dataLogDir=${ZOO_DATA_LOG_DIR}") + util.Iline(b, 0, " echo autopurge.snapRetainCount=10") + util.Iline(b, 0, " echo autopurge.purgeInterval=1") + util.Iline(b, 0, " echo preAllocSize=131072") + util.Iline(b, 0, " echo snapCount=3000000") + util.Iline(b, 0, " echo leaderServes=yes") + util.Iline(b, 0, " echo standaloneEnabled="+strconv.FormatBool(zookeeperConfig.Replica == 1)) + util.Iline(b, 0, " echo 4lw.commands.whitelist=*") + util.Iline(b, 0, "} > /conf/zoo.cfg &&") + util.Iline(b, 0, "{") + util.Iline(b, 0, " echo zookeeper.root.logger=CONSOLE") + util.Iline(b, 0, " echo zookeeper.console.threshold=INFO") + util.Iline(b, 0, " echo log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender") + util.Iline(b, 0, " echo log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout") + util.Iline(b, 0, " echo 'log4j.rootLogger=${zookeeper.root.logger}'") + util.Iline(b, 0, " echo 'log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold}'") + util.Iline(b, 0, " echo 'log4j.appender.CONSOLE.layout.ConversionPattern=%%d{ISO8601} [myid:%%X{myid}] - %%-5p [%%t:%%C{1}@%%L] - %%m%%n'") + util.Iline(b, 0, "} > /conf/log4j.properties &&") + util.Iline(b, 0, "echo JVMFLAGS='-Xms128M -Xmx4G -XX:+CMSParallelRemarkEnabled' > /conf/java.env &&") + util.Iline(b, 0, "if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then") + util.Iline(b, 0, " NAME=${BASH_REMATCH[1]}") + util.Iline(b, 0, " ORD=${BASH_REMATCH[2]}") + util.Iline(b, 0, "else") + util.Iline(b, 0, " echo 'Fialed to parse name and ordinal of Pod'") + util.Iline(b, 0, " exit 1") + util.Iline(b, 0, "fi &&") + util.Iline(b, 0, "mkdir -p ${ZOO_DATA_DIR} &&") + util.Iline(b, 0, "mkdir -p ${ZOO_DATA_LOG_DIR} &&") + util.Iline(b, 0, "export MY_ID=$((ORD+1)) &&") + util.Iline(b, 0, "echo $MY_ID > $ZOO_DATA_DIR/myid &&") + util.Iline(b, 0, "for (( i=1; i<=$SERVERS; i++ )); do") + util.Iline(b, 0, " echo server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT >> /conf/zoo.cfg") + util.Iline(b, 0, "done &&") + util.Iline(b, 0, "chown -Rv zookeeper \"$ZOO_DATA_DIR\" \"$ZOO_DATA_LOG_DIR\" \"$ZOO_LOG_DIR\" \"$ZOO_CONF_DIR\" &&") + util.Iline(b, 0, "cat /conf/zoo.cfg &&") + util.Iline(b, 0, "zkServer.sh start-foreground") + + return corev1.Container{ + Name: zooKeeperContainerName, + Image: defaultZooKeeperDockerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Ports: []corev1.ContainerPort{ + { + Name: zkDefaultClientPortName, + ContainerPort: zookeeperConfig.Port, + }, + { + Name: zkDefaultServerPortName, + ContainerPort: zkDefaultServerPortNumber, + }, + { + Name: zkDefaultLeaderElectionPortName, + ContainerPort: zkDefaultLeaderElectionPortNumber, + }, + { + Name: defaultPrometheusPortName, + ContainerPort: defaultPrometheusPortNumber, + }, + }, + Command: []string{ + "bash", + "-x", + "-c", + b.String(), + }, + ReadinessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + Exec: &corev1.ExecAction{Command: []string{ + "bash", + "-c", + "OK=$(echo ruok | nc 127.0.0.1 " + + strconv.Itoa(int(zookeeperConfig.Port)) + + "); if [[ \"$OK\" == \"imok\" ]]; then exit 0; else exit 1; fi", + }}, + }, + InitialDelaySeconds: 10, + TimeoutSeconds: 5, + }, + LivenessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + Exec: &corev1.ExecAction{Command: []string{ + "bash", + "-c", + "OK=$(echo ruok | nc 127.0.0.1 " + + strconv.Itoa(int(zookeeperConfig.Port)) + + "); if [[ \"$OK\" == \"imok\" ]]; then exit 0; else exit 1; fi", + }}, + }, + InitialDelaySeconds: 10, + TimeoutSeconds: 5, + }, + } +} + // newDefaultLogContainer returns default Log Container func newDefaultLogContainer() corev1.Container { return corev1.Container{ diff --git a/pkg/model/labeler.go b/pkg/model/labeler.go index c9262002..08ffab5f 100644 --- a/pkg/model/labeler.go +++ b/pkg/model/labeler.go @@ -58,6 +58,14 @@ const ( labelServiceValueCluster = "cluster" labelServiceValueShard = "shard" labelServiceValueHost = "host" + labelServiceValueZooKeeper = "zookeeper" + LabelZooKeeper = clickhouseradondbcom.GroupName + "/zooKeeper" + labelZooKeeperValue = "zookeeper" + LabelPodDisruptionBudget = clickhouseradondbcom.GroupName + "/podDisruptionBudget" + labelPodDisruptionBudgetValue = "pdb" + LabelServiceKind = clickhouseradondbcom.GroupName + "/serviceKind" + labelServiceKindServerValue = "server" + labelServiceKindClientValue = "client" // Supplementary service labels - used to cooperate with k8s LabelZookeeperConfigVersion = clickhouseradondbcom.GroupName + "/zookeeper-version" @@ -117,6 +125,35 @@ func (l *Labeler) getLabelsServiceCHI() map[string]string { }) } +// getLabelsServiceHost +func (l *Labeler) getLabelsServiceZooKeeperServer() map[string]string { + return util.MergeStringMapsOverwrite( + l.getLabelsZooKeeperScope(), + map[string]string{ + LabelService: labelServiceValueZooKeeper, + LabelServiceKind: labelServiceKindServerValue, + }) +} + +// getLabelsServiceHost +func (l *Labeler) getLabelsServiceZooKeeperClient() map[string]string { + return util.MergeStringMapsOverwrite( + l.getLabelsZooKeeperScope(), + map[string]string{ + LabelService: labelServiceValueZooKeeper, + LabelServiceKind: labelServiceKindClientValue, + }) +} + +// getLabelsPodDisruptionBudgetZooKeeper +func (l *Labeler) getLabelsPodDisruptionBudgetZooKeeper() map[string]string { + return util.MergeStringMapsOverwrite( + l.getLabelsZooKeeperScope(), + map[string]string{ + LabelPodDisruptionBudget: labelPodDisruptionBudgetValue, + }) +} + // getLabelsServiceCluster func (l *Labeler) getLabelsServiceCluster(cluster *chi.ChiCluster) map[string]string { return util.MergeStringMapsOverwrite( @@ -169,6 +206,31 @@ func (l *Labeler) getSelectorCHIScopeReady() map[string]string { return l.appendReadyLabels(l.getSelectorCHIScope()) } +// getLabelsZooKeeperScope gets labels for ZooKeeper-scoped object +func (l *Labeler) getLabelsZooKeeperScope() map[string]string { + // Combine generated labels and CHI-provided labels + return util.MergeStringMapsOverwrite( + l.getLabelsCHIScope(), + map[string]string{ + LabelZooKeeper: labelZooKeeperValue, + }) +} + +// getSelectorZooKeeperScope gets labels to select a ZooKeeper-scoped object +func (l *Labeler) getSelectorZooKeeperScope() map[string]string { + // Do not include CHI-provided labels + return util.MergeStringMapsOverwrite( + l.getSelectorCHIScope(), + map[string]string{ + LabelZooKeeper: labelZooKeeperValue, + }) +} + +// getLabelsZooKeeperScope gets labels for ZooKeeper-scoped object +func (l *Labeler) getSelectorZooKeeperScopeReady() map[string]string { + return l.appendReadyLabels(l.getSelectorZooKeeperScope()) +} + // getLabelsClusterScope gets labels for Cluster-scoped object func (l *Labeler) getLabelsClusterScope(cluster *chi.ChiCluster) map[string]string { // Combine generated labels and CHI-provided labels @@ -260,7 +322,7 @@ func (l *Labeler) getLabelsHostScopeReady(host *chi.ChiHost, applySupplementaryS return l.appendReadyLabels(l.getLabelsHostScope(host, applySupplementaryServiceLabels)) } -// getSelectorShardScope gets labels to select a Host-scoped object +// GetSelectorHostScope gets labels to select a Host-scoped object func (l *Labeler) GetSelectorHostScope(host *chi.ChiHost) map[string]string { // Do not include CHI-provided labels return map[string]string{ @@ -287,6 +349,14 @@ func (l *Labeler) appendReadyLabels(dst map[string]string) map[string]string { }) } +// getAnnotationZooKeeperScope gets annotations for ZooKeeper-scoped object +func (l *Labeler) getAnnotationsZooKeeperScope() map[string]string { + // We may want to append some annotations in here + return map[string]string{ + "backup.velero.io/backup-volumes": "data", + } +} + // getAnnotationsHostScope gets annotations for Host-scoped object func (l *Labeler) getAnnotationsHostScope(host *chi.ChiHost) map[string]string { // We may want to append some annotations in here @@ -426,6 +496,9 @@ func makeSetFromObjectMeta(objMeta *meta.ObjectMeta) (kublabels.Set, error) { LabelReplicaName, LabelConfigMap, LabelService, + LabelZooKeeper, + LabelServiceKind, + LabelPodDisruptionBudget, } set := kublabels.Set{} diff --git a/pkg/model/namer.go b/pkg/model/namer.go index b6bdcc10..bf369758 100644 --- a/pkg/model/namer.go +++ b/pkg/model/namer.go @@ -69,6 +69,8 @@ const ( macrosReplicaID = "{replicaID}" // macrosReplicaIndex is an index of the replica in the cluster - integer number, converted into string macrosReplicaIndex = "{replicaIndex}" + // macrosZooKeeperIndex is a sanitized zooKeeper name + macrosZooKeeperIndex = "{zooKeeperIndex}" // macrosHostName is a sanitized host name macrosHostName = "{host}" @@ -122,6 +124,21 @@ const ( // configMapDeploymentNamePattern is a template of macros ConfigMap. "chi-{chi}-deploy-confd-{cluster}-{shard}-{host}" configMapDeploymentNamePattern = "chi-" + macrosChiName + "-deploy-confd-" + macrosClusterName + "-" + macrosHostName + // zooKeeperStatefulSetNamePattern is a template of zooKeepers's StatefulSet's name. "zk-{chi}" + zooKeeperStatefulSetNamePattern = "zk-" + macrosChiName + + // zooKeeperServerStatefulSetServiceNamePattern is a template of zooKeepers's StatefulSet's Service name. "zk-server-{chi}" + zooKeeperServerStatefulSetServiceNamePattern = "zk-server-" + macrosChiName + + // zooKeeperClientStatefulSetServiceNamePattern is a template of zooKeepers's StatefulSet's Service name. "zk-client-{chi}" + zooKeeperClientStatefulSetServiceNamePattern = "zk-client-" + macrosChiName + + // zooKeeperPodDisruptionBudgetNamePattern is a template of zooKeepers's PodDisruptionBudget's name. "zk-pdb-{chi}" + zooKeeperPodDisruptionBudgetNamePattern = "zk-pdb-" + macrosChiName + + // zooKeeperPodNamePattern is a template of zooKeepers's Pod's name. "zk-{chi}-{index}" + zooKeeperPodNamePattern = zooKeeperStatefulSetNamePattern + "-" + macrosZooKeeperIndex + // namespaceDomainPattern presents Domain Name pattern of a namespace // In this pattern "%s" is substituted namespace name's value // Ex.: my-dev-namespace.svc.cluster.local @@ -274,6 +291,26 @@ func (n *namer) namePartHostNameID(name string) string { return util.CreateStringID(name, _len) } +func (n *namer) namePartZooKeeperName(name string) string { + var _len int + if n.ctx == namerContextLabels { + _len = namePartReplicaMaxLenLabelsCtx + } else { + _len = namePartReplicaMaxLenNamesCtx + } + return util.StringHead(name, _len) +} + +func (n *namer) namePartZooKeeperIndex(index int) string { + var _len int + if n.ctx == namerContextLabels { + _len = namePartReplicaMaxLenLabelsCtx + } else { + _len = namePartReplicaMaxLenNamesCtx + } + return util.StringHead(strconv.Itoa(index), _len) +} + func (n *namer) getNamePartNamespace(obj interface{}) string { switch obj.(type) { case *chop.ClickHouseInstallation: @@ -398,6 +435,25 @@ func newNameMacroReplacerChi(chi *chop.ClickHouseInstallation) *strings.Replacer ) } +func newNameMacroReplacerZooKeeper(chi *chop.ClickHouseInstallation) *strings.Replacer { + n := newNamer(namerContextNames) + return strings.NewReplacer( + macrosNamespace, n.namePartNamespace(chi.Namespace), + macrosChiName, n.namePartChiName(chi.Name), + macrosChiID, n.namePartChiNameID(chi.Name), + ) +} + +func newNameMacroReplacerZooKeeperPod(chi *chop.ClickHouseInstallation, index int) *strings.Replacer { + n := newNamer(namerContextNames) + return strings.NewReplacer( + macrosNamespace, n.namePartNamespace(chi.Namespace), + macrosChiName, n.namePartChiName(chi.Name), + macrosChiID, n.namePartChiNameID(chi.Name), + macrosZooKeeperIndex, n.namePartZooKeeperIndex(index), + ) +} + func newNameMacroReplacerCluster(cluster *chop.ChiCluster) *strings.Replacer { n := newNamer(namerContextNames) return strings.NewReplacer( @@ -672,6 +728,72 @@ func CreateStatefulSetServiceName(host *chop.ChiHost) string { return newNameMacroReplacerHost(host).Replace(pattern) } +// CreateStatefulSetZooKeeperName creates a name of a StatefulSet for ZooKeeper instance +func CreateStatefulSetZooKeeperName(chi *chop.ClickHouseInstallation) string { + // Name can be generated either from default name pattern, + // or from personal name pattern provided in PodTemplate + + // Start with default name pattern + pattern := zooKeeperStatefulSetNamePattern + + // PodTemplate may have personal name pattern specified + + // Create StatefulSet name based on name pattern available + return newNameMacroReplacerZooKeeper(chi).Replace(pattern) +} + +// CreateStatefulSetServiceZooKeeperServerName returns a name of a StatefulSet-related Service for zooKeeper instance +func CreateStatefulSetServiceZooKeeperServerName(chi *chop.ClickHouseInstallation) string { + // Name can be generated either from default name pattern, + // or from personal name pattern provided in ServiceTemplate + + // Start with default name pattern + pattern := zooKeeperServerStatefulSetServiceNamePattern + + // ServiceTemplate may have personal name pattern specified + + // Create Service name based on name pattern available + return newNameMacroReplacerZooKeeper(chi).Replace(pattern) +} + +// CreateStatefulSetServiceZooKeeperClientName returns a name of a StatefulSet-related Service for zooKeeper instance +func CreateStatefulSetServiceZooKeeperClientName(chi *chop.ClickHouseInstallation) string { + // Name can be generated either from default name pattern, + // or from personal name pattern provided in ServiceTemplate + + // Start with default name pattern + pattern := zooKeeperClientStatefulSetServiceNamePattern + + // ServiceTemplate may have personal name pattern specified + + // Create Service name based on name pattern available + return newNameMacroReplacerZooKeeper(chi).Replace(pattern) +} + +// CreatePodDisruptionBudgetZooKeeperName returns a name of a PodDisruptionBudget for zooKeeper instance +func CreatePodDisruptionBudgetZooKeeperName(chi *chop.ClickHouseInstallation) string { + // Name can be generated either from default name pattern, + // or from personal name pattern provided in ServiceTemplate + + // Start with default name pattern + pattern := zooKeeperPodDisruptionBudgetNamePattern + + // ServiceTemplate may have personal name pattern specified + + // Create Service name based on name pattern available + return newNameMacroReplacerZooKeeper(chi).Replace(pattern) +} + +// CreatePodZooKeepername returns a name of a Pod of a ZooKeeper instance +func CreatePodZooKeepername(chi *chop.ClickHouseInstallation, index int) string { + // Name can be generated either from default name pattern, + // or from personal name pattern provided in ServiceTemplate + + // Start with default name pattern + pattern := zooKeeperPodNamePattern + return newNameMacroReplacerZooKeeperPod(chi, index).Replace(pattern) +} + // CreatePodHostname returns a name of a Pod of a ClickHouse instance func CreatePodHostname(host *chop.ChiHost) string { // Pod has no own hostname - redirect to appropriate Service @@ -700,6 +822,28 @@ func CreatePodFQDN(host *chop.ChiHost) string { ) } +// CreatePodFQDNOfZooKeeper creates a fully qualified domain name of a zookeeper pod +func CreatePodFQDNOfZooKeeper(chi *chop.ClickHouseInstallation, index int) string { + // FQDN can be generated either from default pattern, + // or from personal pattern provided + + // Start with default pattern + pattern := podFQDNPattern + + if chi.Spec.NamespaceDomainPattern != "" { + // NamespaceDomainPattern has been explicitly specified + pattern = "%s." + chi.Spec.NamespaceDomainPattern + } + + // Create FQDN based on pattern available + // {pod}.{svc}.{namespace}.svc.cluster.local + return fmt.Sprintf( + pattern, + CreatePodZooKeepername(chi, index)+"."+CreateStatefulSetServiceZooKeeperServerName(chi), + chi.Namespace, + ) +} + // CreatePodFQDNsOfCluster creates fully qualified domain names of all pods in a cluster func CreatePodFQDNsOfCluster(cluster *chop.ChiCluster) []string { fqdns := make([]string, 0) @@ -710,7 +854,7 @@ func CreatePodFQDNsOfCluster(cluster *chop.ChiCluster) []string { return fqdns } -// CreatePodFQDNsOfShards creates fully qualified domain names of all pods in a shard +// CreatePodFQDNsOfShard creates fully qualified domain names of all pods in a shard func CreatePodFQDNsOfShard(shard *chop.ChiShard) []string { fqdns := make([]string, 0) shard.WalkHosts(func(host *chop.ChiHost) error { @@ -730,7 +874,7 @@ func CreatePodFQDNsOfCHI(chi *chop.ClickHouseInstallation) []string { return fqdns } -// template is defined in operator config: +// CreatePodRegexp template is defined in operator config: // CHConfigNetworksHostRegexpTemplate: chi-{chi}-[^.]+\\d+-\\d+\\.{namespace}.svc.cluster.local$" func CreatePodRegexp(chi *chop.ClickHouseInstallation, template string) string { return newNameMacroReplacerChi(chi).Replace(template) diff --git a/pkg/model/normalizer.go b/pkg/model/normalizer.go index e33c6eb0..0d5c95f0 100644 --- a/pkg/model/normalizer.go +++ b/pkg/model/normalizer.go @@ -1203,12 +1203,24 @@ func (n *Normalizer) calcFingerprints(host *chiv1.ChiHost) error { // normalizeConfigurationZookeeper normalizes .spec.configuration.zookeeper func (n *Normalizer) normalizeConfigurationZookeeper(zk *chiv1.ChiZookeeperConfig) { + // ZooKeeper Cluster Node must larger than zero. + // ZooKeeper Cluster Node must be odd. + if zk.Install && zk.Replica <= 0 { + zk.Replica = 1 + } else if zk.Install && zk.Replica%2 == 0 { + zk.Replica-- + } + // In case no ZK port specified - assign default + if zk.Port == 0 { + zk.Port = zkDefaultClientPortNumber + } + for i := range zk.Nodes { // Convenience wrapper node := &zk.Nodes[i] if node.Port == 0 { - node.Port = zkDefaultPort + node.Port = zkDefaultClientPortNumber } } @@ -1285,8 +1297,8 @@ func (n *Normalizer) normalizeConfigurationUsers(users *chiv1.Settings) { _, okPasswordDoubleSHA1 := (*users)[username+"/password_double_sha1_hex"] // if SHA256 or DoubleSHA1 are not set, initialize SHA256 from the password if pass != "" && !okPasswordSHA256 && !okPasswordDoubleSHA1 { - pass_sha256 := sha256.Sum256([]byte(pass)) - (*users)[username+"/password_sha256_hex"] = chiv1.NewScalarSetting(hex.EncodeToString(pass_sha256[:])) + passSha256 := sha256.Sum256([]byte(pass)) + (*users)[username+"/password_sha256_hex"] = chiv1.NewScalarSetting(hex.EncodeToString(passSha256[:])) okPasswordSHA256 = true }