From f30c183ecbdb268064072a5c4f72b727423c7159 Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Mon, 1 May 2023 13:24:22 +0200 Subject: [PATCH] HBASE-27831 Introduce zookeeper-single-instance component --- .gitignore | 3 + .../hbase-kubernetes-kustomize/pom.xml | 91 +++++ .../kustomize/components/zookeeper/README.md | 23 ++ .../single-instance/kustomization.yaml | 35 ++ .../zookeeper/single-instance/start.sh | 358 ++++++++++++++++++ .../zookeeper/single-instance/zookeeper.yaml | 142 +++++++ .../00-assert-zookeeper.yaml | 26 ++ .../00-kustomize.yaml | 21 + .../kustomization.yaml | 26 ++ .../00-assert-zookeeper.yaml | 49 +++ .../00-kustomize.yaml | 21 + .../kustomization.yaml | 21 + .../test/resources/kustomize_into_tmpdir.sh | 90 +++++ .../resources/kuttl-test-integration.yaml | 37 ++ .../src/test/resources/kuttl-test-unit.yaml | 28 ++ .../src/test/resources/mvn_exec_run_kuttl.sh | 37 ++ hbase-kubernetes-deployment/pom.xml | 1 + 17 files changed, 1009 insertions(+) create mode 100644 hbase-kubernetes-deployment/hbase-kubernetes-kustomize/pom.xml create mode 100644 hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/README.md create mode 100644 hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/kustomization.yaml create mode 100755 hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/start.sh create mode 100644 hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/zookeeper.yaml create mode 100644 hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/00-assert-zookeeper.yaml create mode 100644 hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/00-kustomize.yaml create mode 100644 hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/kustomization.yaml create mode 100644 hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/00-assert-zookeeper.yaml create mode 100644 hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/00-kustomize.yaml create mode 100644 hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/kustomization.yaml create mode 100755 hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kustomize_into_tmpdir.sh create mode 100644 hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-integration.yaml create mode 100644 hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-unit.yaml create mode 100755 hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/mvn_exec_run_kuttl.sh diff --git a/.gitignore b/.gitignore index b440d2b597..35096783dd 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ linklint-*.zip linklint/ .checkstyle **/.checkstyle + +# detritus produced by kuttl +kubeconfig* diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/pom.xml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/pom.xml new file mode 100644 index 0000000000..d30f7a8498 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/pom.xml @@ -0,0 +1,91 @@ + + + + 4.0.0 + + hbase-kubernetes-deployment + org.apache.hbase.operator.tools + ${revision} + .. + + + hbase-kubernetes-kustomize + Apache HBase - Kubernetes Kustomize + Kustomize deployment descriptors. + pom + + + + org.apache.hbase.operator.tools + hbase-kubernetes-testing-image + pom + test + + + + + + + org.codehaus.mojo + exec-maven-plugin + + src/test/resources/mvn_exec_run_kuttl.sh + true + ${skipTests} + + + + kuttl-unit-tests + test + + exec + + + + --config + src/test/resources/kuttl-test-unit.yaml + --artifacts-dir + target/kuttl-reports + + + + + kuttl-integration-tests + verify + + exec + + + + --config + src/test/resources/kuttl-test-integration.yaml + --artifacts-dir + target/kuttl-reports + + + + + + + + diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/README.md b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/README.md new file mode 100644 index 0000000000..b6af185a87 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/README.md @@ -0,0 +1,23 @@ + + +Uses latest image from the apache zookeeper project. +There is then a start script in 'single-instance/start.sh' +which sets parameters for the zookeeper image and process +on startup. Currently only logs to STDOUT/STDERR; there +are no files in /var/log/zookeeper. diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/kustomization.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/kustomization.yaml new file mode 100644 index 0000000000..1e87468cd7 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/kustomization.yaml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +configMapGenerator: +- name: zookeeper-scripts + files: + - start.sh +# Publish where the zk ensemble can be found. +- name: zookeeper-quorum + literals: + # Hard-coded. Default we expect a simple standalone zk at this location. + # One define is for hbase, the other for hadoop. + - HBASE_ZOOKEEPER_QUORUM="zookeeper-0.zookeeper-headless" + - HA_ZOOKEEPER_QUORUM="zookeeper-0.zookeeper-headless:2181" + options: + disableNameSuffixHash: true + +resources: +- zookeeper.yaml diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/start.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/start.sh new file mode 100755 index 0000000000..48b3d91543 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/start.sh @@ -0,0 +1,358 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Derived from work Copyright 2017 The Kubernetes Authors. +# See https://github.com/kow3ns/kubernetes-zookeeper/blob/master/docker/scripts/start-zookeeper for more details +# and then https://github.com/cloudurable/kube-zookeeper-statefulsets/ +# In the below we change the '--heap' argument to '--percentage' so +# could set the server heap as a percentage of the container resource +# limit rather than hard-code it. +# Currently zookeeper.root.logger is CONSOLE only. We do not write +# logs to files. Fix config. if you need it. +# +# +# Usage: start-zookeeper [OPTIONS] +# Starts a ZooKeeper server based on the supplied options. +# --servers The number of servers in the ensemble. The default +# value is 1. +# --data_dir The directory where the ZooKeeper process will store its +# snapshots. The default is /var/lib/zookeeper/data. +# --data_log_dir The directory where the ZooKeeper process will store its +# write ahead log. The default is +# /var/lib/zookeeper/data/log. +# --conf_dir The directory where the ZooKeeper process will store its +# configuration. The default is /opt/zookeeper/conf. +# --client_port The port on which the ZooKeeper process will listen for +# client requests. The default is 2181. + +# --election_port The port on which the ZooKeeper process will perform +# leader election. The default is 3888. + +# --server_port The port on which the ZooKeeper process will listen for +# requests from other servers in the ensemble. The +# default is 2888. + +# --tick_time The length of a ZooKeeper tick in ms. The default is +# 2000. + +# --init_limit The number of Ticks that an ensemble member is allowed +# to perform leader election. The default is 10. + +# --sync_limit The maximum session timeout that the ensemble will +# allows a client to request. The default is 5. + +# --percentage The percentage of container memory to give to the JVM. + +# --max_client_cnxns The maximum number of client connections that the +# ZooKeeper process will accept simultaneously. The +# default is 60. + +# --snap_retain_count The maximum number of snapshots the ZooKeeper process +# will retain if purge_interval is greater than 0. The +# default is 3. + +# --purge_interval The number of hours the ZooKeeper process will wait +# between purging its old snapshots. If set to 0 old +# snapshots will never be purged. The default is 0. + +# --max_session_timeout The maximum time in milliseconds for a client session +# timeout. The default value is 2 * tick time. + +# --min_session_timeout The minimum time in milliseconds for a client session +# timeout. The default value is 20 * tick time. + +# --log_level The log level for the zookeeeper server. Either FATAL, +# ERROR, WARN, INFO, DEBUG. The default is INFO. + +# --quorum_listen_on_all_ips +# When set to true the ZooKeeper server will listen for +# connections from its peers on all available IP addresses, +# and not only the address configured in the server list of +# the configuration file. It affects the connections handling +# the ZAB protocol and the Fast Leader Election protocol. +# Default value is false. +set -x + +ZOOKEEPER_HOME="$( ls -d /apache-zookeeper* )" +USER="$(whoami)" +HOST="$(hostname -s)" +DOMAIN="$(hostname -d)" +LOG_LEVEL=INFO +DATA_DIR="/var/lib/zookeeper/data" +DATA_LOG_DIR="/var/lib/zookeeper/log" +LOG_DIR="/var/log/zookeeper" +CONF_DIR="/opt/zookeeper/conf" +CLIENT_PORT=2181 +SERVER_PORT=2888 +ELECTION_PORT=3888 +PROM_PORT=7001 +TICK_TIME=2000 +INIT_LIMIT=10 +SYNC_LIMIT=5 +JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT=50 +MAX_CLIENT_CNXNS=1000 +SNAP_RETAIN_COUNT=3 +PURGE_INTERVAL=0 +SERVERS=1 +QUORUM_LISTEN_ON_ALL_IPS=false + +function print_usage() { +echo "\ +Usage: start-zookeeper [OPTIONS] +Starts a ZooKeeper server based on the supplied options. + --servers The number of servers in the ensemble. The default + value is 1. + + --data_dir The directory where the ZooKeeper process will store its + snapshots. The default is /var/lib/zookeeper/data. + + --data_log_dir The directory where the ZooKeeper process will store its + write ahead log. The default is + /var/lib/zookeeper/data/log. + + --conf_dir The directoyr where the ZooKeeper process will store its + configuration. The default is /opt/zookeeper/conf. + + --client_port The port on which the ZooKeeper process will listen for + client requests. The default is 2181. + + --election_port The port on which the ZooKeeper process will perform + leader election. The default is 3888. + + --server_port The port on which the ZooKeeper process will listen for + requests from other servers in the ensemble. The + default is 2888. + + --tick_time The length of a ZooKeeper tick in ms. The default is + 2000. + + --init_limit The number of Ticks that an ensemble member is allowed + to perform leader election. The default is 10. + + --sync_limit The maximum session timeout that the ensemble will + allows a client to request. The default is 5. + + --percentage The percentage of container memory to give to the JVM. + + --max_client_cnxns The maximum number of client connections that the + ZooKeeper process will accept simultaneously. The + default is 60. + + --snap_retain_count The maximum number of snapshots the ZooKeeper process + will retain if purge_interval is greater than 0. The + default is 3. + + --purge_interval The number of hours the ZooKeeper process will wait + between purging its old snapshots. If set to 0 old + snapshots will never be purged. The default is 0. + + --max_session_timeout The maximum time in milliseconds for a client session + timeout. The default value is 2 * tick time. + + --min_session_timeout The minimum time in milliseconds for a client session + timeout. The default value is 20 * tick time. + + --log_level The log level for the zookeeeper server. Either FATAL, + ERROR, WARN, INFO, DEBUG. The default is INFO. +" +} + +function create_data_dirs() { + if [ ! -d "$DATA_DIR" ]; then + mkdir -p "$DATA_DIR" + chown -R "$USER":"$USER" "$DATA_DIR" + fi + + if [ ! -d "$DATA_LOG_DIR" ]; then + mkdir -p "$DATA_LOG_DIR" + chown -R "$USER":"$USER" "$DATA_LOG_DIR" + fi + + if [ ! -d "$LOG_DIR" ]; then + mkdir -p "$LOG_DIR" + chown -R "$USER":"$USER" "$LOG_DIR" + fi + if [ ! -f "$ID_FILE" ] && [ "$SERVERS" -gt 1 ]; then + echo "$MY_ID" >> "$ID_FILE" + fi +} + +function print_servers() { + for (( i=1; i<=SERVERS; i++ )) + do + echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT" + done +} + +function create_config() { + rm -f "$CONFIG_FILE" + { + echo "#This file was autogenerated DO NOT EDIT" + echo "clientPort=$CLIENT_PORT" + echo "dataDir=$DATA_DIR" + echo "dataLogDir=$DATA_LOG_DIR" + echo "tickTime=$TICK_TIME" + echo "initLimit=$INIT_LIMIT" + echo "syncLimit=$SYNC_LIMIT" + echo "maxClientCnxns=$MAX_CLIENT_CNXNS" + echo "minSessionTimeout=$MIN_SESSION_TIMEOUT" + echo "maxSessionTimeout=$MAX_SESSION_TIMEOUT" + echo "autopurge.snapRetainCount=$SNAP_RETAIN_COUNT" + echo "autopurge.purgeInteval=$PURGE_INTERVAL" + echo "quorumListenOnAllIPs=$QUORUM_LISTEN_ON_ALL_IPS" + # Allow running all zk commands. + echo "4lw.commands.whitelist=*" + if [ "$SERVERS" -gt 1 ]; then + print_servers + fi + echo "metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider" + echo "metricsProvider.httpPort=$PROM_PORT" + } >> "$CONFIG_FILE" + cat "$CONFIG_FILE" >&2 +} + +function create_jvm_props() { + rm -f "$JAVA_ENV_FILE" + { + echo "SERVER_JVMFLAGS=\"-XX:MaxRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \ + -XX:InitialRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT}\"" + echo "ZOO_LOG_DIR=$LOG_DIR" + echo "JVMFLAGS=" + } >> "$JAVA_ENV_FILE" +} + +function create_log_props() { + rm -f "$LOGGER_PROPS_FILE" + echo "Creating ZooKeeper log4j configuration" + { + echo "zookeeper.root.logger=CONSOLE" + echo "zookeeper.console.threshold=$LOG_LEVEL" + echo "log4j.rootLogger=\${zookeeper.root.logger}" + echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender" + echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}" + echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout" + echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n" + } >> "$LOGGER_PROPS_FILE" +} + +optspec=":hv-:" +while getopts "$optspec" optchar; do + + case "${optchar}" in + -) + case "${OPTARG}" in + servers=*) + SERVERS=${OPTARG##*=} + ;; + data_dir=*) + DATA_DIR=${OPTARG##*=} + ;; + data_log_dir=*) + DATA_LOG_DIR=${OPTARG##*=} + ;; + log_dir=*) + LOG_DIR=${OPTARG##*=} + ;; + conf_dir=*) + CONF_DIR=${OPTARG##*=} + ;; + client_port=*) + CLIENT_PORT=${OPTARG##*=} + ;; + election_port=*) + ELECTION_PORT=${OPTARG##*=} + ;; + server_port=*) + SERVER_PORT=${OPTARG##*=} + ;; + tick_time=*) + TICK_TIME=${OPTARG##*=} + ;; + init_limit=*) + INIT_LIMIT=${OPTARG##*=} + ;; + sync_limit=*) + SYNC_LIMIT=${OPTARG##*=} + ;; + percentage=*) + JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT=${OPTARG##*=} + ;; + max_client_cnxns=*) + MAX_CLIENT_CNXNS=${OPTARG##*=} + ;; + snap_retain_count=*) + SNAP_RETAIN_COUNT=${OPTARG##*=} + ;; + purge_interval=*) + PURGE_INTERVAL=${OPTARG##*=} + ;; + max_session_timeout=*) + MAX_SESSION_TIMEOUT=${OPTARG##*=} + ;; + min_session_timeout=*) + MIN_SESSION_TIMEOUT=${OPTARG##*=} + ;; + quorum_listen_on_all_ips=*) + QUORUM_LISTEN_ON_ALL_IPS=${OPTARG##*=} + ;; + log_level=*) + LOG_LEVEL=${OPTARG##*=} + ;; + *) + echo "Unknown option --${OPTARG}" >&2 + exit 1 + ;; + esac;; + h) + print_usage + exit + ;; + v) + echo "Parsing option: '-${optchar}'" >&2 + ;; + *) + if [ "$OPTERR" != 1 ] || [ "${optspec:0:1}" = ":" ]; then + echo "Non-option argument: '-${OPTARG}'" >&2 + fi + ;; + esac +done + +MIN_SESSION_TIMEOUT=${MIN_SESSION_TIMEOUT:- $((TICK_TIME*2))} +MAX_SESSION_TIMEOUT=${MAX_SESSION_TIMEOUT:- $((TICK_TIME*20))} +ID_FILE="$DATA_DIR/myid" +if [ ! -d "$CONF_DIR" ]; then + mkdir -p "$CONF_DIR" + chown -R "$USER":"$USER" "$CONF_DIR" +fi +CONFIG_FILE="$CONF_DIR/zoo.cfg" +LOGGER_PROPS_FILE="$CONF_DIR/log4j.properties" +JAVA_ENV_FILE="$CONF_DIR/java.env" + +if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then + NAME=${BASH_REMATCH[1]} + ORD=${BASH_REMATCH[2]} +else + echo "Failed to parse name and ordinal of Pod" + exit 1 +fi + +MY_ID=$((ORD+1)) + +export ZOOCFGDIR=${CONF_DIR} +create_config && create_jvm_props && create_log_props && create_data_dirs && exec "${ZOOKEEPER_HOME}/bin/zkServer.sh" start-foreground diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/zookeeper.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/zookeeper.yaml new file mode 100644 index 0000000000..b01022972a --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/zookeeper.yaml @@ -0,0 +1,142 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zookeeper +spec: + replicas: 1 + serviceName: zookeeper-headless + selector: + matchLabels: + cluster: zookeeper # has to match .spec.template.metadata.labels + role: zookeeper + updateStrategy: + type: RollingUpdate + podManagementPolicy: Parallel + template: + metadata: + labels: + cluster: zookeeper + role: zookeeper + spec: + containers: + - image: zookeeper + name: zookeeper + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: '0.1' + memory: 768Mi + limits: + cpu: '1.0' + memory: 1Gi + env: + - name: JAVA_HOME + value: /usr/local/openjdk-11 + - name: LANG + value: C.UTF-8 + - name: PATH + value: /usr/local/openjdk-11/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + - name: ZK_LOG_DIR + value: /var/log/zookeeper + ports: + - containerPort: 2181 + name: client + - containerPort: 2888 + name: server + - containerPort: 3888 + name: leader-election + - containerPort: 7001 + name: metrics + command: + - sh + - -c + - |- + export LOG_DIR="${ZK_LOG_DIR}" + /usr/bin/zookeeper/start.sh --servers=1 --percentage=50 + readinessProbe: + exec: + command: + - /bin/bash + - -c + - |- + [ "$(echo ruok | nc 127.0.0.1 2181)" == "imok" ] + initialDelaySeconds: 10 + timeoutSeconds: 5 + livenessProbe: + exec: + command: + - /bin/bash + - -c + - |- + [ "$(echo ruok | nc 127.0.0.1 2181)" == "imok" ] + initialDelaySeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: datadir + mountPath: /var/lib/zookeeper + - mountPath: /usr/bin/zookeeper + name: zookeeper-scripts + - mountPath: /var/log/zookeeper + name: zk-logs + volumes: + - emptyDir: {} + name: zk-logs + - configMap: + name: zookeeper-scripts + defaultMode: 0555 + name: zookeeper-scripts + volumeClaimTemplates: + - metadata: + name: datadir + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 2Gi +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: zookeeper-pdb +spec: + selector: + matchLabels: + cluster: zookeeper + maxUnavailable: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: zookeeper-headless +spec: + ports: + - port: 2888 + name: server + - port: 3888 + name: leader-election + - port: 2181 + name: client + - port: 8080 + name: http + clusterIP: None + publishNotReadyAddresses: true + # Select our zookeeper app. This is what gets us dns entries + # https://kubernetes.io/docs/concepts/services-networking/service/#with-selectors + selector: + cluster: zookeeper diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/00-assert-zookeeper.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/00-assert-zookeeper.yaml new file mode 100644 index 0000000000..4e109f0846 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/00-assert-zookeeper.yaml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Asserts on the ZooKeeper portion of the deployment. +# +--- +# assert that there is a `StatefulSet` named "zookeeper" that has one live instance. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zookeeper +status: + availableReplicas: 1 diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/00-kustomize.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/00-kustomize.yaml new file mode 100644 index 0000000000..cb2588cf03 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/00-kustomize.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep + +commands: +- script: ../../../resources/kustomize_into_tmpdir.sh diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/kustomization.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/kustomization.yaml new file mode 100644 index 0000000000..4f27d9b394 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/kustomization.yaml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +images: +- name: zookeeper + newName: zookeeper + newTag: 3.8.0 + +components: + - ../../../../main/kustomize/components/zookeeper/single-instance diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/00-assert-zookeeper.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/00-assert-zookeeper.yaml new file mode 100644 index 0000000000..41fc2d8581 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/00-assert-zookeeper.yaml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Asserts on the ZooKeeper portion of the deployment. +# +--- +# assert that there is a `ConfigMap` named "zookeeper-scripts-XXX" +# TODO: kuttl does not support generated names +#apiVersion: v1 +#kind: ConfigMap +#metadata: +# name: zookeeper-scripts-c94h8k249d +--- +# assert that there is a `ConfigMap` named "zookeeper-quorum" +apiVersion: v1 +kind: ConfigMap +metadata: + name: zookeeper-quorum +--- +# assert that there is a `PodDisruptionBudget` named "zookeeper-pdb" +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: zookeeper-pdb +--- +# assert that there is a `StatefulSet` named "zookeeper" that: +# - provides pods labeled role:zookeeper +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zookeeper +spec: + template: + metadata: + labels: + role: zookeeper diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/00-kustomize.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/00-kustomize.yaml new file mode 100644 index 0000000000..cb2588cf03 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/00-kustomize.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep + +commands: +- script: ../../../resources/kustomize_into_tmpdir.sh diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/kustomization.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/kustomization.yaml new file mode 100644 index 0000000000..ad0d9e9015 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/kustomization.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../main/kustomize/components/zookeeper/single-instance diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kustomize_into_tmpdir.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kustomize_into_tmpdir.sh new file mode 100755 index 0000000000..7e5cef1048 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kustomize_into_tmpdir.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Test runner using KUTTL against a target cluster. +# https://kuttl.dev +# https://kind.sigs.k8s.io +# +# Materialize a kustomize directory for a kuttl test. +# +# Kustomize is clunky for automated testing. It's pretty opinionated in that it will only evaluate +# a directory off of disk -- you cannot generate a kustomization and pass it in via stdin. +# In order to use kuttl generated namespaces within the kustomization, we have to modify the +# kustomization.yaml before applying it. If we modify that file in the source tree, we end up with +# the test namespace appended to the file under source control. So, this script creates a temp +# directory, copies all the resources into that directory, and modifies the kustomization.yaml as +# necessary. It then runs `kubectl apply -k` against that temporary directory. +# + +declare DEBUG="${DEBUG:false}" +if [ "${DEBUG}" = 'true' ] ; then + set -x +fi + +set -eou pipefail + +declare NAMESPACE +declare NEW_RESOURCES='[]' +declare NEW_COMPONENTS='[]' +declare kustomize_dir +declare -a rewritten_resources=() +declare -a rewritten_components=() + +kustomize_dir="$(mktemp -d -p /tmp "${NAMESPACE}.XXXXXXXXXX")" +trap '[ -d "${kustomize_dir}" ] && rm -rf "${kustomize_dir}"' EXIT + +cp -r ./* "${kustomize_dir}/" + +for r in $(yq '.resources[]' kustomization.yaml) ; do + if [[ "${r}" =~ ^\.\./.* ]] ; then + # resolve the new relative location for any resource path that is not in the local directory + canonized="$(cd "${r}" ; pwd)" + r="../..${canonized}" + fi + rewritten_resources+=("'${r}'") +done +if [ "${#rewritten_resources[@]}" -gt 0 ] ; then + NEW_RESOURCES="[ $(printf '%s,' "${rewritten_resources[@]}") ]" +fi + +for r in $(yq '.components[]' kustomization.yaml) ; do + if [[ "${r}" =~ ^\.\./.* ]] ; then + # resolve the new relative location for any resource path that is not in the local directory + canonized="$(cd "${r}" ; pwd)" + r="../..${canonized}" + fi + rewritten_components+=("'${r}'") +done +if [ "${#rewritten_components[@]}" -gt 0 ] ; then + NEW_COMPONENTS="[ $(printf '%s,' "${rewritten_components[@]}") ]" +fi + +env NAMESPACE="${NAMESPACE}" \ + NEW_RESOURCES="${NEW_RESOURCES}" \ + NEW_COMPONENTS="${NEW_COMPONENTS}" \ + yq -i ' + .namespace = strenv(NAMESPACE) | + .resources = env(NEW_RESOURCES) | + .components = env(NEW_COMPONENTS) +' "${kustomize_dir}/kustomization.yaml" + +if [ "${DEBUG}" = 'true' ] ; then + cat "${kustomize_dir}/kustomization.yaml" +fi + +kubectl apply -k "${kustomize_dir}" diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-integration.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-integration.yaml new file mode 100644 index 0000000000..3329cdf10d --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-integration.yaml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Test runner using KUTTL against a target cluster. +# https://kuttl.dev +# https://kind.sigs.k8s.io +# +# Test runner using KUTTL against a target cluster. +# https://kuttl.dev +# https://kind.sigs.k8s.io +--- +# Does not use Kuttl's built-in KIND support -- it doesn't quite work correctly with a VM-based +# (Docker Desktop) style of runtime. Instead, assumes the cluster is established outside of kuttl +# and configuration is provided via `--env`. +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +testDirs: +- ./src/test/kustomize/integration +timeout: + # these tests allocate several pods with dependencies between them, allow some time for + # everything to launch and settle. + 300 +reportName: TEST-kuttl-report-integration +reportFormat: xml diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-unit.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-unit.yaml new file mode 100644 index 0000000000..d87b2d6691 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-unit.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Test runner using KUTTL against a target cluster. +# https://kuttl.dev +# https://kind.sigs.k8s.io +# Test runner using https://kuttl.dev +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +startControlPlane: true +testDirs: +- ./src/test/kustomize/unit +reportName: TEST-kuttl-report-unit +reportFormat: xml diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/mvn_exec_run_kuttl.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/mvn_exec_run_kuttl.sh new file mode 100755 index 0000000000..e544d65fac --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/mvn_exec_run_kuttl.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Wrap up the complexities of launching `kubectl kuttl test` via docker container. + +set -euox pipefail + +declare default_run_args +default_run_args="--rm --mount type=bind,source=$(pwd),target=/workspace --workdir /workspace" + +# from the calling environment +declare DOCKER_EXE="${DOCKER_EXE:-"$(command -v docker 2>/dev/null)"}" +declare DOCKER_CONTAINER_RUN_ADDITIONAL_ARGS="${DOCKER_CONTAINER_RUN_ADDITIONAL_ARGS:-"${default_run_args}"}" +declare USER="${USER:-apache}" +declare KUTTL_IMAGE="${KUTTL_IMAGE:-"${USER}/hbase/operator-tools/kuttl:latest"}" + +declare run_args +read -r -a run_args <<< "$DOCKER_CONTAINER_RUN_ADDITIONAL_ARGS" + +exec "${DOCKER_EXE}" container run \ + "${run_args[@]}" \ + "${KUTTL_IMAGE}" \ + "$@" diff --git a/hbase-kubernetes-deployment/pom.xml b/hbase-kubernetes-deployment/pom.xml index 22ee23955f..fa82ecfcda 100644 --- a/hbase-kubernetes-deployment/pom.xml +++ b/hbase-kubernetes-deployment/pom.xml @@ -36,6 +36,7 @@ hbase-kubernetes-testing-image + hbase-kubernetes-kustomize