diff --git a/.gitignore b/.gitignore index b440d2b597..35096783dd 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ linklint-*.zip linklint/ .checkstyle **/.checkstyle + +# detritus produced by kuttl +kubeconfig* diff --git a/dev-support/jenkins/Dockerfile b/dev-support/jenkins/Dockerfile index 5b7aac29a9..17bcf30645 100644 --- a/dev-support/jenkins/Dockerfile +++ b/dev-support/jenkins/Dockerfile @@ -20,16 +20,29 @@ FROM hadolint/hadolint:latest-debian as hadolint FROM maven:3.8-jdk-8 - -# hadolint ignore=DL3008 -RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ - binutils \ - git \ - rsync \ - shellcheck \ - wget && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# hadolint ignore=SC1091 +RUN install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ + && chmod a+r /etc/apt/keyrings/docker.gpg \ + && echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-get -q update \ + && apt-get -q install --no-install-recommends -y \ + binutils=* \ + ca-certificates=* \ + curl=* \ + docker-buildx-plugin=* \ + docker-ce-cli=* \ + git=* \ + gnupg=* \ + rsync=* \ + shellcheck=* \ + wget=* \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* COPY --from=hadolint /bin/hadolint /bin/hadolint diff --git a/dev-support/jenkins/Jenkinsfile b/dev-support/jenkins/Jenkinsfile index e7c9773958..adc6150c31 100644 --- a/dev-support/jenkins/Jenkinsfile +++ b/dev-support/jenkins/Jenkinsfile @@ -49,6 +49,34 @@ pipeline { YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}" YETUSDIR = "${WORKDIR}/${YETUS_REL}" PLUGINS = 'all' + // workaround for INFRA-24621 + NON_ROOT_AS_JENKINS = 'true' + NON_ROOT_USER = """${sh( + returnStdout: true, + script: '''#!/usr/bin/env bash +set -e + +if [ "${DEBUG}" = 'true' ] ; then + set -x +fi + +whoami +''' + )} +""".trim() + NON_ROOT_UID = """${sh( + returnStdout: true, + script: '''#!/usr/bin/env bash +set -e + +if [ "${DEBUG}" = 'true' ] ; then + set -x +fi + +echo "${UID}" +''' + )} +""".trim() } parameters { @@ -78,7 +106,9 @@ pipeline { usernameVariable: 'GITHUB_USER' )]) { sh label: 'test-patch', script: '''#!/bin/bash -e + docker container run --rm --user=0 --mount type=bind,source=${SOURCEDIR},target=/sourcedir --workdir /sourcedir alpine:latest rm -rfv hbase-kubernetes-deployment/hbase-kubernetes-kustomize/target printenv 2>&1 | sort + export DOCKER_EXTRAENVS=("NON_ROOT_AS_JENKINS" "NON_ROOT_USER" "NON_ROOT_UID" ) echo "[INFO] Launching Yetus via ${YETUS_DRIVER}" "${YETUS_DRIVER}" ''' @@ -138,4 +168,4 @@ pipeline { } } } -} \ No newline at end of file +} diff --git a/dev-support/jenkins/jenkins_precommit_github_yetus.sh b/dev-support/jenkins/jenkins_precommit_github_yetus.sh index 2cede2f047..40f3ead1a9 100755 --- a/dev-support/jenkins/jenkins_precommit_github_yetus.sh +++ b/dev-support/jenkins/jenkins_precommit_github_yetus.sh @@ -111,6 +111,8 @@ YETUS_ARGS+=("--tests-filter=test4tests") # Dockerfile since we don't want to use the auto-pulled version. YETUS_ARGS+=("--docker") YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") +# enabled docker-in-docker so that we can build container images +YETUS_ARGS+=("--dockerind=true") YETUS_ARGS+=("--mvn-custom-repos") YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}") # effectively treat dev-support as a custom maven module @@ -127,4 +129,4 @@ YETUS_ARGS+=("--proclimit=5000") echo "Launching yetus with command line:" echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}" -/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" \ No newline at end of file +/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/pom.xml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/pom.xml new file mode 100644 index 0000000000..d30f7a8498 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/pom.xml @@ -0,0 +1,91 @@ + + + + 4.0.0 + + hbase-kubernetes-deployment + org.apache.hbase.operator.tools + ${revision} + .. + + + hbase-kubernetes-kustomize + Apache HBase - Kubernetes Kustomize + Kustomize deployment descriptors. + pom + + + + org.apache.hbase.operator.tools + hbase-kubernetes-testing-image + pom + test + + + + + + + org.codehaus.mojo + exec-maven-plugin + + src/test/resources/mvn_exec_run_kuttl.sh + true + ${skipTests} + + + + kuttl-unit-tests + test + + exec + + + + --config + src/test/resources/kuttl-test-unit.yaml + --artifacts-dir + target/kuttl-reports + + + + + kuttl-integration-tests + verify + + exec + + + + --config + src/test/resources/kuttl-test-integration.yaml + --artifacts-dir + target/kuttl-reports + + + + + + + + diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/README.md b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/README.md new file mode 100644 index 0000000000..b6af185a87 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/README.md @@ -0,0 +1,23 @@ + + +Uses latest image from the apache zookeeper project. +There is then a start script in 'single-instance/start.sh' +which sets parameters for the zookeeper image and process +on startup. Currently only logs to STDOUT/STDERR; there +are no files in /var/log/zookeeper. diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/kustomization.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/kustomization.yaml new file mode 100644 index 0000000000..1e87468cd7 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/kustomization.yaml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +configMapGenerator: +- name: zookeeper-scripts + files: + - start.sh +# Publish where the zk ensemble can be found. +- name: zookeeper-quorum + literals: + # Hard-coded. Default we expect a simple standalone zk at this location. + # One define is for hbase, the other for hadoop. + - HBASE_ZOOKEEPER_QUORUM="zookeeper-0.zookeeper-headless" + - HA_ZOOKEEPER_QUORUM="zookeeper-0.zookeeper-headless:2181" + options: + disableNameSuffixHash: true + +resources: +- zookeeper.yaml diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/start.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/start.sh new file mode 100755 index 0000000000..48b3d91543 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/start.sh @@ -0,0 +1,358 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Derived from work Copyright 2017 The Kubernetes Authors. +# See https://github.com/kow3ns/kubernetes-zookeeper/blob/master/docker/scripts/start-zookeeper for more details +# and then https://github.com/cloudurable/kube-zookeeper-statefulsets/ +# In the below we change the '--heap' argument to '--percentage' so +# could set the server heap as a percentage of the container resource +# limit rather than hard-code it. +# Currently zookeeper.root.logger is CONSOLE only. We do not write +# logs to files. Fix config. if you need it. +# +# +# Usage: start-zookeeper [OPTIONS] +# Starts a ZooKeeper server based on the supplied options. +# --servers The number of servers in the ensemble. The default +# value is 1. +# --data_dir The directory where the ZooKeeper process will store its +# snapshots. The default is /var/lib/zookeeper/data. +# --data_log_dir The directory where the ZooKeeper process will store its +# write ahead log. The default is +# /var/lib/zookeeper/data/log. +# --conf_dir The directory where the ZooKeeper process will store its +# configuration. The default is /opt/zookeeper/conf. +# --client_port The port on which the ZooKeeper process will listen for +# client requests. The default is 2181. + +# --election_port The port on which the ZooKeeper process will perform +# leader election. The default is 3888. + +# --server_port The port on which the ZooKeeper process will listen for +# requests from other servers in the ensemble. The +# default is 2888. + +# --tick_time The length of a ZooKeeper tick in ms. The default is +# 2000. + +# --init_limit The number of Ticks that an ensemble member is allowed +# to perform leader election. The default is 10. + +# --sync_limit The maximum session timeout that the ensemble will +# allows a client to request. The default is 5. + +# --percentage The percentage of container memory to give to the JVM. + +# --max_client_cnxns The maximum number of client connections that the +# ZooKeeper process will accept simultaneously. The +# default is 60. + +# --snap_retain_count The maximum number of snapshots the ZooKeeper process +# will retain if purge_interval is greater than 0. The +# default is 3. + +# --purge_interval The number of hours the ZooKeeper process will wait +# between purging its old snapshots. If set to 0 old +# snapshots will never be purged. The default is 0. + +# --max_session_timeout The maximum time in milliseconds for a client session +# timeout. The default value is 2 * tick time. + +# --min_session_timeout The minimum time in milliseconds for a client session +# timeout. The default value is 20 * tick time. + +# --log_level The log level for the zookeeeper server. Either FATAL, +# ERROR, WARN, INFO, DEBUG. The default is INFO. + +# --quorum_listen_on_all_ips +# When set to true the ZooKeeper server will listen for +# connections from its peers on all available IP addresses, +# and not only the address configured in the server list of +# the configuration file. It affects the connections handling +# the ZAB protocol and the Fast Leader Election protocol. +# Default value is false. +set -x + +ZOOKEEPER_HOME="$( ls -d /apache-zookeeper* )" +USER="$(whoami)" +HOST="$(hostname -s)" +DOMAIN="$(hostname -d)" +LOG_LEVEL=INFO +DATA_DIR="/var/lib/zookeeper/data" +DATA_LOG_DIR="/var/lib/zookeeper/log" +LOG_DIR="/var/log/zookeeper" +CONF_DIR="/opt/zookeeper/conf" +CLIENT_PORT=2181 +SERVER_PORT=2888 +ELECTION_PORT=3888 +PROM_PORT=7001 +TICK_TIME=2000 +INIT_LIMIT=10 +SYNC_LIMIT=5 +JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT=50 +MAX_CLIENT_CNXNS=1000 +SNAP_RETAIN_COUNT=3 +PURGE_INTERVAL=0 +SERVERS=1 +QUORUM_LISTEN_ON_ALL_IPS=false + +function print_usage() { +echo "\ +Usage: start-zookeeper [OPTIONS] +Starts a ZooKeeper server based on the supplied options. + --servers The number of servers in the ensemble. The default + value is 1. + + --data_dir The directory where the ZooKeeper process will store its + snapshots. The default is /var/lib/zookeeper/data. + + --data_log_dir The directory where the ZooKeeper process will store its + write ahead log. The default is + /var/lib/zookeeper/data/log. + + --conf_dir The directoyr where the ZooKeeper process will store its + configuration. The default is /opt/zookeeper/conf. + + --client_port The port on which the ZooKeeper process will listen for + client requests. The default is 2181. + + --election_port The port on which the ZooKeeper process will perform + leader election. The default is 3888. + + --server_port The port on which the ZooKeeper process will listen for + requests from other servers in the ensemble. The + default is 2888. + + --tick_time The length of a ZooKeeper tick in ms. The default is + 2000. + + --init_limit The number of Ticks that an ensemble member is allowed + to perform leader election. The default is 10. + + --sync_limit The maximum session timeout that the ensemble will + allows a client to request. The default is 5. + + --percentage The percentage of container memory to give to the JVM. + + --max_client_cnxns The maximum number of client connections that the + ZooKeeper process will accept simultaneously. The + default is 60. + + --snap_retain_count The maximum number of snapshots the ZooKeeper process + will retain if purge_interval is greater than 0. The + default is 3. + + --purge_interval The number of hours the ZooKeeper process will wait + between purging its old snapshots. If set to 0 old + snapshots will never be purged. The default is 0. + + --max_session_timeout The maximum time in milliseconds for a client session + timeout. The default value is 2 * tick time. + + --min_session_timeout The minimum time in milliseconds for a client session + timeout. The default value is 20 * tick time. + + --log_level The log level for the zookeeeper server. Either FATAL, + ERROR, WARN, INFO, DEBUG. The default is INFO. +" +} + +function create_data_dirs() { + if [ ! -d "$DATA_DIR" ]; then + mkdir -p "$DATA_DIR" + chown -R "$USER":"$USER" "$DATA_DIR" + fi + + if [ ! -d "$DATA_LOG_DIR" ]; then + mkdir -p "$DATA_LOG_DIR" + chown -R "$USER":"$USER" "$DATA_LOG_DIR" + fi + + if [ ! -d "$LOG_DIR" ]; then + mkdir -p "$LOG_DIR" + chown -R "$USER":"$USER" "$LOG_DIR" + fi + if [ ! -f "$ID_FILE" ] && [ "$SERVERS" -gt 1 ]; then + echo "$MY_ID" >> "$ID_FILE" + fi +} + +function print_servers() { + for (( i=1; i<=SERVERS; i++ )) + do + echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT" + done +} + +function create_config() { + rm -f "$CONFIG_FILE" + { + echo "#This file was autogenerated DO NOT EDIT" + echo "clientPort=$CLIENT_PORT" + echo "dataDir=$DATA_DIR" + echo "dataLogDir=$DATA_LOG_DIR" + echo "tickTime=$TICK_TIME" + echo "initLimit=$INIT_LIMIT" + echo "syncLimit=$SYNC_LIMIT" + echo "maxClientCnxns=$MAX_CLIENT_CNXNS" + echo "minSessionTimeout=$MIN_SESSION_TIMEOUT" + echo "maxSessionTimeout=$MAX_SESSION_TIMEOUT" + echo "autopurge.snapRetainCount=$SNAP_RETAIN_COUNT" + echo "autopurge.purgeInteval=$PURGE_INTERVAL" + echo "quorumListenOnAllIPs=$QUORUM_LISTEN_ON_ALL_IPS" + # Allow running all zk commands. + echo "4lw.commands.whitelist=*" + if [ "$SERVERS" -gt 1 ]; then + print_servers + fi + echo "metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider" + echo "metricsProvider.httpPort=$PROM_PORT" + } >> "$CONFIG_FILE" + cat "$CONFIG_FILE" >&2 +} + +function create_jvm_props() { + rm -f "$JAVA_ENV_FILE" + { + echo "SERVER_JVMFLAGS=\"-XX:MaxRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT} \ + -XX:InitialRAMPercentage=${JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT}\"" + echo "ZOO_LOG_DIR=$LOG_DIR" + echo "JVMFLAGS=" + } >> "$JAVA_ENV_FILE" +} + +function create_log_props() { + rm -f "$LOGGER_PROPS_FILE" + echo "Creating ZooKeeper log4j configuration" + { + echo "zookeeper.root.logger=CONSOLE" + echo "zookeeper.console.threshold=$LOG_LEVEL" + echo "log4j.rootLogger=\${zookeeper.root.logger}" + echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender" + echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}" + echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout" + echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n" + } >> "$LOGGER_PROPS_FILE" +} + +optspec=":hv-:" +while getopts "$optspec" optchar; do + + case "${optchar}" in + -) + case "${OPTARG}" in + servers=*) + SERVERS=${OPTARG##*=} + ;; + data_dir=*) + DATA_DIR=${OPTARG##*=} + ;; + data_log_dir=*) + DATA_LOG_DIR=${OPTARG##*=} + ;; + log_dir=*) + LOG_DIR=${OPTARG##*=} + ;; + conf_dir=*) + CONF_DIR=${OPTARG##*=} + ;; + client_port=*) + CLIENT_PORT=${OPTARG##*=} + ;; + election_port=*) + ELECTION_PORT=${OPTARG##*=} + ;; + server_port=*) + SERVER_PORT=${OPTARG##*=} + ;; + tick_time=*) + TICK_TIME=${OPTARG##*=} + ;; + init_limit=*) + INIT_LIMIT=${OPTARG##*=} + ;; + sync_limit=*) + SYNC_LIMIT=${OPTARG##*=} + ;; + percentage=*) + JVM_HEAP_PERCENTAGE_OF_RESOURCE_LIMIT=${OPTARG##*=} + ;; + max_client_cnxns=*) + MAX_CLIENT_CNXNS=${OPTARG##*=} + ;; + snap_retain_count=*) + SNAP_RETAIN_COUNT=${OPTARG##*=} + ;; + purge_interval=*) + PURGE_INTERVAL=${OPTARG##*=} + ;; + max_session_timeout=*) + MAX_SESSION_TIMEOUT=${OPTARG##*=} + ;; + min_session_timeout=*) + MIN_SESSION_TIMEOUT=${OPTARG##*=} + ;; + quorum_listen_on_all_ips=*) + QUORUM_LISTEN_ON_ALL_IPS=${OPTARG##*=} + ;; + log_level=*) + LOG_LEVEL=${OPTARG##*=} + ;; + *) + echo "Unknown option --${OPTARG}" >&2 + exit 1 + ;; + esac;; + h) + print_usage + exit + ;; + v) + echo "Parsing option: '-${optchar}'" >&2 + ;; + *) + if [ "$OPTERR" != 1 ] || [ "${optspec:0:1}" = ":" ]; then + echo "Non-option argument: '-${OPTARG}'" >&2 + fi + ;; + esac +done + +MIN_SESSION_TIMEOUT=${MIN_SESSION_TIMEOUT:- $((TICK_TIME*2))} +MAX_SESSION_TIMEOUT=${MAX_SESSION_TIMEOUT:- $((TICK_TIME*20))} +ID_FILE="$DATA_DIR/myid" +if [ ! -d "$CONF_DIR" ]; then + mkdir -p "$CONF_DIR" + chown -R "$USER":"$USER" "$CONF_DIR" +fi +CONFIG_FILE="$CONF_DIR/zoo.cfg" +LOGGER_PROPS_FILE="$CONF_DIR/log4j.properties" +JAVA_ENV_FILE="$CONF_DIR/java.env" + +if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then + NAME=${BASH_REMATCH[1]} + ORD=${BASH_REMATCH[2]} +else + echo "Failed to parse name and ordinal of Pod" + exit 1 +fi + +MY_ID=$((ORD+1)) + +export ZOOCFGDIR=${CONF_DIR} +create_config && create_jvm_props && create_log_props && create_data_dirs && exec "${ZOOKEEPER_HOME}/bin/zkServer.sh" start-foreground diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/zookeeper.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/zookeeper.yaml new file mode 100644 index 0000000000..b01022972a --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/main/kustomize/components/zookeeper/single-instance/zookeeper.yaml @@ -0,0 +1,142 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zookeeper +spec: + replicas: 1 + serviceName: zookeeper-headless + selector: + matchLabels: + cluster: zookeeper # has to match .spec.template.metadata.labels + role: zookeeper + updateStrategy: + type: RollingUpdate + podManagementPolicy: Parallel + template: + metadata: + labels: + cluster: zookeeper + role: zookeeper + spec: + containers: + - image: zookeeper + name: zookeeper + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: '0.1' + memory: 768Mi + limits: + cpu: '1.0' + memory: 1Gi + env: + - name: JAVA_HOME + value: /usr/local/openjdk-11 + - name: LANG + value: C.UTF-8 + - name: PATH + value: /usr/local/openjdk-11/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + - name: ZK_LOG_DIR + value: /var/log/zookeeper + ports: + - containerPort: 2181 + name: client + - containerPort: 2888 + name: server + - containerPort: 3888 + name: leader-election + - containerPort: 7001 + name: metrics + command: + - sh + - -c + - |- + export LOG_DIR="${ZK_LOG_DIR}" + /usr/bin/zookeeper/start.sh --servers=1 --percentage=50 + readinessProbe: + exec: + command: + - /bin/bash + - -c + - |- + [ "$(echo ruok | nc 127.0.0.1 2181)" == "imok" ] + initialDelaySeconds: 10 + timeoutSeconds: 5 + livenessProbe: + exec: + command: + - /bin/bash + - -c + - |- + [ "$(echo ruok | nc 127.0.0.1 2181)" == "imok" ] + initialDelaySeconds: 10 + timeoutSeconds: 5 + volumeMounts: + - name: datadir + mountPath: /var/lib/zookeeper + - mountPath: /usr/bin/zookeeper + name: zookeeper-scripts + - mountPath: /var/log/zookeeper + name: zk-logs + volumes: + - emptyDir: {} + name: zk-logs + - configMap: + name: zookeeper-scripts + defaultMode: 0555 + name: zookeeper-scripts + volumeClaimTemplates: + - metadata: + name: datadir + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 2Gi +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: zookeeper-pdb +spec: + selector: + matchLabels: + cluster: zookeeper + maxUnavailable: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: zookeeper-headless +spec: + ports: + - port: 2888 + name: server + - port: 3888 + name: leader-election + - port: 2181 + name: client + - port: 8080 + name: http + clusterIP: None + publishNotReadyAddresses: true + # Select our zookeeper app. This is what gets us dns entries + # https://kubernetes.io/docs/concepts/services-networking/service/#with-selectors + selector: + cluster: zookeeper diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/00-assert-zookeeper.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/00-assert-zookeeper.yaml new file mode 100644 index 0000000000..4e109f0846 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/00-assert-zookeeper.yaml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Asserts on the ZooKeeper portion of the deployment. +# +--- +# assert that there is a `StatefulSet` named "zookeeper" that has one live instance. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zookeeper +status: + availableReplicas: 1 diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/00-kustomize.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/00-kustomize.yaml new file mode 100644 index 0000000000..cb2588cf03 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/00-kustomize.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep + +commands: +- script: ../../../resources/kustomize_into_tmpdir.sh diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/kustomization.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/kustomization.yaml new file mode 100644 index 0000000000..4f27d9b394 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/integration/components_zookeeper_single/kustomization.yaml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +images: +- name: zookeeper + newName: zookeeper + newTag: 3.8.0 + +components: + - ../../../../main/kustomize/components/zookeeper/single-instance diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/00-assert-zookeeper.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/00-assert-zookeeper.yaml new file mode 100644 index 0000000000..41fc2d8581 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/00-assert-zookeeper.yaml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Asserts on the ZooKeeper portion of the deployment. +# +--- +# assert that there is a `ConfigMap` named "zookeeper-scripts-XXX" +# TODO: kuttl does not support generated names +#apiVersion: v1 +#kind: ConfigMap +#metadata: +# name: zookeeper-scripts-c94h8k249d +--- +# assert that there is a `ConfigMap` named "zookeeper-quorum" +apiVersion: v1 +kind: ConfigMap +metadata: + name: zookeeper-quorum +--- +# assert that there is a `PodDisruptionBudget` named "zookeeper-pdb" +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: zookeeper-pdb +--- +# assert that there is a `StatefulSet` named "zookeeper" that: +# - provides pods labeled role:zookeeper +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zookeeper +spec: + template: + metadata: + labels: + role: zookeeper diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/00-kustomize.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/00-kustomize.yaml new file mode 100644 index 0000000000..cb2588cf03 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/00-kustomize.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep + +commands: +- script: ../../../resources/kustomize_into_tmpdir.sh diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/kustomization.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/kustomization.yaml new file mode 100644 index 0000000000..ad0d9e9015 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/kustomize/unit/components_zookeeper_single/kustomization.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../main/kustomize/components/zookeeper/single-instance diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kustomize_into_tmpdir.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kustomize_into_tmpdir.sh new file mode 100755 index 0000000000..7e5cef1048 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kustomize_into_tmpdir.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Test runner using KUTTL against a target cluster. +# https://kuttl.dev +# https://kind.sigs.k8s.io +# +# Materialize a kustomize directory for a kuttl test. +# +# Kustomize is clunky for automated testing. It's pretty opinionated in that it will only evaluate +# a directory off of disk -- you cannot generate a kustomization and pass it in via stdin. +# In order to use kuttl generated namespaces within the kustomization, we have to modify the +# kustomization.yaml before applying it. If we modify that file in the source tree, we end up with +# the test namespace appended to the file under source control. So, this script creates a temp +# directory, copies all the resources into that directory, and modifies the kustomization.yaml as +# necessary. It then runs `kubectl apply -k` against that temporary directory. +# + +declare DEBUG="${DEBUG:false}" +if [ "${DEBUG}" = 'true' ] ; then + set -x +fi + +set -eou pipefail + +declare NAMESPACE +declare NEW_RESOURCES='[]' +declare NEW_COMPONENTS='[]' +declare kustomize_dir +declare -a rewritten_resources=() +declare -a rewritten_components=() + +kustomize_dir="$(mktemp -d -p /tmp "${NAMESPACE}.XXXXXXXXXX")" +trap '[ -d "${kustomize_dir}" ] && rm -rf "${kustomize_dir}"' EXIT + +cp -r ./* "${kustomize_dir}/" + +for r in $(yq '.resources[]' kustomization.yaml) ; do + if [[ "${r}" =~ ^\.\./.* ]] ; then + # resolve the new relative location for any resource path that is not in the local directory + canonized="$(cd "${r}" ; pwd)" + r="../..${canonized}" + fi + rewritten_resources+=("'${r}'") +done +if [ "${#rewritten_resources[@]}" -gt 0 ] ; then + NEW_RESOURCES="[ $(printf '%s,' "${rewritten_resources[@]}") ]" +fi + +for r in $(yq '.components[]' kustomization.yaml) ; do + if [[ "${r}" =~ ^\.\./.* ]] ; then + # resolve the new relative location for any resource path that is not in the local directory + canonized="$(cd "${r}" ; pwd)" + r="../..${canonized}" + fi + rewritten_components+=("'${r}'") +done +if [ "${#rewritten_components[@]}" -gt 0 ] ; then + NEW_COMPONENTS="[ $(printf '%s,' "${rewritten_components[@]}") ]" +fi + +env NAMESPACE="${NAMESPACE}" \ + NEW_RESOURCES="${NEW_RESOURCES}" \ + NEW_COMPONENTS="${NEW_COMPONENTS}" \ + yq -i ' + .namespace = strenv(NAMESPACE) | + .resources = env(NEW_RESOURCES) | + .components = env(NEW_COMPONENTS) +' "${kustomize_dir}/kustomization.yaml" + +if [ "${DEBUG}" = 'true' ] ; then + cat "${kustomize_dir}/kustomization.yaml" +fi + +kubectl apply -k "${kustomize_dir}" diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-integration.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-integration.yaml new file mode 100644 index 0000000000..3329cdf10d --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-integration.yaml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Test runner using KUTTL against a target cluster. +# https://kuttl.dev +# https://kind.sigs.k8s.io +# +# Test runner using KUTTL against a target cluster. +# https://kuttl.dev +# https://kind.sigs.k8s.io +--- +# Does not use Kuttl's built-in KIND support -- it doesn't quite work correctly with a VM-based +# (Docker Desktop) style of runtime. Instead, assumes the cluster is established outside of kuttl +# and configuration is provided via `--env`. +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +testDirs: +- ./src/test/kustomize/integration +timeout: + # these tests allocate several pods with dependencies between them, allow some time for + # everything to launch and settle. + 300 +reportName: TEST-kuttl-report-integration +reportFormat: xml diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-unit.yaml b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-unit.yaml new file mode 100644 index 0000000000..d87b2d6691 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/kuttl-test-unit.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Test runner using KUTTL against a target cluster. +# https://kuttl.dev +# https://kind.sigs.k8s.io +# Test runner using https://kuttl.dev +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +startControlPlane: true +testDirs: +- ./src/test/kustomize/unit +reportName: TEST-kuttl-report-unit +reportFormat: xml diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/mvn_exec_run_kuttl.sh b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/mvn_exec_run_kuttl.sh new file mode 100755 index 0000000000..e544d65fac --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-kustomize/src/test/resources/mvn_exec_run_kuttl.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Wrap up the complexities of launching `kubectl kuttl test` via docker container. + +set -euox pipefail + +declare default_run_args +default_run_args="--rm --mount type=bind,source=$(pwd),target=/workspace --workdir /workspace" + +# from the calling environment +declare DOCKER_EXE="${DOCKER_EXE:-"$(command -v docker 2>/dev/null)"}" +declare DOCKER_CONTAINER_RUN_ADDITIONAL_ARGS="${DOCKER_CONTAINER_RUN_ADDITIONAL_ARGS:-"${default_run_args}"}" +declare USER="${USER:-apache}" +declare KUTTL_IMAGE="${KUTTL_IMAGE:-"${USER}/hbase/operator-tools/kuttl:latest"}" + +declare run_args +read -r -a run_args <<< "$DOCKER_CONTAINER_RUN_ADDITIONAL_ARGS" + +exec "${DOCKER_EXE}" container run \ + "${run_args[@]}" \ + "${KUTTL_IMAGE}" \ + "$@" diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/README.md b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/README.md new file mode 100644 index 0000000000..31755c0a12 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/README.md @@ -0,0 +1,70 @@ + + +# hbase-kubernetes-testing-image + +This directory builds a docker image containing everything required to run `kubectl-kuttl` in +"mocked control plane" mode. This image is used as the basis for both dev and test environments. + +## Build + +Building the docker image locally is a little picky because there's lots of input arguments. These +are managed via the [docker-bake.override.hcl](./src/main/docker/docker-bake.override.hcl). + +Start by creating a buildx context that supports (optionally) multi-platform images. If you've +created this context previously, it's enough to ensure that it's active via `docker buildx ls`. + +```shell +$ docker buildx create \ + --driver docker-container \ + --platform linux/amd64,linux/arm64 \ + --use \ + --bootstrap +``` + +Finally, build the image using `maven package`, or manually, using, + +```shell +$ docker buildx bake \ + --file src/main/docker/docker-bake.hcl \ + --file src/main/docker/docker-bake.override.hcl \ + --pull \ + --load +``` + +This exports an image to your local repository that is tagged as `${USER}/hbase/operator-tools/kuttl:latest`. + +## Usage + +The image is configured with `kuttle` as the entrypoint. + +```shell +$ docker container run --rm -it ${USER}/hbase/operator-tools/kuttl:latest --help + +``` + +Running tests in the image requires mounting the workspace into the container image and passing +appropriate parameters to `kuttl`. For example, run the "small" tests like this: + +```shell +$ docker container run \ + --mount type=bind,source=$(pwd),target=/workspace \ + --workdir /workspace \ + ${USER}/hbase/operator-tools/kuttl:latest \ + --config tests/kuttl-test-small.yaml +``` diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/pom.xml b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/pom.xml new file mode 100644 index 0000000000..3596697eb9 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/pom.xml @@ -0,0 +1,126 @@ + + + + 4.0.0 + + hbase-kubernetes-deployment + org.apache.hbase.operator.tools + ${revision} + .. + + + hbase-kubernetes-testing-image + Apache HBase - Kubernetes Testing Image + A container image to facilitate testing of Kubernetes Deployment. + pom + + + linux/amd64 + + + + + + + + org.codehaus.mojo + exec-maven-plugin + + true + + + + docker-buildx-bake-print + + exec + + package + + /usr/bin/env + + sh + -c + + 2>&1 \ + src/build/shell/exec_docker_buildx_bake.sh \ + --print \ + --file src/main/docker/docker-bake.hcl \ + --file src/main/docker/docker-bake.override.hcl + + + + + + docker-buildx-bake + + exec + + package + + /usr/bin/env + + sh + -c + + 2>&1 \ + src/build/shell/exec_docker_buildx_bake.sh \ + --progress plain \ + --pull \ + --load \ + --set *.platform=${container_image.platforms} \ + --file src/main/docker/docker-bake.hcl \ + --file src/main/docker/docker-bake.override.hcl + + + + + + + + + + + + amd64 + + + x86_64 + + + + linux/amd64 + + + + arm64 + + + aarch64 + + + + linux/arm64 + + + + diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/build/shell/exec_docker_buildx_bake.sh b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/build/shell/exec_docker_buildx_bake.sh new file mode 100755 index 0000000000..de9083e5f3 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/build/shell/exec_docker_buildx_bake.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A work-around for INFRA-24621 establishing container user name and id in the jenkins environment. +# When `NON_ROOT_AS_JENKINS=true`, injects the values of `NON_ROOT_USER` and `NON_ROOT_UID` into +# the bake environment. + +set -e +set -o pipefail +set -x + +declare NON_ROOT_AS_JENKINS +declare NON_ROOT_USER +declare NON_ROOT_UID +declare -a bake_args=( "$@" ) + +NON_ROOT_AS_JENKINS="${NON_ROOT_AS_JENKINS:-false}" + +if [ "${NON_ROOT_AS_JENKINS}" = 'true' ] ; then + bake_args+=( "--set=*.args.NON_ROOT_USER=${NON_ROOT_USER}" ) + bake_args+=( "--set=*.args.NON_ROOT_USER_ID=${NON_ROOT_UID}" ) +fi + +docker buildx bake "${bake_args[@]}" diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/Dockerfile b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/Dockerfile new file mode 100644 index 0000000000..5bdbff0cc3 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/Dockerfile @@ -0,0 +1,164 @@ +# syntax=docker/dockerfile:1.4 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# hadolint global ignore=DL3020 + +ARG IMG_BASE='golang' +ARG IMG_BASE_TAG='1.19-alpine' +ARG BASE_IMG_ALPINE='alpine' +ARG BASE_IMG_TAG_ALPINE='latest' +ARG YQ_IMG='mikefarah/yq' +ARG YQ_IMG_TAG='latest' + +FROM ${BASE_IMG_ALPINE}:${BASE_IMG_TAG_ALPINE} as alpine +RUN apk add --no-cache bash~=5 + +FROM ${YQ_IMG}:${YQ_IMG_TAG} as yq + +## -- Stages kubectl_${TARGETARCH} -- +# Define stages that facilitate bringing in platform-specific binaries. +FROM alpine as kubectl_amd64 +ARG KUBECTL_SHA_AMD64_URL +ARG KUBECTL_SHA_AMD64 +ARG KUBECTL_BIN_AMD64_URL +ARG KUBECTL_BIN_AMD64 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +WORKDIR /tmp +ADD --link ${KUBECTL_SHA_AMD64_URL} /tmp/ +ADD --link ${KUBECTL_BIN_AMD64_URL} /tmp/ +RUN echo "$(cat "${KUBECTL_SHA_AMD64}") ${KUBECTL_BIN_AMD64}" | sha512sum -c +ENV KUBECTL_BIN "${KUBECTL_BIN_AMD64}" + +FROM alpine as kubectl_arm64 +ARG KUBECTL_SHA_ARM64_URL +ARG KUBECTL_SHA_ARM64 +ARG KUBECTL_BIN_ARM64_URL +ARG KUBECTL_BIN_ARM64 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +WORKDIR /tmp +ADD --link ${KUBECTL_SHA_ARM64_URL} /tmp/ +ADD --link ${KUBECTL_BIN_ARM64_URL} /tmp/ +RUN echo "$(cat "${KUBECTL_SHA_ARM64}") ${KUBECTL_BIN_ARM64}" | sha512sum -c +ENV KUBECTL_BIN "${KUBECTL_BIN_ARM64}" + +ARG TARGETARCH +# hadolint ignore=DL3006 +FROM kubectl_${TARGETARCH} as kubectl +RUN mv "/tmp/${KUBECTL_BIN}" /tmp/kubectl \ + && chmod a+x /tmp/kubectl + +## -- Stages kuttl_${TARGETARCH} -- +# Define stages that facilitate bringing in platform-specific binaries. +FROM alpine as kuttl_amd64 +ARG KUTTL_CHECKSUMS_URL +ARG KUTTL_CHECKSUMS +ARG KUTTL_BIN_AMD64_URL +ARG KUTTL_BIN_AMD64 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +WORKDIR /tmp +ADD --link ${KUTTL_CHECKSUMS_URL} /tmp/ +ADD --link ${KUTTL_BIN_AMD64_URL} /tmp/ +RUN sha256sum -c <(grep "${KUTTL_BIN_AMD64}" "${KUTTL_CHECKSUMS}") +ENV KUTTL_BIN "${KUTTL_BIN_AMD64}" + +FROM alpine as kuttl_arm64 +ARG KUTTL_CHECKSUMS_URL +ARG KUTTL_CHECKSUMS +ARG KUTTL_BIN_ARM64_URL +ARG KUTTL_BIN_ARM64 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +WORKDIR /tmp +ADD --link ${KUTTL_CHECKSUMS_URL} /tmp/ +ADD --link ${KUTTL_BIN_ARM64_URL} /tmp/ +RUN sha256sum -c <(grep "${KUTTL_BIN_ARM64}" "${KUTTL_CHECKSUMS}") +ENV KUTTL_BIN "${KUTTL_BIN_ARM64}" + +ARG TARGETARCH +# hadolint ignore=DL3006 +FROM kuttl_${TARGETARCH} as kuttl +RUN mv "/tmp/${KUTTL_BIN}" /tmp/kubectl-kuttl \ + && chmod a+x /tmp/kubectl-kuttl + +## -- Stages kustomize_${TARGETARCH} -- +# Define stages that facilitate bringing in platform-specific binaries. +FROM alpine as kustomize_amd64 +ARG KUSTOMIZE_CHECKSUMS_URL +ARG KUSTOMIZE_CHECKSUMS +ARG KUSTOMIZE_BIN_AMD64_TGZ_URL +ARG KUSTOMIZE_BIN_AMD64_TGZ +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +WORKDIR /tmp +ADD --link ${KUSTOMIZE_CHECKSUMS_URL} /tmp/ +ADD --link ${KUSTOMIZE_BIN_AMD64_TGZ_URL} /tmp/ +RUN sha256sum -c <(grep "${KUSTOMIZE_BIN_AMD64_TGZ}" "${KUSTOMIZE_CHECKSUMS}") +ENV KUSTOMIZE_BIN_TGZ "${KUSTOMIZE_BIN_AMD64_TGZ}" + +FROM alpine as kustomize_arm64 +ARG KUSTOMIZE_CHECKSUMS_URL +ARG KUSTOMIZE_CHECKSUMS +ARG KUSTOMIZE_BIN_ARM64_TGZ_URL +ARG KUSTOMIZE_BIN_ARM64_TGZ +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +WORKDIR /tmp +ADD --link ${KUSTOMIZE_CHECKSUMS_URL} /tmp/ +ADD --link ${KUSTOMIZE_BIN_ARM64_TGZ_URL} /tmp/ +RUN sha256sum -c <(grep "${KUSTOMIZE_BIN_ARM64_TGZ}" "${KUSTOMIZE_CHECKSUMS}") +ENV KUSTOMIZE_BIN_TGZ "${KUSTOMIZE_BIN_ARM64_TGZ}" + +ARG TARGETARCH +# hadolint ignore=DL3006 +FROM kustomize_${TARGETARCH} as kustomize +RUN tar xzf "/tmp/${KUSTOMIZE_BIN_TGZ}" \ + && chmod a+x /tmp/kustomize + +FROM ${IMG_BASE}:${IMG_BASE_TAG} as final +ARG IMG_BASE +ARG IMG_BASE_TAG +ARG NON_ROOT_USER +ARG NON_ROOT_USER_ID +ARG NON_ROOT_USER_HOME + +COPY --from=yq /usr/bin/yq /usr/bin/yq +COPY --from=kubectl /tmp/kubectl /usr/local/bin/ +COPY --from=kuttl /tmp/kubectl-kuttl /usr/local/bin/ +COPY --from=kustomize /tmp/kustomize /usr/local/bin/ +COPY src/main/docker/entrypoint.sh /bin/ + +# nonroot user as defined in https://github.com/GoogleContainerTools/distroless +ENV NON_ROOT_USER="${NON_ROOT_USER}" +ENV NON_ROOT_USER_ID="${NON_ROOT_USER_ID}" +ENV NON_ROOT_USER_HOME="${NON_ROOT_USER_HOME}" + +# hadolint ignore=DL3018 +RUN apk add --update --no-cache \ + bash~=5 \ + docker-cli \ + && adduser -D -u "${NON_ROOT_USER_ID}" -h "${NON_ROOT_USER_HOME}" "${NON_ROOT_USER}" + +# replicate the test-related bits generated by `kubebuilder` into its Makefile. +ENV GOBIN="${GOPATH}/bin" +ENV ENVTEST_K8S_VERSION='1.23.x' +RUN chmod a+x /bin/entrypoint.sh \ + && go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest +WORKDIR "${NON_ROOT_USER_HOME}" +USER "${NON_ROOT_USER}" +RUN "${GOBIN}/setup-envtest" use "${ENVTEST_K8S_VERSION}" +# disable downloading remote content henceforth +ENV ENVTEST_INSTALLED_ONLY=true + +ENTRYPOINT ["/bin/entrypoint.sh"] diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/docker-bake.hcl b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/docker-bake.hcl new file mode 100644 index 0000000000..595a82f53c --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/docker-bake.hcl @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# A convenience script for build the kuttl image. +# See hbase-kubernetes-deployment/dockerfiles/kuttl/README.md +# + +# input variables +variable KUBECTL_SHA_AMD64_URL {} +variable KUBECTL_SHA_AMD64 {} +variable KUBECTL_BIN_AMD64_URL {} +variable KUBECTL_BIN_AMD64 {} +variable KUBECTL_SHA_ARM64_URL {} +variable KUBECTL_SHA_ARM64 {} +variable KUBECTL_BIN_ARM64_URL {} +variable KUBECTL_BIN_ARM64 {} +variable KUTTL_CHECKSUMS_URL {} +variable KUTTL_CHECKSUMS {} +variable KUTTL_BIN_AMD64_URL {} +variable KUTTL_BIN_AMD64 {} +variable KUTTL_BIN_ARM64_URL {} +variable KUTTL_BIN_ARM64 {} +variable KUSTOMIZE_CHECKSUMS_URL {} +variable KUSTOMIZE_CHECKSUMS {} +variable KUSTOMIZE_BIN_AMD64_TGZ_URL {} +variable KUSTOMIZE_BIN_AMD64_TGZ {} +variable KUSTOMIZE_BIN_ARM64_TGZ_URL {} +variable KUSTOMIZE_BIN_ARM64_TGZ {} + +# output variables +variable NON_ROOT_USER { + default = "nonroot" +} +variable NON_ROOT_USER_ID { + default = "65532" +} +variable NON_ROOT_USER_HOME { + default = "/home/${NON_ROOT_USER}" +} +variable USER { + default = "apache" +} +variable IMAGE_TAG { + default = "latest" +} +variable IMAGE_NAME { + default = "${USER}/hbase/operator-tools/kuttl" +} + +group default { + targets = [ "kuttl" ] +} + +target kuttl { + dockerfile = "src/main/docker/Dockerfile" + args = { + KUBECTL_SHA_AMD64_URL = KUBECTL_SHA_AMD64_URL + KUBECTL_SHA_AMD64 = KUBECTL_SHA_AMD64 + KUBECTL_BIN_AMD64_URL = KUBECTL_BIN_AMD64_URL + KUBECTL_BIN_AMD64 = KUBECTL_BIN_AMD64 + KUBECTL_SHA_ARM64_URL = KUBECTL_SHA_ARM64_URL + KUBECTL_SHA_ARM64 = KUBECTL_SHA_ARM64 + KUBECTL_BIN_ARM64_URL = KUBECTL_BIN_ARM64_URL + KUBECTL_BIN_ARM64 = KUBECTL_BIN_ARM64 + KUTTL_CHECKSUMS_URL = KUTTL_CHECKSUMS_URL + KUTTL_CHECKSUMS = KUTTL_CHECKSUMS + KUTTL_BIN_AMD64_URL = KUTTL_BIN_AMD64_URL + KUTTL_BIN_AMD64 = KUTTL_BIN_AMD64 + KUTTL_BIN_ARM64_URL = KUTTL_BIN_ARM64_URL + KUTTL_BIN_ARM64 = KUTTL_BIN_ARM64 + KUSTOMIZE_CHECKSUMS_URL = KUSTOMIZE_CHECKSUMS_URL + KUSTOMIZE_CHECKSUMS = KUSTOMIZE_CHECKSUMS + KUSTOMIZE_BIN_AMD64_TGZ_URL = KUSTOMIZE_BIN_AMD64_TGZ_URL + KUSTOMIZE_BIN_AMD64_TGZ = KUSTOMIZE_BIN_AMD64_TGZ + KUSTOMIZE_BIN_ARM64_TGZ_URL = KUSTOMIZE_BIN_ARM64_TGZ_URL + KUSTOMIZE_BIN_ARM64_TGZ = KUSTOMIZE_BIN_ARM64_TGZ + NON_ROOT_USER = NON_ROOT_USER + NON_ROOT_USER_ID = NON_ROOT_USER_ID + NON_ROOT_USER_HOME = NON_ROOT_USER_HOME + } + target = "final" + platforms = [ + "linux/amd64", + "linux/arm64" + ] + tags = [ "${IMAGE_NAME}:${IMAGE_TAG}" ] +} diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/docker-bake.override.hcl b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/docker-bake.override.hcl new file mode 100644 index 0000000000..fa33354830 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/docker-bake.override.hcl @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Externalize default values of build parameters and document how to retrieve them. +# + +function "basename" { + params = [a] + result = split("/", a)[length(split("/", a)) - 1] +} + +variable KUBECTL_VERSION { + default = "1.24.10" +} + +variable KUBECTL_SHA_AMD64_URL { + default = "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha512" +} + +variable KUBECTL_SHA_AMD64 { + default = "${basename(KUBECTL_SHA_AMD64_URL)}" +} + +variable KUBECTL_BIN_AMD64_URL { + default = "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl" +} + +variable KUBECTL_BIN_AMD64 { + default = "${basename(KUBECTL_BIN_AMD64_URL)}" +} + +variable KUBECTL_SHA_ARM64_URL { + default = "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/arm64/kubectl.sha512" +} + +variable KUBECTL_SHA_ARM64 { + default = "${basename(KUBECTL_SHA_ARM64_URL)}" +} + +variable KUBECTL_BIN_ARM64_URL { + default = "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/arm64/kubectl" +} + +variable KUBECTL_BIN_ARM64 { + default = "${basename(KUBECTL_BIN_ARM64_URL)}" +} + +variable KUTTL_VERSION { + default = "0.15.0" +} + +variable KUTTL_CHECKSUMS_URL { + default = "https://github.com/kudobuilder/kuttl/releases/download/v${KUTTL_VERSION}/checksums.txt" +} + +variable KUTTL_CHECKSUMS { + default = "${basename(KUTTL_CHECKSUMS_URL)}" +} + +variable KUTTL_BIN_AMD64_URL { + default = "https://github.com/kudobuilder/kuttl/releases/download/v${KUTTL_VERSION}/kubectl-kuttl_${KUTTL_VERSION}_linux_x86_64" +} + +variable KUTTL_BIN_AMD64 { + default = "${basename(KUTTL_BIN_AMD64_URL)}" +} + +variable KUTTL_BIN_ARM64_URL { + default = "https://github.com/kudobuilder/kuttl/releases/download/v${KUTTL_VERSION}/kubectl-kuttl_${KUTTL_VERSION}_linux_arm64" +} + +variable KUTTL_BIN_ARM64 { + default = "${basename(KUTTL_BIN_ARM64_URL)}" +} + +variable KUSTOMIZE_VERSION { + default = "4.5.4" +} + +variable KUSTOMIZE_CHECKSUMS_URL { + default = "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/checksums.txt" +} + +variable KUSTOMIZE_CHECKSUMS { + default = "${basename(KUSTOMIZE_CHECKSUMS_URL)}" +} + +variable KUSTOMIZE_BIN_AMD64_TGZ_URL { + default = "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" +} + +variable KUSTOMIZE_BIN_AMD64_TGZ { + default = "${basename(KUSTOMIZE_BIN_AMD64_TGZ_URL)}" +} + +variable KUSTOMIZE_BIN_ARM64_TGZ_URL { + default = "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_arm64.tar.gz" +} + +variable KUSTOMIZE_BIN_ARM64_TGZ { + default = "${basename(KUSTOMIZE_BIN_ARM64_TGZ_URL)}" +} diff --git a/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/entrypoint.sh b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/entrypoint.sh new file mode 100644 index 0000000000..8436e82600 --- /dev/null +++ b/hbase-kubernetes-deployment/hbase-kubernetes-testing-image/src/main/docker/entrypoint.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -o pipefail +set -x + +declare ENVTEST_K8S_VERSION + +declare KUBEBUILDER_ASSETS +KUBEBUILDER_ASSETS="$(setup-envtest use -i "${ENVTEST_K8S_VERSION}" -p path)" +export KUBEBUILDER_ASSETS + +/usr/local/bin/kubectl kuttl test "$@" diff --git a/hbase-kubernetes-deployment/pom.xml b/hbase-kubernetes-deployment/pom.xml index f0f8053dd8..fa82ecfcda 100644 --- a/hbase-kubernetes-deployment/pom.xml +++ b/hbase-kubernetes-deployment/pom.xml @@ -2,40 +2,45 @@ - - 4.0.0 - - hbase-operator-tools - org.apache.hbase.operator.tools - ${revision} - .. - + + 4.0.0 + + hbase-operator-tools + org.apache.hbase.operator.tools + ${revision} + .. + - hbase-kubernetes-deployment - Apache HBase - Kubernetes Deployment - Resource definitions for deploying HBase on Kubernetes. - pom + hbase-kubernetes-deployment + Apache HBase - Kubernetes Deployment + Resource definitions for deploying HBase on Kubernetes. + pom - + + hbase-kubernetes-testing-image + hbase-kubernetes-kustomize + - - + + + + diff --git a/pom.xml b/pom.xml index 3bfeff5b07..392d4e242a 100644 --- a/pom.xml +++ b/pom.xml @@ -163,6 +163,12 @@ hbase-tools ${project.version} + + org.apache.hbase.operator.tools + hbase-kubernetes-testing-image + pom + ${project.version} + @@ -322,6 +328,11 @@ true + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + @@ -429,6 +440,13 @@ + + + kr.motd.maven + os-maven-plugin + 1.7.1 + +