HBASE-27945 Introduce hdfs+hbase colocated pod definition

.gitignore

@@ -21,3 +21,6 @@ linklint-*.zip
 linklint/
 .checkstyle
 **/.checkstyle
+
+# detritus produced by kuttl
+kubeconfig*

dev-support/jenkins/Dockerfile

@@ -20,16 +20,29 @@
 FROM hadolint/hadolint:latest-debian as hadolint
 
 FROM maven:3.8-jdk-8
-
-# hadolint ignore=DL3008
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
-       binutils \
-       git \
-       rsync \
-       shellcheck \
-       wget && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+# hadolint ignore=SC1091
+RUN install -m 0755 -d /etc/apt/keyrings \
+    && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
+    && chmod a+r /etc/apt/keyrings/docker.gpg \
+    && echo \
+        "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+        "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
+        tee /etc/apt/sources.list.d/docker.list > /dev/null \
+    && apt-get -q update \
+    && apt-get -q install --no-install-recommends -y \
+        binutils=* \
+        ca-certificates=* \
+        curl=* \
+        docker-buildx-plugin=* \
+        docker-ce-cli=* \
+        git=* \
+        gnupg=* \
+        rsync=* \
+        shellcheck=* \
+        wget=* \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY --from=hadolint /bin/hadolint /bin/hadolint
 

dev-support/jenkins/jenkins_precommit_github_yetus.sh

@@ -111,6 +111,8 @@ YETUS_ARGS+=("--tests-filter=test4tests")
 # Dockerfile since we don't want to use the auto-pulled version.
 YETUS_ARGS+=("--docker")
 YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
+# enabled docker-in-docker so that we can build container images
+YETUS_ARGS+=("--dockerind=true")
 YETUS_ARGS+=("--mvn-custom-repos")
 YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}")
 # effectively treat dev-support as a custom maven module
@@ -127,4 +129,4 @@ YETUS_ARGS+=("--proclimit=5000")
 echo "Launching yetus with command line:"
 echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"
 
-/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
+/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"

hbase-kubernetes-deployment/README.md

@@ -0,0 +1,27 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# HBase Kubernetes Deployment
+
+This module contains Kubernetes configurations suitable for deploying ZooKeeper, HDFS, and HBase.
+It supports several physical topologies, including:
+ * a minimum deployment footprint consisting of a single instance of each architectural component.
+ * a high-avaiability deployment footprint consisting of redundancies for each architectural
+   component.
+ * a deployment where HBase region servers and HDFS data nodes share a pod, enabling short-circuit
+   read between them.

hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/README.md

@@ -0,0 +1,56 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# hbase-kubernetes-hadoop-image
+
+There is a contract that must be maintained between the container image run in the pod and the
+infrastructure that launched the pod. Details like paths, users/groups, permissions, and
+environment variables must align so that the deployment layer can pass runtime concerns down to
+the container.
+
+Start with the official hadoop image and extend it from there. Note that `apache/hadoop:3` is only
+published for `linux/amd64` at this time.
+
+## Build
+
+Input arguments are managed by [docker-bake.override.hcl](./docker-bake.override.hcl).
+
+Start by creating a buildx context that supports at least `linux/amd64` images. If you've created
+this context previously, it's enough to ensure that it's active via `docker buildx ls`.
+
+```shell
+$ docker buildx create \
+  --driver docker-container \
+  --platform linux/amd64,linux/arm64 \
+  --use \
+  --bootstrap
+```
+
+Finally, build the image using `mvn package`, or manually, using,
+
+```shell
+$ docker buildx bake \
+  --file src/main/docker/docker-bake.hcl \
+  --file src/main/docker/docker-bake.override.hcl \
+  --set '*.platform=linux/amd64' \
+  --pull \
+  --load
+```
+
+This exports an image to your local repository that is tagged as
+`${USER}/hbase/operator-tools/hadoop:latest`.

hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/pom.xml

@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <!--
+ /**
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+  -->
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <artifactId>hbase-kubernetes-deployment</artifactId>
+    <groupId>org.apache.hbase.operator.tools</groupId>
+    <version>${revision}</version>
+    <relativePath>..</relativePath>
+  </parent>
+
+  <artifactId>hbase-kubernetes-hadoop-image</artifactId>
+  <name>Apache HBase - Kubernetes Hadoop Image</name>
+  <description>A container image to for running Hadoop in Kubernetes.</description>
+  <packaging>pom</packaging>
+
+  <properties>
+    <!--
+        the upstream hadoop image is only published for linux/amd64, so only support that platform
+    -->
+    <container_image.platforms>linux/amd64</container_image.platforms>
+  </properties>
+
+  <dependencies />
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <configuration>
+          <useMavenLogger>true</useMavenLogger>
+        </configuration>
+        <executions>
+          <execution>
+            <id>docker-buildx-bake-print</id>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <phase>package</phase>
+            <configuration>
+              <executable>/usr/bin/env</executable>
+              <arguments>
+                <argument>sh</argument>
+                <argument>-c</argument>
+                <argument>
+                  2>&amp;1 \
+                  docker buildx bake \
+                    --print \
+                    --file src/main/docker/docker-bake.hcl \
+                    --file src/main/docker/docker-bake.override.hcl
+                </argument>
+              </arguments>
+            </configuration>
+          </execution>
+          <execution>
+            <id>docker-buildx-bake</id>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <phase>package</phase>
+            <configuration>
+              <executable>/usr/bin/env</executable>
+              <arguments>
+                <argument>sh</argument>
+                <argument>-c</argument>
+                <argument>
+                  2>&amp;1 \
+                  docker buildx bake \
+                    --progress plain \
+                    --pull \
+                    --load \
+                    --set *.platform=${container_image.platforms} \
+                    --file src/main/docker/docker-bake.hcl \
+                    --file src/main/docker/docker-bake.override.hcl
+                </argument>
+              </arguments>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+   </plugins>
+  </build>
+</project>

hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/src/main/docker/Dockerfile

@@ -0,0 +1,57 @@
+# syntax=docker/dockerfile:1.4
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# hadolint global ignore=DL3020
+
+ARG BASE_IMG='apache/hadoop'
+ARG BASE_TAG='3'
+ARG BASE_IMG_ALPINE='alpine'
+ARG BASE_IMG_TAG_ALPINE='latest'
+
+FROM ${BASE_IMG_ALPINE}:${BASE_IMG_TAG_ALPINE} as alpine
+
+## -- Stage installed_jmx_exporter --
+# A stage that prepares the JMX Exporter installation directory. The jar must be in well-known
+# location so can refer to it in command yaml value.
+FROM alpine AS installed_jmx_exporter
+ARG JMX_PROMETHEUS_JAR_URL
+ADD --link ${JMX_PROMETHEUS_JAR_URL} /tmp/jmx_prometheus_javaagent.jar
+
+FROM ${BASE_IMG}:${BASE_TAG} as final
+ARG CORRETTO_KEY_URL
+ARG CORRETTO_KEY
+ARG CORRETTO_REPO_URL
+ARG CORRETTO_REPO
+USER root
+ADD --link ${CORRETTO_KEY_URL} /tmp/
+ADD --link ${CORRETTO_REPO_URL} /etc/yum.repos.d/
+
+RUN rpm --import "/tmp/${CORRETTO_KEY}" \
+    && yum -y remove java-1.8.0-* \
+    && yum -y update \
+    && yum -y install java-17-amazon-corretto-devel \
+    && yum -y clean all \
+    && rm -rf /var/cache
+
+COPY --from=installed_jmx_exporter \
+  --chown=hadoop:users \
+  /tmp/jmx_prometheus_javaagent.jar /opt/hadoop/jmx_prometheus_javaagent.jar
+
+ENV JMX_PROMETHEUS_JAR /opt/hadoop/jmx_prometheus_javaagent.jar
+ENV USER='hadoop'
+USER ${USER}

hbase-kubernetes-deployment/hbase-kubernetes-hadoop-image/src/main/docker/docker-bake.hcl

@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# A convenience script for build the kuttl image.
+# See hbase-kubernetes-deployment/dockerfiles/kuttl/README.md
+#
+
+variable BASE_IMG {
+  default = "apache/hadoop"
+}
+variable BASE_TAG {
+  default = "3"
+}
+variable USER {
+  default = "apache"
+}
+variable IMAGE_TAG {
+  default = "latest"
+}
+variable IMAGE_NAME {
+  default = "${USER}/hbase/operator-tools/hadoop"
+}
+variable CORRETTO_KEY_URL {}
+variable CORRETTO_KEY {}
+variable CORRETTO_REPO_URL {}
+variable CORRETTO_REPO {}
+variable JMX_PROMETHEUS_JAR_URL {}
+variable JMX_PROMETHEUS_JAR {}
+
+group default {
+  targets = [ "hadoop" ]
+}
+
+target hadoop {
+  dockerfile = "src/main/docker/Dockerfile"
+  args = {
+    BASE_IMG = BASE_IMG
+    BASE_TAG = BASE_TAG
+    CORRETTO_KEY_URL = CORRETTO_KEY_URL
+    CORRETTO_KEY = CORRETTO_KEY
+    CORRETTO_REPO_URL = CORRETTO_REPO_URL
+    CORRETTO_REPO = CORRETTO_REPO
+    JMX_PROMETHEUS_JAR_URL = JMX_PROMETHEUS_JAR_URL
+    JMX_PROMETHEUS_JAR = JMX_PROMETHEUS_JAR
+  }
+  target = "final"
+  platforms = [
+    # upstream image only provides linux/amd64
+    "linux/amd64"
+  ]
+  tags = [ "${IMAGE_NAME}:${IMAGE_TAG}" ]
+}