diff --git a/6-base/platform/centos/Dockerfile b/6-base/platform/centos/Dockerfile deleted file mode 100644 index b340426..0000000 --- a/6-base/platform/centos/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM centos:7 - -RUN yum update -y && yum install -y \ - epel-release \ - wget \ - gcc-c++ \ - boost-regex \ - openldap \ - libicu \ - m4 \ - libtool \ - libxslt \ - libxml \ - openssh-server \ - openssh-clients \ - expect \ - libarchive \ - rsync \ - apr \ - apr-util \ - zip \ - tbb \ - net-tools \ - sudo \ - psmisc \ - blas.x86_64 \ - atlas.x86_64 \ - which \ - numa-libs \ - vim-minimal \ - python34 \ - python34-libs - -RUN /usr/bin/ssh-keygen -A -RUN mkdir -p /var/run/sshd - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/6-rc/Dockerfile b/6-rc/Dockerfile deleted file mode 100644 index 8236b69..0000000 --- a/6-rc/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM hpccsystems/hpcc-base:6-xenial - -RUN apt-get update && apt-get install -y iputils-ping - -RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc - -ENV VERSION 6.4.28 -ENV FULL_VERSION 6.4.28-rc1 -#ENV PLATFORM_DOWNLOAD_MD5 a62b43a1ef1ecf7d8d702d9033e9cf39 -ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}xenial_amd64.deb - -RUN set -x \ - && wget "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \ - #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \ - && dpkg -i "${PLATFORM_PACKAGE}" \ - && rm -rf "${PLATFORM_PACKAGE}" - -RUN mkdir -p /var/run/dbus && mkdir -p /var/run/sshd -EXPOSE 8010 8002 8015 9876 - -COPY docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/6-rc/docker-entrypoint.sh b/6-rc/docker-entrypoint.sh deleted file mode 100755 index 4343cf9..0000000 --- a/6-rc/docker-entrypoint.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -set -e - -/usr/bin/ssh-keygen -A -#/etc/init.d/hpcc-init start - -if [ "${EXEC_IN_LOOP}" = "true" ] -then - interval=5 - [ -n EXEC_INTERVAL ] && interval=${EXEC_INTERVAL} - while [ 1 ] - do - [ -e "$1" ] && exec "$@" - sleep $interval - done -else - exec "$@" -fi diff --git a/6-rc/platform/centos/Dockerfile b/6-rc/platform/centos/Dockerfile deleted file mode 100644 index 276c1a1..0000000 --- a/6-rc/platform/centos/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM hpccsystems/hpcc-base:7-el7 - -RUN chmod u+s $(which ping) -#RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc - -RUN /usr/bin/ssh-keygen -A - -ENV VERSION 6.4.22 -ENV FULL_VERSION 6.4.28-rc1 -#ENV PLATFORM_DOWNLOAD_MD5 a62b43a1ef1ecf7d8d702d9033e9cf39 -ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}.el7.x86_64.rpm - -RUN set -x \ - && wget "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \ - #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \ - && yum install --nogpgcheck -y "${PLATFORM_PACKAGE}" \ - && rm -rf "${PLATFORM_PACKAGE}" - -#RUN mkdir -p /var/run/dbus -RUN mkdir -p /var/run/sshd -EXPOSE 8010 8002 8015 9876 - -COPY docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/6-rc/platform/centos/docker-entrypoint.sh b/6-rc/platform/centos/docker-entrypoint.sh deleted file mode 100755 index 4343cf9..0000000 --- a/6-rc/platform/centos/docker-entrypoint.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -set -e - -/usr/bin/ssh-keygen -A -#/etc/init.d/hpcc-init start - -if [ "${EXEC_IN_LOOP}" = "true" ] -then - interval=5 - [ -n EXEC_INTERVAL ] && interval=${EXEC_INTERVAL} - while [ 1 ] - do - [ -e "$1" ] && exec "$@" - sleep $interval - done -else - exec "$@" -fi diff --git a/6/Dockerfile b/6/Dockerfile deleted file mode 100644 index d35e81e..0000000 --- a/6/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM hpccsystems/hpcc-base:6-xenial - -RUN apt-get update && apt-get install -y iputils-ping -RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc - -ENV VERSION 6.4.40 -ENV FULL_VERSION 6.4.40-1 -#ENV PLATFORM_DOWNLOAD_MD5 a62b43a1ef1ecf7d8d702d9033e9cf39 -ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}xenial_amd64.deb - -RUN set -x \ - && wget "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \ - #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \ - && dpkg -i "${PLATFORM_PACKAGE}" \ - && rm -rf "${PLATFORM_PACKAGE}" - -EXPOSE 8010 8002 8015 9876 - -COPY docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/6/docker-entrypoint.sh b/6/docker-entrypoint.sh deleted file mode 100755 index 4343cf9..0000000 --- a/6/docker-entrypoint.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -set -e - -/usr/bin/ssh-keygen -A -#/etc/init.d/hpcc-init start - -if [ "${EXEC_IN_LOOP}" = "true" ] -then - interval=5 - [ -n EXEC_INTERVAL ] && interval=${EXEC_INTERVAL} - while [ 1 ] - do - [ -e "$1" ] && exec "$@" - sleep $interval - done -else - exec "$@" -fi diff --git a/6/platform/centos/Dockerfile b/6/platform/centos/Dockerfile deleted file mode 100644 index 3063b57..0000000 --- a/6/platform/centos/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM hpccsystems/hpcc-base:6-el7 - -RUN chmod u+s $(which ping) -#RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc - -ENV VERSION 6.4.28 -ENV FULL_VERSION 6.4.28-1 -#ENV PLATFORM_DOWNLOAD_MD5 a62b43a1ef1ecf7d8d702d9033e9cf39 -ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}.el7.x86_64.rpm - -RUN set -x \ - && wget "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \ - #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \ - && yum install --nogpgcheck -y "${PLATFORM_PACKAGE}" \ - && rm -rf "${PLATFORM_PACKAGE}" - -EXPOSE 8010 8002 8015 9876 - -COPY docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/6/platform/centos/docker-entrypoint.sh b/6/platform/centos/docker-entrypoint.sh deleted file mode 100755 index 4343cf9..0000000 --- a/6/platform/centos/docker-entrypoint.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -set -e - -/usr/bin/ssh-keygen -A -#/etc/init.d/hpcc-init start - -if [ "${EXEC_IN_LOOP}" = "true" ] -then - interval=5 - [ -n EXEC_INTERVAL ] && interval=${EXEC_INTERVAL} - while [ 1 ] - do - [ -e "$1" ] && exec "$@" - sleep $interval - done -else - exec "$@" -fi diff --git a/7-rc/Dockerfile b/7-rc/Dockerfile deleted file mode 100644 index e914529..0000000 --- a/7-rc/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM hpccsystems/hpcc-base:7-xenial - -RUN apt-get update && apt-get install -y iputils-ping curl xterm -RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc - -ENV VERSION 7.0.0 -ENV FULL_VERSION 7.0.0-rc4 -#ENV PLATFORM_DOWNLOAD_MD5 ef58e94dcc66a45cb0fc8202e68b50d9 -ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}xenial_amd64.deb - -RUN set -x \ - && wget "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \ - #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \ - && dpkg -i "${PLATFORM_PACKAGE}" \ - && rm -rf "${PLATFORM_PACKAGE}" - -EXPOSE 8010 8002 8015 9876 - -RUN cp -r /home/hpcc/.ssh ~/ - -# If this image will be in Spark cluster add following files -# It requires to build from parent directory -#COPY stack/admin/spark/* /opt/HPCCSystems/componentfiles/configxml/ - -COPY docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/7-rc/Dockerfile_FB b/7-rc/Dockerfile_FB deleted file mode 100644 index 1a3343b..0000000 --- a/7-rc/Dockerfile_FB +++ /dev/null @@ -1,32 +0,0 @@ -FROM hpccsystems/hpcc-base:7-xenial - -RUN apt-get update && apt-get install -y iputils-ping curl xterm apt-transport-https -RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc -RUN echo "deb https://packages.elastic.co/beats/apt stable main" | tee -a /etc/apt/sources.list.d/beats.list -RUN wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | apt-key add - -RUN apt-get update -RUN apt-get install -y filebeat - -ENV VERSION 7.0.0 -ENV FULL_VERSION 7.0.0-rc4 -#ENV PLATFORM_DOWNLOAD_MD5 ef58e94dcc66a45cb0fc8202e68b50d9 -ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}xenial_amd64.deb - -RUN set -x \ - && wget "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \ - #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \ - && dpkg -i "${PLATFORM_PACKAGE}" \ - && rm -rf "${PLATFORM_PACKAGE}" - -EXPOSE 8010 8002 8015 9876 - -RUN cp -r /home/hpcc/.ssh ~/ - -# If this image will be in Spark cluster add following files -# It requires to build from parent directory -#COPY stack/admin/spark/* /opt/HPCCSystems/componentfiles/configxml/ - -COPY docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/7-rc/Dockerfile_Spark b/7-rc/Dockerfile_Spark deleted file mode 100644 index 0a90dd6..0000000 --- a/7-rc/Dockerfile_Spark +++ /dev/null @@ -1,35 +0,0 @@ -FROM hpccsystems/hpcc-base:7-xenial - -RUN rm -rf /var/lib/apt/lists/* -RUN apt-get update -RUN apt-get install -y iputils-ping curl xterm -RUN apt-get install -y openjdk-8-jdk iproute2 - -RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc - -ENV VERSION 7.0.0 -ENV FULL_VERSION 7.0.0-rc4 -#ENV PLATFORM_DOWNLOAD_MD5 a62b43a1ef1ecf7d8d702d9033e9cf39 -ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}xenial_amd64.deb - -RUN set -x \ - && wget "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \ - #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \ - && dpkg -i "${PLATFORM_PACKAGE}" \ - && rm -rf "${PLATFORM_PACKAGE}" - -ENV PLUGIN_PACKAGE hpccsystems-plugin-spark_${FULL_VERSION}xenial_amd64.deb -RUN set -x \ - && wget "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/plugins/${PLUGIN_PACKAGE}" \ - #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLUGIN_PACKAGE}" | md5sum -c - \ - && dpkg -i "${PLUGIN_PACKAGE}" \ - && rm -rf "${PLUGIN_PACKAGE}" - -EXPOSE 8010 8002 8015 9876 - -RUN cp -r /home/hpcc/.ssh ~/ - -COPY docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/7-rc/docker-entrypoint.sh b/7-rc/docker-entrypoint.sh deleted file mode 100755 index 4343cf9..0000000 --- a/7-rc/docker-entrypoint.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -set -e - -/usr/bin/ssh-keygen -A -#/etc/init.d/hpcc-init start - -if [ "${EXEC_IN_LOOP}" = "true" ] -then - interval=5 - [ -n EXEC_INTERVAL ] && interval=${EXEC_INTERVAL} - while [ 1 ] - do - [ -e "$1" ] && exec "$@" - sleep $interval - done -else - exec "$@" -fi diff --git a/7-rc/platform/centos/Dockerfile b/7-rc/platform/centos/Dockerfile deleted file mode 100644 index 74306f1..0000000 --- a/7-rc/platform/centos/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM hpccsystems/hpcc-base:7-el7 - -RUN yum install -y curl -#RUN yum install -y java-1.8.0-openjdk -RUN chmod u+s $(which ping) -#RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc - -ENV VERSION 7.0.0 -ENV FULL_VERSION 7.0.0-rc4 -#ENV PLATFORM_DOWNLOAD_MD5 c435045d4319880f141e8faf23555b83 -ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}.el7.x86_64.rpm - -RUN set -x \ - && wget "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \ - #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \ - && yum install --nogpgcheck -y "${PLATFORM_PACKAGE}" \ - && rm -rf "${PLATFORM_PACKAGE}" - -EXPOSE 8010 8002 8015 9876 - -RUN cp -r /home/hpcc/.ssh ~/ -COPY docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/7-rc/platform/centos/docker-entrypoint.sh b/7-rc/platform/centos/docker-entrypoint.sh deleted file mode 100755 index 4343cf9..0000000 --- a/7-rc/platform/centos/docker-entrypoint.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -set -e - -/usr/bin/ssh-keygen -A -#/etc/init.d/hpcc-init start - -if [ "${EXEC_IN_LOOP}" = "true" ] -then - interval=5 - [ -n EXEC_INTERVAL ] && interval=${EXEC_INTERVAL} - while [ 1 ] - do - [ -e "$1" ] && exec "$@" - sleep $interval - done -else - exec "$@" -fi diff --git a/7/Dockerfile b/7/Dockerfile deleted file mode 100644 index e40fb06..0000000 --- a/7/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM hpccsystems/hpcc-base:7-xenial - -RUN apt-get update && apt-get install -y iputils-ping curl xterm -RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc - -ENV VERSION 7.0.18 -ENV FULL_VERSION 7.0.18-1 -#ENV PLATFORM_DOWNLOAD_MD5 ef58e94dcc66a45cb0fc8202e68b50d9 -ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}xenial_amd64.deb - -RUN set -x \ - && wget "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \ - #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \ - && dpkg -i "${PLATFORM_PACKAGE}" \ - && rm -rf "${PLATFORM_PACKAGE}" - -EXPOSE 8010 8002 8015 9876 - -RUN cp -r /home/hpcc/.ssh ~/ - -# If this image will be in Spark cluster add following files -# It requires to build from parent directory -#COPY stack/admin/spark/* /opt/HPCCSystems/componentfiles/configxml/ - -COPY docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/7/Dockerfile_FB b/7/Dockerfile_FB deleted file mode 100644 index 9904da6..0000000 --- a/7/Dockerfile_FB +++ /dev/null @@ -1,32 +0,0 @@ -FROM hpccsystems/hpcc-base:7-xenial - -RUN apt-get update && apt-get install -y iputils-ping curl xterm apt-transport-https -RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc -RUN echo "deb https://packages.elastic.co/beats/apt stable main" | tee -a /etc/apt/sources.list.d/beats.list -RUN wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | apt-key add - -RUN apt-get update -RUN apt-get install -y filebeat - -ENV VERSION 7.0.18 -ENV FULL_VERSION 7.0.18-1 -#ENV PLATFORM_DOWNLOAD_MD5 ef58e94dcc66a45cb0fc8202e68b50d9 -ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}xenial_amd64.deb - -RUN set -x \ - && wget "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \ - #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \ - && dpkg -i "${PLATFORM_PACKAGE}" \ - && rm -rf "${PLATFORM_PACKAGE}" - -EXPOSE 8010 8002 8015 9876 - -RUN cp -r /home/hpcc/.ssh ~/ - -# If this image will be in Spark cluster add following files -# It requires to build from parent directory -#COPY stack/admin/spark/* /opt/HPCCSystems/componentfiles/configxml/ - -COPY docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/7/Dockerfile_Spark b/7/Dockerfile_Spark deleted file mode 100644 index 6e70b27..0000000 --- a/7/Dockerfile_Spark +++ /dev/null @@ -1,35 +0,0 @@ -FROM hpccsystems/hpcc-base:7-xenial - -RUN rm -rf /var/lib/apt/lists/* -RUN apt-get update -RUN apt-get install -y iputils-ping curl xterm -RUN apt-get install -y openjdk-8-jdk iproute2 - -RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc - -ENV VERSION 7.0.18 -ENV FULL_VERSION 7.0.18-1 -#ENV PLATFORM_DOWNLOAD_MD5 a62b43a1ef1ecf7d8d702d9033e9cf39 -ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}xenial_amd64.deb - -RUN set -x \ - && wget "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \ - #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \ - && dpkg -i "${PLATFORM_PACKAGE}" \ - && rm -rf "${PLATFORM_PACKAGE}" - -ENV PLUGIN_PACKAGE hpccsystems-plugin-spark_${FULL_VERSION}xenial_amd64.deb -RUN set -x \ - && wget "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/plugins/${PLUGIN_PACKAGE}" \ - #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLUGIN_PACKAGE}" | md5sum -c - \ - && dpkg -i "${PLUGIN_PACKAGE}" \ - && rm -rf "${PLUGIN_PACKAGE}" - -EXPOSE 8010 8002 8015 9876 - -RUN cp -r /home/hpcc/.ssh ~/ - -COPY docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/7/docker-entrypoint.sh b/7/docker-entrypoint.sh deleted file mode 100755 index 4343cf9..0000000 --- a/7/docker-entrypoint.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -set -e - -/usr/bin/ssh-keygen -A -#/etc/init.d/hpcc-init start - -if [ "${EXEC_IN_LOOP}" = "true" ] -then - interval=5 - [ -n EXEC_INTERVAL ] && interval=${EXEC_INTERVAL} - while [ 1 ] - do - [ -e "$1" ] && exec "$@" - sleep $interval - done -else - exec "$@" -fi diff --git a/7/platform/centos/Dockerfile b/7/platform/centos/Dockerfile deleted file mode 100644 index c019e4b..0000000 --- a/7/platform/centos/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM hpccsystems/hpcc-base:7-el7 - -RUN yum install -y curl -#RUN yum install -y java-1.8.0-openjdk -RUN chmod u+s $(which ping) -#RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc - -ENV VERSION 7.0.0 -ENV FULL_VERSION 7.0.0-1 -#ENV PLATFORM_DOWNLOAD_MD5 c435045d4319880f141e8faf23555b83 -ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}.el7.x86_64.rpm - -RUN set -x \ - && wget "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \ - #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \ - && yum install --nogpgcheck -y "${PLATFORM_PACKAGE}" \ - && rm -rf "${PLATFORM_PACKAGE}" - -EXPOSE 8010 8002 8015 9876 - -RUN cp -r /home/hpcc/.ssh ~/ -COPY docker-entrypoint.sh / -ENTRYPOINT ["/docker-entrypoint.sh"] - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/7/platform/centos/docker-entrypoint.sh b/7/platform/centos/docker-entrypoint.sh deleted file mode 100755 index 4343cf9..0000000 --- a/7/platform/centos/docker-entrypoint.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh -set -e - -/usr/bin/ssh-keygen -A -#/etc/init.d/hpcc-init start - -if [ "${EXEC_IN_LOOP}" = "true" ] -then - interval=5 - [ -n EXEC_INTERVAL ] && interval=${EXEC_INTERVAL} - while [ 1 ] - do - [ -e "$1" ] && exec "$@" - sleep $interval - done -else - exec "$@" -fi diff --git a/PUSH.md b/PUSH.md new file mode 100644 index 0000000..07ed6b2 --- /dev/null +++ b/PUSH.md @@ -0,0 +1,66 @@ +# Push Docker Image to Repositories + +We currently push HPCC Systems Docker Images to three Docker Image Registries +- Docker Hub +- Gitlab (gitlab.ins.risk.regn.net) +- AWS Elatic Container Registry (ECR) + + +## Build/Tag/Push + +### Build +```console +sudo docker build -t : --build-arg version= . +``` + +```console +#sudo docker tag : : +#for example, +sudo docker tag hpccsystems/platform:7.4.24-1 446598291512.dkr.ecr.us-east-2.amazonaws.com/hpccsystems/platform:7.4.24-1 +``` + +### Push +```console +sudo docker push : +``` +## Docker Hub +### Login +```console +sudo docker login -u -p +``` + +### Push +```console +sudo docker push hpccsystems/: +``` + + +## Gitlab + +### Login +```console +sudo docker login -u -p gitlab.ins.risk.regn.net:4567 +``` + +### Push +```console +sudo docker push gitlab.ins.risk.regn.net:4567/docker-images/hpccsystems/ln-platform-wp|ln-clienttools>: +``` + + +## Elastic Container Registry (ECR) + +ECR is regional service not global service. Pulling images cross regions is possible but will have additional latency. + +### Login +```console +sudo $(aws ecr get-login --no-include-email --region us-east-2) + +``` + +### Push +```console +#sudo docker push .dkr.ecr.us-east-2.amazonaws.com/hpcc-systems-platform: +#for example, +sudo docker push 446598291512.dkr.ecr.us-east-2.amazonaws.com/hpccsystems/platform:7.4.24-1 +``` diff --git a/README.md b/README.md index aa22e99..b2c4bdf 100644 --- a/README.md +++ b/README.md @@ -1,87 +1,55 @@ +# HPCC Systems Platform Community Version + platform/ + - ce/base/<version>/Dockerfile: HPCC Systems Platform Community version prequisites (Ubuntu 18.04) + Docker Hub: hpccsystems/hpcc-base + - ce/Dockerfile HPCC Systems Platform Community version Docker build file with version as input argument. + Docker Hub: hpccsystems/platform + - ln/base/<version>/Dockerfile: HPCC Systems Platform Internal version prequisites (CentOS 7) + - ln/Dockerfile HPCC Systems Platform Internal with Plugins Docker build file with version as input argument -# Docker HPCC -### Supported tags and respective Dockerfile links -* 7.0.18-1 latest [(7/Dockerfile)](https://github.com/hpcc-systems/docker-hpcc/tree/master/7/Dockerfile) -* 7.0.16-1 -* 7.0.14-1 -* 7.0.8-1 -* 7.0.6-1 -* 7.0.4-1 -* 7.0.2-1 -* 7.0.0-1 -* 6.4.40-1 [(6/Dockerfile)](https://github.com/hpcc-systems/docker-hpcc/tree/master/6/Dockerfile) -* 6.4.38-1 -* 6.4.36-1 -* 6.4.32-1 -* 6.4.30-1 -* 6.4.28-1 -* 6.4.26-1 -* 6.4.24-1 + clienttools/ + - ce/Dockerfile: HPCC Systems Clienttools Community Docker version build file with version as input argument + Docker Hub: hpccsystems/clienttools + - ln/Dockerfile: HPCC Systems Clienttools Internal version Docker build file with version as input argument + plugins/ + - ce/Dockerfile: HPCC Systems Plugins Community version Docker build file (based Platform Docker image) with version as input argument -## What is HPCCSystems (HPCC) -The HPCC Systems server platform is a free, open source, massively scalable platform for big data analytics. Download the HPCC Systems server platform now and take the reins of the same core technology that LexisNexis has used for over a decade to analyze massive data sets for its customers in industry, law enforcement, government, and science. + spark/ -For more information and related downloads for HPCC Systems products, please visit https://hpccsystems.com - - -## How to use Docker HPCC image -You can start the Docker HPCC image in interactive (-i -t) or daemon mode (-d). You must start the HPCC processes then go to ECLWatch to submit jobs, query, and explore your data with the HPCC Systems platform. - -To map a docker container port to a host port. - - "-p <host port>:<docker container port>" - -The default ECLWatch port is 8010. - -### Ubuntu -To start Docker in interactive mode and map ECLWatch port 8010 to host 8010: -```sh -sudo docker run -t -i -p 8010:8010 hpccsystems/hpcc /bin/bash +## How to buid +Create HPCC Systems Platform Docker image : +For example: +```console +sudo docker build -t hpccsystems/platform:7.4.8-1 --build-arg version=7.4.8-1 . ``` -To start Docker in daemon mode and map ECLWatch port 8010 to host 8020: -```sh -sudo docker run -d -p 8020:8010 hpccsystems/hpcc -``` +# Development and custom Build + dev/<version> + <bionic|disco|eoan|el7|gcc7|gcc8|gcc9>/ + - base/<verson>/Dockerfile: HPCC Systems Platform community version prequisites + - bldsvr/Dockerfile: Build Server Docker build file + - platform/<ce|ln>/Dockerfile: Docker build file for compiling and building HPCC Systems Platform image +## How to build + The compiling and building Docker build file has "ARG" for various input parameter. -### CentOS -To start Docker in interactive mode and map ECLWatch port 8020 to host 8010: -```sh -sudo docker run -t -i --privileged -e "container=docker" -p 8020:8010 hpccsystems/hpcc:el7 /bin/bash +Build master branch: +```console +sudo docker build -t hpccsystems/platform:master . ``` -To start Docker in daemon mode and map ECLWatch port 8010 to host 8010: -```sh -sudo docker run -d --privileged -e "container=docker"-p 8010:8010 hpccsystems/hpcc:el7 +Build branch or tag, for example community_7.4.10-rc1: +```console +sudo docker build -t hpccsystems/platform:7.4.10-rc1 --build-arg branch=community_7.4.10-rc1 . ``` -## How to build Docker HPCC image -You can always create your own Dockerfile to build a Docker HPCC image. We provide some pre-defined Dockerfiles and tools here. -These are for the HPCC Platform for Ubuntu 16.04 and CentOS 7. HPCC Plugins for Dockerfiles to be provided later. - -### Checkout this git reposiotry -```sh -git clone https://github.com/hpcc-systems/docker-hpcc.git -``` -Currently there are two release versions: Gold Release: 6/ and Release Candidate: 6-rc/. There are two product types: platform and plugins. -There are two Linux distributions: Ubuntu (Ubuntu 16.04, default) and CentOS (CentOS 7) - -### Update HPCC Platform Version -Make sure the HPCC Platform version is available in HPCCSystems.com -> "Download" -> "HPCC SYSTEM PLATFORM" -Update the version with update.sh. For example, if you want to build 6.4.14-1: -```sh -./update 6.4.18-1 -``` -### Build -You can go to the directory of interest which contains the Dockerfile to run the build, or you can run the test-build.sh script which includes test: -```sh -sudo ./test-build.sh -v 6 -l ubuntu -p platform +Build your own repo and branch: +```console +sudo docker build -t /platform: --build-arg owner= --build-arg branch= . ``` -This will build HPCC Platform 6.x for Ubuntu 16.04. -To upload a build image to Docker Hub: -```sh -docker push : For example: docker push hpccsystems/hpcc:latest +Build a private repo: +```console +sudo docker build -t /platform: --build-arg owner= --build-arg branch=--build-arg user= --build-arg password= . ``` diff --git a/clienttools/ce/Dockerfile b/clienttools/ce/Dockerfile new file mode 100644 index 0000000..51cb4db --- /dev/null +++ b/clienttools/ce/Dockerfile @@ -0,0 +1,20 @@ +FROM hpccsystems/hpcc-base:7-bionic + +ARG version= +RUN if [ -z "${version}" ] ; then echo Missing HPCC version; exit 1; else : ; fi + + +RUN set -ex; \ + apt-get update; \ + apt-get install -y curl libarchive13 openjdk-8-jdk; \ + export VERSION_MMP=${version%-*}; \ + export CLIENTTOOLS_PACKAGE=hpccsystems-clienttools-community_${version}bionic_amd64.deb; \ + wget "http://d2wulyp08c6njk.cloudfront.net/releases/CE-Candidate-${VERSION_MMP}/bin/clienttools/${CLIENTTOOLS_PACKAGE}"; \ + #echo "$CLIENTTOOLS_DOWNLOAD_MD5 ${CLIENTTOOLS_PACKAGE}" | md5sum -c -; \ + dpkg -i "${CLIENTTOOLS_PACKAGE}" ;\ + rm -rf "${CLIENTTOOLS_PACKAGE}" + + +ENTRYPOINT ["/docker-entrypoint.sh"] + +CMD ["sleep.sh"] diff --git a/clienttools/ce/README.md b/clienttools/ce/README.md new file mode 100644 index 0000000..16002aa --- /dev/null +++ b/clienttools/ce/README.md @@ -0,0 +1,42 @@ +# Supported tags and respective Dockerfile links +- [7.4.10-1, 7.4, 7](https://github.com/hpcc-systems/docker-hpcc/tree/clienttools/ce/Dockerfile) + +# Quick reference +- **Where to get help**: + [HPCCSystems Forum](https://hpccsystems.com/bb/) +- **Where to file issues**: + [JIRA HPCC](https://track.hpccsystems.com/projects/HPCC/summary) +- **Maintained by**: +builds@hpccsystems.com +- **Source of this project**: + [https://github.com/hpcc-systems/docker-hpcc](https://github.com/hpcc-systems/docker-hpcc) + + +# What is HPCC Systems + +The HPCC Systems server platform is a free, open source, massively scalable platform for big data analytics. Download the HPCC Systems server platform now and take the reins of the same core technology that LexisNexis has used for over a decade to analyze massive data sets for its customers in industry, law enforcement, government, and science. + +For more information and related downloads for HPCC Systems products, please visit +https://hpccsystems.com + +![alt text](https://hpccsystems.com/sites/default/files/hpcc-systems-horiz.png "HPCCSystems Logo") + +# How to use this image +You can start the Docker HPCC Systems image in interactive (-i -t) or daemon mode (-d). + +To start Docker in interactive mode: +``` +sudo docker run -t -i hpccsystems/clienttools /bin/bash +``` + +# How is this image build +```console +sudo docker build -t : --build-arg version= . +``` +For example, +```console +sudo docker build -t hpccsystems/clienttools:7.4 --build-arg version=7.4.22-1 . +``` + +# License +Licensed under the Apache License, Version 2.0 (the "License") diff --git a/clienttools/ln/Dockerfile b/clienttools/ln/Dockerfile new file mode 100644 index 0000000..e37856b --- /dev/null +++ b/clienttools/ln/Dockerfile @@ -0,0 +1,39 @@ +FROM hpccsystems/hpcc-base:7-el7 + +ARG version= +RUN if [ -z "${version}" ] ; then echo Missing HPCC version; exit 1; else : ; fi + +RUN yum update -y && yum install -y \ + curl \ + git \ + java-1.8.0-openjdk \ + R-core-devel + +RUN set -ex; \ + wget http://mysql.mirrors.hoobly.com/Downloads/MySQL-5.6/MySQL-devel-5.6.45-1.el7.x86_64.rpm; \ + rpm -i MySQL-devel-5.6.45-1.el7.x86_64.rpm; \ + rm -rf MySQL-devel-5.6.45-1.el7.x86_64.rpm; \ + \ + wget https://cran.r-project.org/src/contrib/Archive/Rcpp/Rcpp_0.12.19.tar.gz; \ + R CMD INSTALL Rcpp_0.12.19.tar.gz; \ + rm -rf Rcpp_0.12.19.tar.gz; \ + \ + wget https://cran.r-project.org/src/contrib/Archive/RInside/RInside_0.2.12.tar.gz; \ + R CMD INSTALL RInside_0.2.12.tar.gz; \ + rm -rf RInside_0.2.12.tar.gz; \ + \ + chmod u+s $(which ping); \ + #groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc; \ + \ + export VERSION_MMP=${version%-*}; \ + export PLATFORM_PACKAGE=hpccsystems-clienttools-internal_${version}.el7.x86_64.rpm; \ + wget "http://10.240.32.242/builds/LN-Candidate-${VERSION_MMP}/bin/clienttools/${PLATFORM_PACKAGE}"; \ + #echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c -; \ + yum install --nogpgcheck -y "${PLATFORM_PACKAGE}"; \ + rm -rf "${PLATFORM_PACKAGE}" + +#RUN package-cleanup --leaves --all + +ENTRYPOINT ["/docker-entrypoint.sh"] + +CMD ["/usr/sbin/sshd", "-D"] diff --git a/stack/README.md b/deployment/README.md similarity index 94% rename from stack/README.md rename to deployment/README.md index 4fb3cac..0b0c820 100644 --- a/stack/README.md +++ b/deployment/README.md @@ -1,7 +1,7 @@ ## Prerequisites -Tested on Ubuntu 16.04. -Need ubuntu 16.04 update 4 or above +Tested on Ubuntu 18.04. +Need ubuntu 18.04 update 4 or above Install git ```sh @@ -17,7 +17,7 @@ https://docs.docker.com/install/linux/docker-ce/ubuntu/#set-up-the-repository ```sh sudo docker swarm init --advertise-addr ``` - is one of the ip on this Linux +<ip> is one of the ip on this Linux For example, for Virtualbox it can be host-only or bridged ip or docker0 ip @@ -77,7 +77,7 @@ Get docker container id for admin node ./cluster_query -q id -g admin ``` -You can use to access the container: +You can use <id> to access the container: ```sh sudo docker exec -it /bin/bash ``` @@ -128,4 +128,4 @@ This will : cd to HPCC-Docker-Stack and run ```sh sudo docker stack rm hpcc -``` +``` \ No newline at end of file diff --git a/deployment/admin/Dockerfile b/deployment/admin/Dockerfile new file mode 100644 index 0000000..4265c93 --- /dev/null +++ b/deployment/admin/Dockerfile @@ -0,0 +1,32 @@ +FROM hpccsystems/hpcc-base:7-bionic + +ARG version= +RUN if [ -z "${version}" ] ; then echo Missing HPCC version; exit 1; else : ; fi + +#ENV PLATFORM_DOWNLOAD_MD5 a62b43a1ef1ecf7d8d702d9033e9cf39 +RUN set -ex; \ + apt-get update; \ + apt-get install -y iputils-ping curl xterm libarchive13 ansible iputils-ping python3-pip libssl-dev ; \ + pip3 install kubernetes ; \ + groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc ; \ + export VERSION_MMP=${version%-*}; \ + export PLATFORM_PACKAGE=hpccsystems-platform-community_${version}bionic_amd64.deb; \ + wget "http://d2wulyp08c6njk.cloudfront.net/releases/CE-Candidate-${VERSION_MMP}/bin/platform/${PLATFORM_PACKAGE}"; \ + #wget "http://10.240.32.242/builds/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}"; \ + #wget "http://10.240.32.242/builds/custom/Spark/envgen2/${PLATFORM_PACKAGE}"; \ + #echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c -; \ + dpkg -i "${PLATFORM_PACKAGE}" ;\ + rm -rf "${PLATFORM_PACKAGE}" + +EXPOSE 8015 18015 + +#COPY id_rsa* /home/hpcc/.ssh/ +#RUN set -ex; \ +# chmod 400 /home/hpcc/.ssh/id_rsa; \ +# chmod 644 /home/hpcc/.ssh/id_rsa.pub + +RUN cp -r /home/hpcc/.ssh ~/ +COPY hpcc-tools /opt/hpcc-tools +ENTRYPOINT ["/docker-entrypoint.sh"] + +CMD ["/usr/sbin/sshd", "-D"] diff --git a/stack/admin/docker-entrypoint.sh b/deployment/admin/docker-entrypoint.sh similarity index 100% rename from stack/admin/docker-entrypoint.sh rename to deployment/admin/docker-entrypoint.sh diff --git a/deployment/admin/hpcc-tools/CollectIPs.py b/deployment/admin/hpcc-tools/CollectIPs.py new file mode 100755 index 0000000..0a713d0 --- /dev/null +++ b/deployment/admin/hpcc-tools/CollectIPs.py @@ -0,0 +1,89 @@ +#!/usr/bin/python3 +import sys +import getopt +import json +import os.path +from copyreg import constructor + +class CollectIPs(object): + def __init__(self): + ''' + constructor + ''' + self._out_dir = "/tmp/hpcc_cluster" + + @property + def out_dir(self): + return self._out_dir + + @out_dir.setter + def out_dir(self, value): + self._out_dir = value + + def retrieveIPs(self, out_dir, input_fn): + try: + if out_dir: + self._out_dir = out_dir + if not os.path.exists(self._out_dir): + os.makedirs(self._out_dir) + + if input_fn.lower().endswith('.json') or \ + input_fn.lower().endswith('.lst'): + self.retrieveIPsFromCloud(input_fn) + else: + print("Unsupport input file extension\n") + except Exception as e: + raise type(e)(str(e) + + ' Error in retrive IPs').with_traceback(sys.exc_info()[2]) + + def retrieveIPsFromCloud(self, input_fn): + pass + + def clean_dir(self, target_dir): + for f in os.listdir(target_dir): + f_path = os.path.join(target_dir, f) + if os.path.isfile(f_path): + os.unlink(f_path) + + def write_to_file(self, base_dir, comp_type, ip): + file_name = os.path.join(base_dir, comp_type) + if os.path.exists(file_name): + f_ips = open (file_name, 'a') + else: + f_ips = open (file_name, 'w') + f_ips.write(ip + "\n") + f_ips.close() + + + def main(self): + try: + input_filname = "" + opts, args = getopt.getopt(sys.argv[1:],":d:i:h", + ["help", "ip-dir", "in-file"]) + + for arg, value in opts: + if arg in ("-?", "--help"): + self.usage() + exit(0) + elif arg in ("-d", "--ip-dir"): + self.out_dir = value + elif arg in ("-i", "--in-file"): + input_filename = value + + self.retrieveIPs("", input_filename) + + except getopt.GetoptError as err: + print(str(err)) + self.usage() + exit(0) + + except Exception as e: + print(e) + print("Use -h or--help to see the usage.\n"); + + def usage(self): + print("Usage CollectIPs.py [option(s)]\n") + print(" -d --ip-dir output pod ip directory. The default is /tmp/ips.") + print(" -i --in-file input docker network file in json format.") + print(" -h --help print this usage help.") + print("\n"); diff --git a/stack/admin/hpcc-tools/ansible/hpcc_run.yaml b/deployment/admin/hpcc-tools/ansible/hpcc_run.yaml similarity index 100% rename from stack/admin/hpcc-tools/ansible/hpcc_run.yaml rename to deployment/admin/hpcc-tools/ansible/hpcc_run.yaml diff --git a/stack/admin/hpcc-tools/ansible/push_env.yaml b/deployment/admin/hpcc-tools/ansible/push_env.yaml similarity index 66% rename from stack/admin/hpcc-tools/ansible/push_env.yaml rename to deployment/admin/hpcc-tools/ansible/push_env.yaml index 932bf3f..a38506a 100644 --- a/stack/admin/hpcc-tools/ansible/push_env.yaml +++ b/deployment/admin/hpcc-tools/ansible/push_env.yaml @@ -2,9 +2,9 @@ --- - name: Execute a script. hosts: '{{ hosts }}' - remote_user: 'root' + remote_user: 'hpcc' tasks: - name: Transfer the environment.xml copy: src=/etc/HPCCSystems/source/environment.xml dest=/etc/HPCCSystems/ mode=0644 - - name: Change the owner to user hpcc - command: chown hpcc:hpcc /etc/HPCCSystems/environment.xml + #- name: Change the owner to user hpcc + # command: chown hpcc:hpcc /etc/HPCCSystems/environment.xml diff --git a/stack/admin/hpcc-tools/ansible/refresh_dali.yaml b/deployment/admin/hpcc-tools/ansible/refresh_dali.yaml similarity index 92% rename from stack/admin/hpcc-tools/ansible/refresh_dali.yaml rename to deployment/admin/hpcc-tools/ansible/refresh_dali.yaml index 3bf9e4f..b3e7157 100644 --- a/stack/admin/hpcc-tools/ansible/refresh_dali.yaml +++ b/deployment/admin/hpcc-tools/ansible/refresh_dali.yaml @@ -2,7 +2,7 @@ --- - name: Execute a script. hosts: 'dali' - remote_user: 'root' + remote_user: 'hpcc' tasks: - name: Execute the script command: rm -rf /var/lib/HPCCSystems/myroxie/slaves; rm -rf /var/lib/HPCCSystems/mythor/slaves; rm -rf /var/lib/HPCCSystems/mythor/uslaves diff --git a/stack/admin/hpcc-tools/ansible/restart_thor.yaml b/deployment/admin/hpcc-tools/ansible/restart_thor.yaml similarity index 66% rename from stack/admin/hpcc-tools/ansible/restart_thor.yaml rename to deployment/admin/hpcc-tools/ansible/restart_thor.yaml index ed85f5e..628a161 100644 --- a/stack/admin/hpcc-tools/ansible/restart_thor.yaml +++ b/deployment/admin/hpcc-tools/ansible/restart_thor.yaml @@ -2,7 +2,7 @@ --- - name: Execute a script. hosts: 'thor' - remote_user: 'root' + remote_user: 'hpcc' tasks: - name: Execute the script - command: bash /etc/init.d/hpcc-init -c '{{thor_name}}' start + command: sudo /etc/init.d/hpcc-init -c '{{thor_name}}' start diff --git a/stack/admin/hpcc-tools/ansible/set_hpcc_owner.yaml b/deployment/admin/hpcc-tools/ansible/set_hpcc_owner.yaml similarity index 100% rename from stack/admin/hpcc-tools/ansible/set_hpcc_owner.yaml rename to deployment/admin/hpcc-tools/ansible/set_hpcc_owner.yaml diff --git a/stack/admin/hpcc-tools/ansible/setup.sh b/deployment/admin/hpcc-tools/ansible/setup.sh similarity index 100% rename from stack/admin/hpcc-tools/ansible/setup.sh rename to deployment/admin/hpcc-tools/ansible/setup.sh diff --git a/stack/admin/hpcc-tools/ansible/start_hpcc.yaml b/deployment/admin/hpcc-tools/ansible/start_hpcc.yaml similarity index 80% rename from stack/admin/hpcc-tools/ansible/start_hpcc.yaml rename to deployment/admin/hpcc-tools/ansible/start_hpcc.yaml index 59066d8..ceb0619 100644 --- a/stack/admin/hpcc-tools/ansible/start_hpcc.yaml +++ b/deployment/admin/hpcc-tools/ansible/start_hpcc.yaml @@ -4,7 +4,7 @@ --- - name: Execute a script. hosts: '{{ hosts }}' - remote_user: 'root' + remote_user: 'hpcc' tasks: - name: Execute the script - command: bash /etc/init.d/hpcc-init start + command: sudo /etc/init.d/hpcc-init start diff --git a/stack/admin/hpcc-tools/ansible/stop_hpcc.yaml b/deployment/admin/hpcc-tools/ansible/stop_hpcc.yaml similarity index 81% rename from stack/admin/hpcc-tools/ansible/stop_hpcc.yaml rename to deployment/admin/hpcc-tools/ansible/stop_hpcc.yaml index a2a1bb9..010b114 100644 --- a/stack/admin/hpcc-tools/ansible/stop_hpcc.yaml +++ b/deployment/admin/hpcc-tools/ansible/stop_hpcc.yaml @@ -4,7 +4,7 @@ --- - name: Execute a script. hosts: '{{ hosts }}' - remote_user: 'root' + remote_user: 'hpcc' tasks: - name: Execute the script - command: bash /etc/init.d/hpcc-init stop + command: sudo /etc/init.d/hpcc-init stop diff --git a/stack/admin/hpcc-tools/config_hpcc.sh b/deployment/admin/hpcc-tools/config_hpcc.sh similarity index 51% rename from stack/admin/hpcc-tools/config_hpcc.sh rename to deployment/admin/hpcc-tools/config_hpcc.sh index 7cbfeb2..debeac6 100755 --- a/stack/admin/hpcc-tools/config_hpcc.sh +++ b/deployment/admin/hpcc-tools/config_hpcc.sh @@ -1,17 +1,26 @@ -#!/bin/bash -e +#!/bin/bash -x SCRIPT_DIR=$(dirname $0) +if [ -n "${KUBERNETES_PORT}" ] +then + CLOUD_PLATFORM=kubernetes + source ${SCRIPT_DIR}/kube/config_hpcc_functions +else + source ${SCRIPT_DIR}/docker/config_hpcc_functions +fi + + function usage() { cat < : - -a: app name. The default is hpcc + -a: app name for docker stack. The default is hpcc -D: HPCC home directory. The default is /opt/HPCCSystems -d: directory to save collecting ips. The default is /tmp/ips -e: number of esp nodes. The default is 1 - -n: network name. The default is _ovnet. + -n: network name for docker stack. The default is _ovnet. -N: do not push environment.xml and restart environment.xml -r: number of roxie nodes -s: number of support nodes @@ -20,7 +29,8 @@ function usage() and environment.xml with real ip. Re-generate ansible host file, run updtdalienv and restart thor master. -x: do not retrieve cluster ips. The ips should be under directory /tmp/ - cluster ips file name __ ${ipDir2}/dali - if [ -e ${ipDirs2}/node ] + if [ -e ${ipDir2}/node ] then node_to_process=node else @@ -539,7 +357,14 @@ ps -efa | grep -v sshd | grep -q sshd || $SUDOCMD mkdir -p /var/run/sshd; $SUD #------------------------------------------ # Collect containers' ips # -cluster_ips=${networkName}.json +if [ "${CLOUD_PLATFORM}" = "kubernetes" ] +then + # This is Kubernetes + cluster_ips="pods_ip.lst" +else + # This is Docker Stack + cluster_ips=${networkName}.json +fi [ -d ${ipDir} ] && rm -rf ${ipDir} collect_ips diff --git a/deployment/admin/hpcc-tools/docker/CollectIPsFromDocker.py b/deployment/admin/hpcc-tools/docker/CollectIPsFromDocker.py new file mode 100755 index 0000000..49f4200 --- /dev/null +++ b/deployment/admin/hpcc-tools/docker/CollectIPsFromDocker.py @@ -0,0 +1,50 @@ +#!/usr/bin/python3 +import os +import sys +import json + +sys.path.append(os.path.dirname(os.path.abspath(__file__)) + os.sep + "..") +from CollectIPs import CollectIPs + + +class CollectIPsFromDocker (CollectIPs): + + def __init__(self): + ''' + constructor + ''' + super(CollectIPs, self).__init__() + + + def retrieveIPsFromCloud(self, input_fn): + with open(input_fn) as json_file: + network_data = json.load(json_file) + #print("open json file") + #print(repr(network_data)) + self.clean_dir(self._out_dir) + + #print(repr(network_data['Containers'])) + for key in network_data['Containers']: + node_name = (network_data['Containers'][key]['Name']).split('_')[1].split('.')[0] + if ( node_name.startswith('admin') or + node_name.startswith('dali') or + node_name.startswith('esp') or + node_name.startswith('thor') or + node_name.startswith('roxie') or + node_name.startswith('eclcc') or + node_name.startswith('scheduler') or + node_name.startswith('backup') or + node_name.startswith('sasha') or + node_name.startswith('dropzone') or + node_name.startswith('support') or + node_name.startswith('spark') or + node_name.startswith('node')): + #print("node name: " + node_name) + node_ip = (network_data['Containers'][key]['IPv4Address']).split('/')[0] + #print("node ip: " + node_ip) + self.write_to_file(self._out_dir, node_name, node_ip + ";") + +if __name__ == '__main__': + + cip = CollectIPsFromDocker() + cip.main() diff --git a/deployment/admin/hpcc-tools/docker/config_hpcc_functions b/deployment/admin/hpcc-tools/docker/config_hpcc_functions new file mode 100755 index 0000000..f22f55a --- /dev/null +++ b/deployment/admin/hpcc-tools/docker/config_hpcc_functions @@ -0,0 +1,230 @@ +function add_comp_to_envxml() +{ + _comp=$1 + _default_name=$2 + env_in=$3 + env_out=$4 + + # If nothing to process + cp $env_in $env_out + + [ ! -e ${ipDir}/${_comp}* ] && return + + index=1 + env_in_tmp=${env_in} + env_out_tmp=${wkDir}/tmp/env_out_tmp_${index}.xml + ls ${ipDir} | grep ${_comp}* | while read ip_file + do + name=$(echo ${ip_file} | cut -d '-' -s -f 2) + [ -z "$name" ] && name=$_default_name + cmd="$SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in $env_in_tmp -env-out $env_out_tmp \ + -add-node ${_comp}#${name}@ipfile=${ipDir}/${ip_file}" + + comp_opts_var="$(echo "$ip_file" | tr 'a-z' 'A-Z' | tr '-' '_')_OPTS" + comp_opts=${!comp_opts_var} + [ -n "comp_opts" ] && cmd="$cmd -mod sw:${_comp}#${name}@${comp_opts}" + + inst_comp_opts_var="INSTANCE_${comp_opts_var}" + inst_comp_opts=${!inst_comp_opts_var} + [ -n "inst_comp_opts" ] && cmd="$cmd -mod sw:${_comp}#${name}:instance@${inst_comp_opts}" + + echo "$cmd" + eval "$cmd" + + cp ${env_out_tmp} ${env_out} + + index=$(expr $index \+ 1) + env_in_tmp=${env_out_tmp} + env_out_tmp=${wkDir}/tmp/env_out_tmp_${index}.xml + done +} + +function add_roxie_to_envxml() +{ + env_in=$1 + env_out=$2 + + # If nothing to process + cp $env_in $env_out + + [ ! -e ${ipDir}/roxie* ] && return + + index=1 + env_in_tmp=${env_in} + env_out_tmp=${wkDir}/tmp/env_out_tmp_${index}.xml_ + ls ${ipDir} | grep roxie* | while read ip_file + do + roxie_name=$(echo ${ip_file} | cut -d '-' -s -f 2) + [ -z "$roxie_name" ] && roxie_name=myroxie + + # Add roxie nodes + env_out_tmp=${wkDir}/tmp/env_roxie_${index}.xml + echo "$SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in $env_in_tmp -env-out ${env_out_tmp} \ + -add-node roxie#${roxie_name}@ipfile=${ipDir}/${ip_file}" + $SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in $env_in_tmp -env-out ${env_out_tmp} \ + -add-node roxie#${roxie_name}@ipfile=${ipDir}/${ip_file} + + cp ${env_out_tmp} ${env_out} + + index=$(expr $index \+ 1) + env_in_tmp=${env_out_tmp} + env_out_tmp=${wkDir}/tmp/env_out_tmp_${index}.xml + done +} + +function add_thor_to_envxml() +{ + env_in=$1 + env_out=$2 + + # If nothing to process + cp $env_in $env_out + + [ ! -e ${ipDir}/thor-* ] && return + + index=1 + master_index=1 # In chance no master thor provided and there are multiple support nodes + env_in_tmp=${env_in} + env_out_tmp=${wkDir}/tmp/env_out_tmp_${index}.xml + ls ${ipDir} | grep thor-* | while read ip_file + do + thor_name=$(echo ${ip_file} | cut -d '-' -s -f 2) + [ -z "$thor_name" ] && thor_name=mythor + + # Add thor master node + if [ -e ${ipDir}/thormaster_${thor_name} ] + then + master_ip=$(cat ${ipDir}/thormaster_${thor_name}* | sed 's/;//g') + else + master_ip=$(cat ${ipDir}/support | head -n ${master_index} | tail -n 1 | sed 's/;//g') + let master_index="($master_index + 1) % ${support_nodes} + 1" + fi + + # Add thor nodes + + cmd="$SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in $env_in_tmp -env-out ${env_out_tmp}" + cmd="$cmd -add-node thor#${thor_name}:master@ip=${master_ip}:slave@ipfile=${ipDir}/${ip_file}" + + #thor_name + node_group_name= + found=false + for ng in $(echo $NODE_GROUP | tr ';' ' ') + do + ng_name=$(echo $ng | cut -d':' -f1) + ng_body=$(echo $ng | cut -d':' -f2) + for thor_name2 in $(echo $ng_body | tr ',' ' ') + do + if [ "$thor_name2" = "$thor_name" ] + then + node_group_name=$ng_name + found=true + break + fi + done + [ "$found" = "true" ] && break + done + [ -n "$node_group_name" ] && cmd="$cmd -mod sw:thor#${thor_name}@nodeGroup=${node_group_name}" + + echo "$cmd" + eval "$cmd" + + cp ${env_out_tmp} ${env_out} + + index=$(expr $index \+ 1) + env_in_tmp=${env_out_tmp} + env_out_tmp=${wkDir}/tmp/env_out_tmp_${index}.xml + done +} + +function create_topology() +{ + env_in=$1 + env_out=$2 + + # If nothing to process + cp $env_in $env_out + [ -z "$TOPOLOGY" ] && return + + if [ "$TOPOLOGY" = "default" ] + then + cmd="$SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in $env_in -env-out ${env_out} -add-topology default" + echo "$cmd" + eval "$cmd" + return + fi + + env_in_tmp=${env_in} + index=1 + env_out_tmp=${wkDir}/tmp/env_topo_tmp_${index}.xml + for topology in $(echo $TOPOLOGY | tr '#' ' ') + do + #echo "" + topo_name=$(echo $topology | cut -d'%' -f1) + #echo "Topology name $topo_name" + topo_body=$(echo $topology | cut -d'%' -f2) + + for cluster in $(echo $topo_body | tr ';' ' ') + do + cluster_name=$(echo $cluster | cut -d':' -f1) + cluster_body=$(echo $cluster | cut -d':' -f2) + #echo " cluster name: $cluster_name" + + cmd="$SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in $env_in_tmp -env-out ${env_out_tmp} -add-topology ${topo_name}:cluster@name=${cluster_name}" + for process in $(echo $cluster_body | tr ',' ' ') + do + process_tag=$(echo $process | cut -d'@' -f1) + process_name=$(echo $process | cut -d'@' -f2) + #echo " process: $process_tag name: $process_name" + cmd="${cmd}:${process_tag}@process=${process_name}" + done + + echo "$cmd" + eval "$cmd" + + cp ${env_out_tmp} ${env_out} + + index=$(expr $index \+ 1) + env_in_tmp=${env_out_tmp} + env_out_tmp=${wkDir}/tmp/env_topo_tmp_${index}.xml + done + done +} + +function collect_ips() +{ + mkdir -p $ipDir + trials=3 + while [ $trials -gt 0 ] + do + [ $notGetIps -eq 0 ] && ${SCRIPT_DIR}/docker/get_ips.sh ${networkName} + ${SCRIPT_DIR}/docker/CollectIPsFromDocker.py -d ${ipDir} -i /tmp/${cluster_ips} + [ $? -eq 0 ] && break + trials=$(expr $trials \- 1) + sleep 5 + done +} + +function process_category_settings() +{ + return +} + +function process_override_settings() +{ + return +} + +function process_xpathattrs_settings() +{ + return +} + +function process_comp_settings() +{ + return +} + +function add_xml_contents() +{ + return +} diff --git a/stack/admin/hpcc-tools/get_ips.sh b/deployment/admin/hpcc-tools/docker/get_ips.sh similarity index 100% rename from stack/admin/hpcc-tools/get_ips.sh rename to deployment/admin/hpcc-tools/docker/get_ips.sh diff --git a/deployment/admin/hpcc-tools/get_ips.sh b/deployment/admin/hpcc-tools/get_ips.sh new file mode 100755 index 0000000..b1f5566 --- /dev/null +++ b/deployment/admin/hpcc-tools/get_ips.sh @@ -0,0 +1,8 @@ +#!/bin/bash -e + +networkName=hpcc_ovnet +[ -n "$1" ] && networkName=$1 + +echo "wget http://${HOST_IP}:2375/networks/${networkName}" +wget http://${HOST_IP}:2375/networks/${networkName} +mv ${networkName} /tmp/${networkName}.json diff --git a/stack/admin/hpcc-tools/ip/CollectIPs.py b/deployment/admin/hpcc-tools/ip/CollectIPs.py similarity index 100% rename from stack/admin/hpcc-tools/ip/CollectIPs.py rename to deployment/admin/hpcc-tools/ip/CollectIPs.py diff --git a/deployment/admin/hpcc-tools/kube/CollectIPsFromKube.py b/deployment/admin/hpcc-tools/kube/CollectIPsFromKube.py new file mode 100755 index 0000000..687954b --- /dev/null +++ b/deployment/admin/hpcc-tools/kube/CollectIPsFromKube.py @@ -0,0 +1,56 @@ +#!/usr/bin/python3 +import os +import sys +import json + +sys.path.append(os.path.dirname(os.path.abspath(__file__)) + os.sep + "..") +from CollectIPs import CollectIPs + + +class CollectIPsFromKube (CollectIPs): + + def __init__(self): + ''' + constructor + ''' + super(CollectIPs, self).__init__() + + + def retrieveIPsFromCloud(self, input_fn): + self.clean_dir(self._out_dir) + with open(input_fn) as pods_ip_file: + lines = pods_ip_file.readlines() + + for line in lines: + full_node_name,node_ip = line.split(' ') + node_ip = node_ip.rstrip('\r\n') + node_name_items = full_node_name.split('-') + if ( len(node_name_items) >= 3 ) and ( node_name_items[0] != 'support' ): + node_name = node_name_items[0] + "-" + node_name_items[1] + elif node_name_items[0] == 'thormaster': + node_name = node_name_items[0] + '-' + node_name_items[1] + else: + node_name = node_name_items[0] + if ( 'admin' in node_name or + node_name.startswith('dali') or + node_name.startswith('esp') or + node_name.startswith('thor') or + node_name.startswith('thor_roxie') or + node_name.startswith('roxie') or + node_name.startswith('eclcc') or + node_name.startswith('scheduler') or + node_name.startswith('backup') or + node_name.startswith('sasha') or + node_name.startswith('dropzone') or + node_name.startswith('support') or + node_name.startswith('spark') or + node_name.startswith('ldap ') or + node_name.startswith('node')): + print("node name: " + node_name) + print("node ip: " + node_ip) + self.write_to_file(self._out_dir, node_name, node_ip + ";") + +if __name__ == '__main__': + + cip = CollectIPsFromKube() + cip.main() diff --git a/deployment/admin/hpcc-tools/kube/config_hpcc_functions b/deployment/admin/hpcc-tools/kube/config_hpcc_functions new file mode 100755 index 0000000..dd022f1 --- /dev/null +++ b/deployment/admin/hpcc-tools/kube/config_hpcc_functions @@ -0,0 +1,335 @@ +function add_comp_to_envxml() +{ + _comp=$1 + _default_name=$2 + env_in=$3 + env_out=$4 + + + [ ! -e ${ipDir}/${_comp}* ] && return + + env_in_temp=$env_in + env_out_temp=/tmp/env-out.xml + ls ${ipDir} | grep ${_comp}* | while read ip_file + do + name=$(echo ${ip_file} | cut -d '-' -s -f 2) + [ -z "$name" ] && name=$_default_name + cmd="$SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in $env_in_temp -env-out $env_out_temp \ + -add-node ${_comp}#${name}@ipfile=${ipDir}/${ip_file}" + + comp_opts_var="$(echo "$ip_file" | tr 'a-z' 'A-Z' | tr '-' '_')_OPTS" + comp_opts=${!comp_opts_var} + [ -n "${comp_opts}" ] && cmd="$cmd -mod sw:${_comp}#${name}@${comp_opts}" + + inst_comp_opts_var="INSTANCE_${comp_opts_var}" + inst_comp_opts=${!inst_comp_opts_var} + [ -n "${inst_comp_opts}" ] && cmd="$cmd -mod sw:${_comp}#${name}:instance@${inst_comp_opts}" + + echo "$cmd" + eval "$cmd" + + [ "$env_in_tmp" != "${env_out_temp}" ] && env_in_temp=${env_out_temp} + done + process_comp_settings ${_comp} $env_out_temp $env_out_temp + cp $env_out_temp $env_out + +} + +function add_roxie_to_envxml() +{ + env_in=$1 + env_out=$2 + + [ ! -e ${ipDir}/roxie* ] && [ ! -e ${ipDir}/thor_roxie* ] && return + + env_in_temp=$env_in + env_out_temp=/tmp/roxie_env_out.xml + ls ${ipDir} | grep roxie* | while read ip_file + do + cluster_type=$(echo ${ip_file} | cut -d '-' -s -f 1) + roxie_name=$(echo ${ip_file} | cut -d '-' -s -f 2) + if [ "cluster_type" = "thor_roxie" ] + then + roxie_name=$(echo ${roxie_name} | cut -d '_' -s -f 2) + elif [ -z "$roxie_name" ] + then + roxie_name=myroxie + fi + + # Add roxie nodes + echo "$SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in ${env_in_temp} -env-out ${env_out_temp} \ + -add-node roxie#${roxie_name}@ipfile=${ipDir}/${ip_file}" + $SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in ${env_in_temp} -env-out ${env_out_temp} \ + -add-node roxie#${roxie_name}@ipfile=${ipDir}/${ip_file} + + # todo process property file + + [ "$env_in_temp" != "${env_out_temp}" ] && env_in_temp=${env_out_temp} + done + process_comp_settings roxie $env_out_temp $env_out_temp + cp ${env_out_temp} ${env_out} + + +} + +function add_thor_to_envxml() +{ + env_in=$1 + env_out=$2 + + [ ! -e ${ipDir}/thor-* ] && [ ! -e ${ipDir}/thor_roxie-* ] && return + + env_in_temp=${env_in} + env_out_temp=/tmp/thor_env_out.xml + master_index=1 # in chance we need multiple thor masters from suppport nodes + ls ${ipDir} | grep "thor-\|thorroxie" | while read ip_file + do + cluster_type=$(echo ${ip_file} | cut -d '-' -s -f 1) + thor_name=$(echo ${ip_file} | cut -d '-' -s -f 2) + if [ "cluster_type" = "thor_roxie" ] + then + thor_name=$(echo ${thor_name} | cut -d '_' -s -f 1) + elif [ -z "$thor_name" ] + then + thor_name=mythor + fi + + + # Add thor master node + if [ -e ${ipDir}/thormaster-${thor_name} ] + then + master_ip=$(cat ${ipDir}/thormaster-${thor_name}* | sed 's/;//g') + else + master_ip=$(cat ${ipDir}/support | head -n ${master_index} | tail -n 1 | sed 's/;//g') + let master_index="($master_index + 1) % ${support_nodes} + 1" + fi + + # Add thor nodes + + cmd="$SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in $env_in_temp -env-out ${env_out_temp}" + cmd="$cmd -add-node thor#${thor_name}:master@ip=${master_ip}:slave@ipfile=${ipDir}/${ip_file}" + + #thor_name + node_group_name= + found=false + for ng in $(echo $NODE_GROUP | tr ';' ' ') + do + ng_name=$(echo $ng | cut -d':' -f1) + ng_body=$(echo $ng | cut -d':' -f2) + for thor_name2 in $(echo $ng_body | tr ',' ' ') + do + if [ "$thor_name2" = "$thor_name" ] + then + node_group_name=$ng_name + found=true + break + fi + done + [ "$found" = "true" ] && break + done + [ -n "$node_group_name" ] && cmd="$cmd -mod sw:thor#${thor_name}@nodeGroup=${node_group_name}" + + echo "$cmd" + eval "$cmd" + + [ "$env_in_temp" != "${env_out_temp}" ] && env_in_temp=${env_out_temp} + done + process_comp_settings roxie $env_out_temp $env_out_temp + cp ${env_out_temp} ${env_out} +} + +function get_prop_value() +{ + prop_file_name=$1 + prop_key=$2 + default_value=$3 + + line=$(cat $prop_file_name | grep "^[[:space:]]*${prop_key}[[:space:]]*=") + line=$(echo $line | xargs) + if [ -n "$line" ] + then + echo $line | cut -d '=' -f2 | xargs + else + echo "$default_value" + fi +} +function create_topology() +{ + env_in=$1 + env_out=$2 + + topo_props=/etc/hpcc_config_map/topology.properties + [ ! -e "$topo_props" ] && return + + cat $topo_props | grep -q "^[[:space:]]*default=true" + if [ $? -eq 0 ] + then + cmd="$SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in $env_in -env-out ${env_out} -add-topology default" + echo "$cmd" + eval "$cmd" + return + fi + + # Add Topology for roxie + env_in_temp=${env_in} + env_out_temp=/tmp/topo_env_out.xml + ls ${ipDir} | grep "roxie-\|thor-" | while read cluster + do + echo $cluster | grep -q "roxie" + if [ $? -eq 0 ] + then + cluster_tag=RoxieCluster + else + cluster_tag=ThorCluster + fi + name=$(echo ${cluster} | cut -d '-' -s -f 2) + topo_name=$(get_prop_value $topo_props "${cluster_name}.topology" "topology") + cluster_name=$(get_prop_value $topo_props "${name}.name" $name) + #get_prop_value $topo_props "${name}.name" $name + cmd="$SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in ${env_in_temp} -env-out ${env_out_temp} -add-topology ${topo_name}:cluster@name=${cluster_name}" + + cmd="${cmd}:${cluster_tag}@process=${name}" + + eclcc=$(get_prop_value $topo_props "${name}.eclccserver" "myeclccserver") + cmd="${cmd}:EclCCServerProcess@process=${eclcc}" + + eclscheduler=$(get_prop_value $topo_props "${name}.eclscheduler" "myeclscheduler") + cmd="${cmd}:EclSchedulerProcess@process=${eclscheduler}" + + if [ "$cluster_tag" = "ThorCluster" ] + then + eclagent=$(get_prop_value $topo_props "${name}.eclagent" "myeclagent") + cmd="${cmd}:EclAgentProcess@process=${eclagent}" + fi + + echo "$cmd" + eval "$cmd" + + [ "$env_in_temp" != "${env_out_temp}" ] && env_in_temp=${env_out_temp} + + done + + cp ${env_out_temp} ${env_out} + +} + +function collect_ips() +{ + mkdir -p $ipDir + trials=3 + while [ $trials -gt 0 ] + do + [ $notGetIps -eq 0 ] && ${SCRIPT_DIR}/kube/get_pods.py + ${SCRIPT_DIR}/kube/CollectIPsFromKube.py -d ${ipDir} -i /tmp/${cluster_ips} + [ $? -eq 0 ] && break + trials=$(expr $trials \- 1) + sleep 5 + done +} + +function process_category_settings() +{ + _env_in=$1 + _env_out=$2 + _tmp_file=/tmp/category_out + cp ${_env_in} ${_tmp_file} + + category_props=/etc/hpcc_config_map/category.properties + [ ! -e "$category_props" ] && return + + cmd="$SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in ${_tmp_file} -env-out ${_tmp_file}" + + while read line + do + line2=$(echo $line | grep "^[[:space:]]*[^#]") + [ -z "$line2" ] && continue + cmd="$cmd -o $line2" + done < ${category_props} + echo "$cmd" + eval "$cmd" + cp ${_tmp_file} ${_env_out} + +} + +function process_envgen2_settings() +{ + _file_in=$1 + _env_in=$2 + _env_out=$3 + _tmp_file=/tmp/envgen2_out + cp ${_env_in} ${_tmp_file} + + input_file=/etc/hpcc_config_map/${_file_in} + [ ! -e "$input_file" ] && return + + cmd_base="$SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in ${_tmp_file} -env-out ${_tmp_file}" + cat ${input_file} | grep "^[[:space:]]*[^#]" | while read line + do + cmd="${cmd_base} ${line}" + echo "$cmd" + eval "$cmd" + done + cp ${_tmp_file} ${_env_out} + +} + +function process_override_settings() +{ + _env_in=$1 + _env_out=$2 + + override_file=/etc/hpcc_config_map/override.envgen2 + [ ! -e "${override_file}" ] && return + + cmd="$SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in ${_env_in} -env-out ${_env_out}" + while read line + do + line2=$(echo $line | grep "^[[:space:]]*[^#]") + [ -z "$line2" ] && continue + export cmd="$cmd -o $line2" + done < ${override_file} + echo "$cmd" + eval "$cmd" +} + +function process_xpathattrs_settings() +{ + _env_in=$1 + _env_out=$2 + + xpathattrs_file=/etc/hpcc_config_map/xpath_attributes.properties + [ ! -e "$xpathattrs" ] && return + + cmd="$SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in ${_env_in} -env-out ${_env_out}" + while read line + do + line2=$(echo $line | grep "^[[:space:]]*[^#]") + [ -z "$line2" ] && continue + setting=$(echo $line2 | sed -e 's/\\/\\\\\\/g' | tr [','] [' '] ) + cmd="$cmd -set-xpath-attrib-value ${setting}" + done < ${xpathattrs_file} + echo "$cmd" + eval "$cmd" +} + +function add_xml_contents() +{ + _env_in=$1 + _env_out=$2 + + _tmp_file=/tmp/xml_content_out + cp ${_env_in} ${_tmp_file} + cmd_base="$SUDOCMD ${HPCC_HOME}/sbin/envgen2 -env-in ${_tmp_file} -env-out ${_tmp_file}" + contents_dir=/etc/hpcc_config_map + ls ${contents_dir}/*.xml | sort | while read file + do + xpath=$(cat ${file} | grep "^[[:space:]]*//g' | awk '{$1=$1};1') + + [ -z "${xpath}" ] && continue + cmd="${cmd_base} -add-content ${xpath} ${file}" + echo "$cmd" + eval "$cmd" + + done + cp ${_tmp_file} ${_env_out} +} diff --git a/deployment/admin/hpcc-tools/kube/events_watcher.py b/deployment/admin/hpcc-tools/kube/events_watcher.py new file mode 100755 index 0000000..83eb072 --- /dev/null +++ b/deployment/admin/hpcc-tools/kube/events_watcher.py @@ -0,0 +1,102 @@ +#!/usr/bin/python3 + +import os.path +import signal +import sys +import time +import datetime +from kubernetes import client, config, watch + +def signal_handler(signum, frame): + sys.exit(0) + +def current_time(): + ts = time.time() + return datetime.datetime.fromtimestamp(ts).strftime('%Y%m%d_%H:%M:%S') + +def main(): + # Configs can be set in Configuration class directly or using helper + # utility. If no argument provided, the config will be loaded from + # default location. + #config.load_kube_config() + config.load_incluster_config() + + v1 = client.CoreV1Api() + current_namespace = open("/var/run/secrets/kubernetes.io/serviceaccount/namespace").read() + + event_log = "/tmp/pod_events.log" + if os.path.exists(event_log): + os.remove(event_log) + + event_err = "/tmp/pod_events_error.log" + if os.path.exists(event_err): + os.remove(event_err) + + config_log = "/tmp/hpcc_config.log" + if os.path.exists(config_log): + os.remove(config_log) + + not_ready_list = [] + resource_version=0; + while True: + + #print("Wait for events ...") + #f_event = open (event_log, 'a') + #f_event.write("%s - Wait for events ... \n" % (current_time())) + #f_event.write("Number of not ready pods: %d\n" % (len(not_ready_list))) + #f_event.close() + w = watch.Watch() + try: + new_running_list = [] + f_event = open (event_log, 'a') + for event in w.stream(v1.list_namespaced_pod, namespace=current_namespace, timeout_seconds=5, resource_version=resource_version): + #print("Event: %s %s %s %s" % (event['type'],event['object'].kind, event['object'].metadata.name, event['object'].status.phase)) + f_event.write("%s - Event: %s %s %s %s\n" % (current_time(),event['type'],event['object'].kind, event['object'].metadata.name, event['object'].status.phase)) + #print("Resource_version: %s" % (event['object'].metadata.resource_version)) + if int(resource_version) < int(event['object'].metadata.resource_version): + resource_version = event['object'].metadata.resource_version + pod_name = event['object'].metadata.name + state = event['object'].status.phase + if (pod_name.startswith('dali') or + pod_name.startswith('esp') or + pod_name.startswith('thor') or + pod_name.startswith('thor_roxie') or + pod_name.startswith('roxie') or + pod_name.startswith('eclcc') or + pod_name.startswith('scheduler') or + pod_name.startswith('backup') or + pod_name.startswith('sasha') or + pod_name.startswith('dropzone') or + pod_name.startswith('support') or + pod_name.startswith('spark') or + pod_name.startswith('ldap') or + pod_name.startswith('node')): + + if state == 'Running': + if pod_name not in new_running_list: + new_running_list.append(pod_name) + if pod_name in not_ready_list: + not_ready_list.remove(pod_name) + else: + if pod_name not in not_ready_list: + not_ready_list.append(pod_name) + + f_event.close() + #print("not_ready_list size: %d, new_running_list size: %d" % (len(not_ready_list), len(new_running_list))) + if len(not_ready_list) == 0 and len(new_running_list) > 0: + # Configure and restart HPCC cluster + #print("/opt/hpcc-tools/config_hpcc.sh") + cmd = "/opt/hpcc-tools/config_hpcc.sh > " + config_log + " 2>&1" + os.system(cmd) + + except Exception as e: + #print(e) + f_error = open (event_err, 'a') + f_error.write(str(e)) + f_error.close() + + +if __name__ == '__main__': + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + main() diff --git a/deployment/admin/hpcc-tools/kube/get_ips.sh b/deployment/admin/hpcc-tools/kube/get_ips.sh new file mode 100755 index 0000000..ab44d84 --- /dev/null +++ b/deployment/admin/hpcc-tools/kube/get_ips.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +KUBE_TOKEN=$( + : + -D: run sshd with "-D" + -e: environment, such as kube, swarm, azure, gce, etc. Only kube and swarm are currently supported. + -h: print this help + +EOF + exit 2 +} + +environment=swarm +run_sshd_foreground=false + +# Process command-line parameters +while getopts "*De:h" arg +do + case $arg in + D) run_sshd_foreground=true + ;; + e) environment=${OPTARG} + ;; + h) usage + ;; + ?) + echo "Unknown option $OPTARG" + usage + ;; + esac +done + + +[ ! -e ~/.ssh/id_rsa ] && cp -r /home/hpcc/.ssh ~/ +if [ "${run_sshd_foreground}" = "true" ] +then + /usr/sbin/sshd -D +else + service ssh start + if [ "${environment}" = "kube" ] + then + ${SCRIPT_DIR}/kube/events_watcher.py + + fi +fi diff --git a/stack/admin/hpcc-tools/start_hpcc.sh b/deployment/admin/hpcc-tools/start_hpcc.sh similarity index 100% rename from stack/admin/hpcc-tools/start_hpcc.sh rename to deployment/admin/hpcc-tools/start_hpcc.sh diff --git a/stack/admin/hpcc-tools/stop_hpcc.sh b/deployment/admin/hpcc-tools/stop_hpcc.sh similarity index 100% rename from stack/admin/hpcc-tools/stop_hpcc.sh rename to deployment/admin/hpcc-tools/stop_hpcc.sh diff --git a/stack/admin/spark/spark-defaults.xsl b/deployment/admin/spark/spark-defaults.xsl similarity index 100% rename from stack/admin/spark/spark-defaults.xsl rename to deployment/admin/spark/spark-defaults.xsl diff --git a/stack/admin/spark/sparkThor.xsd b/deployment/admin/spark/sparkThor.xsd similarity index 100% rename from stack/admin/spark/sparkThor.xsd rename to deployment/admin/spark/sparkThor.xsd diff --git a/stack/admin/spark/sparkThor.xsl b/deployment/admin/spark/sparkThor.xsl similarity index 100% rename from stack/admin/spark/sparkThor.xsl rename to deployment/admin/spark/sparkThor.xsl diff --git a/stack/bin/clean_run.sh b/deployment/bin/clean_run.sh similarity index 100% rename from stack/bin/clean_run.sh rename to deployment/bin/clean_run.sh diff --git a/stack/bin/cluster_config.sh b/deployment/bin/cluster_config.sh similarity index 100% rename from stack/bin/cluster_config.sh rename to deployment/bin/cluster_config.sh diff --git a/stack/bin/cluster_env.sh b/deployment/bin/cluster_env.sh similarity index 100% rename from stack/bin/cluster_env.sh rename to deployment/bin/cluster_env.sh diff --git a/stack/bin/cluster_query.sh b/deployment/bin/cluster_query.sh similarity index 100% rename from stack/bin/cluster_query.sh rename to deployment/bin/cluster_query.sh diff --git a/stack/bin/cluster_run.sh b/deployment/bin/cluster_run.sh similarity index 100% rename from stack/bin/cluster_run.sh rename to deployment/bin/cluster_run.sh diff --git a/stack/bin/common b/deployment/bin/common similarity index 100% rename from stack/bin/common rename to deployment/bin/common diff --git a/stack/docker-stack-elk.yml b/deployment/docker-stack-elk.yml similarity index 100% rename from stack/docker-stack-elk.yml rename to deployment/docker-stack-elk.yml diff --git a/stack/docker-stack-simple.yml b/deployment/docker-stack-simple.yml similarity index 100% rename from stack/docker-stack-simple.yml rename to deployment/docker-stack-simple.yml diff --git a/stack/docker-stack-spark.yml b/deployment/docker-stack-spark.yml similarity index 100% rename from stack/docker-stack-spark.yml rename to deployment/docker-stack-spark.yml diff --git a/stack/docker-stack-template.yml b/deployment/docker-stack-template.yml similarity index 100% rename from stack/docker-stack-template.yml rename to deployment/docker-stack-template.yml diff --git a/stack/docker-stack.yml b/deployment/docker-stack.yml similarity index 100% rename from stack/docker-stack.yml rename to deployment/docker-stack.yml diff --git a/6-base/Dockerfile b/dev/7/bionic/base/7/Dockerfile similarity index 75% rename from 6-base/Dockerfile rename to dev/7/bionic/base/7/Dockerfile index a7c903b..4a23ec5 100644 --- a/6-base/Dockerfile +++ b/dev/7/bionic/base/7/Dockerfile @@ -1,11 +1,11 @@ -FROM ubuntu:xenial +FROM ubuntu:bionic RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ wget \ libaprutil1 \ - libboost-regex1.58.0 \ - libicu55 \ + libboost-regex1.65.1 \ + libicu60 \ libldap-2.4-2 \ libtbb2 \ libnuma1 \ @@ -20,12 +20,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ sudo \ libblas3 \ libatlas3-base \ + psmisc \ + libmemcached11 \ + libmemcachedutil2 \ + libpython2.7 \ + libpython3.6 \ vim \ net-tools \ - psmisc \ --fix-missing \ && rm -rf /var/lib/apt/lists/* RUN mkdir -p /var/run/dbus && mkdir -p /var/run/sshd -CMD ["/usr/sbin/sshd", "-D"] +COPY docker-entrypoint.sh / +COPY sleep.sh / + +CMD ["/sleep.sh"] diff --git a/dev/7/bionic/bldsvr/7/Dockerfile b/dev/7/bionic/bldsvr/7/Dockerfile new file mode 100644 index 0000000..c72acd5 --- /dev/null +++ b/dev/7/bionic/bldsvr/7/Dockerfile @@ -0,0 +1,72 @@ +From ubuntu:18.04 + +RUN set -ex; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + wget \ + zip \ + build-essential \ + binutils-dev \ + libldap2-dev \ + libcppunit-dev \ + libboost-regex-dev \ + libarchive-dev \ + libv8-dev \ + libapr1-dev \ + libaprutil1-dev \ + libiberty-dev \ + libhiredis-dev \ + libtbb-dev \ + libxalan-c-dev \ + libnuma-dev \ + libmemcached-dev \ + libboost-thread-dev \ + libboost-filesystem-dev \ + bison \ + flex \ + libatlas-base-dev \ + python2.7-dev \ + python3-dev \ + libgtk2.0-dev \ + libcurl4-gnutls-dev \ + default-jdk \ + g++ \ + gcc \ + curl \ + libfreetype6-dev \ + make \ + git \ + libicu-dev \ + libxslt1-dev \ + zlib1g-dev \ + libevent-dev \ + libsqlite3-dev \ + libtool \ + autotools-dev \ + automake \ + m4 \ + libssl-dev \ + --fix-missing; \ + \ + curl -sL https://deb.nodesource.com/setup_8.x | bash - ; \ + apt-get install -y nodejs; \ + \ + export cmake_version=3.15.2; \ + export cmake_name=cmake-${cmake_version}-Linux-x86_64; \ + wget https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_name}.tar.gz; \ + tar -zxf ${cmake_name}.tar.gz; \ + rm -rf ${cmake_name}.tar.gz; \ + cp -r ${cmake_name}/bin ${cmake_name}/share /usr/local/ ; \ + rm -rf ${cmake_name} + +# Import GPG key +#RUN set -ex; \ +# wget http://10.240.32.242/data3/build/gpg/HPCCSystems.priv; \ + #aws s3 cp s3://@S3_BUCKET@/HPCCSystems.priv . ; \ + #gpg --pinentry-mode=loopback --passphrase @GPG_PASSPHRASE@ --import HPCCSystems.priv; \ + #rm -rf HPCCSystems.priv + +# Documenttion build requisites +# libsaxonb-java +# fop +# xsltproc diff --git a/dev/7/bionic/platform/ce/Dockerfile b/dev/7/bionic/platform/ce/Dockerfile new file mode 100644 index 0000000..6234ff0 --- /dev/null +++ b/dev/7/bionic/platform/ce/Dockerfile @@ -0,0 +1,51 @@ +From hpccsystems/hpcc-dev:7-bionic AS bionic-builder +# Usage: sudo docker build -t --build-arg branch= --build-arg owner= . +# The default branch is "master". The default github project owner is "hpcc-systems". +# user/password is used to clone non-public project such as LN. +ARG branch= +ARG owner="hpcc-systems" +ARG user= +ARG password= + +WORKDir /hpcc-dev +RUN set -ex; \ + if [ -n "${user}" ] ; \ + then \ + echo "machine github.com login ${user} password ${password}" > ~/.netrc ; \ + else : ; fi; \ + git clone https://github.com/${owner}/HPCC-Platform.git; \ + cd HPCC-Platform; \ + if [ -n "${branch}" ] ; \ + then \ + rc=$(git tag | grep ${branch}) || : ; \ + if [ -n "$rc" ]; \ + then \ + git fetch && git fetch --tags; \ + else : ; fi; \ + git checkout ${branch}; \ + else : ; fi; \ + git submodule update --init --recursive; \ + cd ..; \ + mkdir -p build; \ + cd build; \ + rm -rf * ; \ + cmake ../HPCC-Platform; \ + make -j 4 ; \ + make package + + +From hpccsystems/hpcc-base:7-bionic + +WORKDir /Downloads +COPY --from=0 /hpcc-dev/build/hpccsystems*.deb . +RUN set -ex; \ + apt-get update -y; \ + apt-get install -y iputils-ping curl xterm libarchive13 ; \ + groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc ; \ + dpkg -i hpccsystems*.deb; \ + rm -rf hpccsystems*.deb + +EXPOSE 8010 8002 8015 9876 18010 18002 18015 19876 + +ENTRYPOINT ["/docker-entrypoint.sh"] +CD ["/usr/sbin/sshd", "-D"] diff --git a/7-base/Dockerfile b/dev/7/disco/base/7/Dockerfile similarity index 84% rename from 7-base/Dockerfile rename to dev/7/disco/base/7/Dockerfile index 5224119..7309526 100644 --- a/7-base/Dockerfile +++ b/dev/7/disco/base/7/Dockerfile @@ -1,11 +1,11 @@ -FROM ubuntu:xenial +FROM ubuntu:disco RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ wget \ libaprutil1 \ - libboost-regex1.58.0 \ - libicu55 \ + libboost-regex1.67.0 \ + libicu63 \ libldap-2.4-2 \ libtbb2 \ libnuma1 \ @@ -24,7 +24,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libmemcached11 \ libmemcachedutil2 \ libpython2.7 \ - libpython3.5 \ + libpython3.6 \ vim \ net-tools \ --fix-missing \ @@ -32,4 +32,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN mkdir -p /var/run/dbus && mkdir -p /var/run/sshd -CMD ["/usr/sbin/sshd", "-D"] +COPY docker-entrypoint.sh / +COPY sleep.sh / + +CMD ["/sleep.sh"] diff --git a/dev/7/disco/bldsvr/Dockerfile b/dev/7/disco/bldsvr/Dockerfile new file mode 100644 index 0000000..d0b4dac --- /dev/null +++ b/dev/7/disco/bldsvr/Dockerfile @@ -0,0 +1,74 @@ +From ubuntu:19.04 + +RUN set -ex; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + wget \ + zip \ + build-essential \ + binutils-dev \ + libldap2-dev \ + libcppunit-dev \ + libboost-regex-dev \ + libarchive-dev \ + libv8-dev \ + libapr1-dev \ + libaprutil1-dev \ + libiberty-dev \ + libhiredis-dev \ + libtbb-dev \ + libxalan-c-dev \ + libnuma-dev \ + libmemcached-dev \ + libboost-thread-dev \ + libboost-filesystem-dev \ + bison \ + flex \ + libatlas-base-dev \ + python2.7-dev \ + python3-dev \ + libgtk2.0-dev \ + libcurl4-gnutls-dev \ + default-jdk \ + g++ \ + gcc \ + curl \ + libfreetype6-dev \ + make \ + git \ + libicu-dev \ + libxslt1-dev \ + zlib1g-dev \ + libevent-dev \ + libsqlite3-dev \ + libtool \ + autotools-dev \ + automake \ + m4 \ + libssl-dev \ + --fix-missing; \ + \ + curl -sL https://deb.nodesource.com/setup_11.x | bash - ; \ + apt-get install -y nodejs; \ + \ + export cmake_version=3.15.2; \ + export cmake_name=cmake-${cmake_version}-Linux-x86_64; \ + wget https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_name}.tar.gz; \ + tar -zxf ${cmake_name}.tar.gz; \ + rm -rf ${cmake_name}.tar.gz; \ + cp -r ${cmake_name}/bin ${cmake_name}/share /usr/local/ ; \ + rm -rf ${cmake_name} + +# Import GPG key +#RUN set -ex; \ + #wget http://10.240.32.242/data3/build/gpg/HPCCSystems.priv; \ + #aws s3 cp s3://@S3_BUCKET@/HPCCSystems.priv . ; \ + #gpg --pinentry-mode=loopback --passphrase @GPG_PASSPHRASE@ --import HPCCSystems.priv; \ + #rm -rf HPCCSystems.priv + + + +# Documenttion build requisites +# libsaxonb-java +# fop +# xsltproc diff --git a/dev/7/disco/platform/ce/Dockerfile b/dev/7/disco/platform/ce/Dockerfile new file mode 100644 index 0000000..08d589a --- /dev/null +++ b/dev/7/disco/platform/ce/Dockerfile @@ -0,0 +1,49 @@ +From hpccsystems/hpcc-dev:7-disco AS disco-builder +# Usage: sudo docker build -t --build-arg branch= --build-arg owner= . +# The default branch is "master". The default github user is "hpcc-systems". +# user/password is used to clone non-public project such as LN. +ARG branch= +ARG owner="hpcc-systems" +ARG user= +ARG password= + +WORKDir /hpcc-dev +RUN set -ex; \ + if [ -n "${user}" ] ; \ + then \ + echo "machine github.com login ${user} password ${password}" > ~/.netrc ; \ + else : ; fi; \ + git clone https://github.com/${owner}/HPCC-Platform.git; \ + cd HPCC-Platform; \ + if [ -n "${branch}" ] ; \ + then \ + tag=$(git tag | grep ${branch}) || :; \ + if [ -n "${tag}" ]; \ + then \ + git fetch && git fetch --tags; \ + else : ; fi; \ + git checkout ${branch}; \ + else : ; fi; \ + git submodule update --init --recursive; \ + cd ..; \ + mkdir -p build; \ + cd build; \ + rm -rf * ; \ + cmake ../HPCC-Platform; \ + make -j 4 ; \ + make package + + +From hpccsystems/hpcc-base:7-disco + +WORKDir /Downloads +COPY --from=0 /hpcc-dev/build/hpccsystems*.deb . +RUN set -ex; \ + apt-get update -y; \ + apt-get install -y iputils-ping curl xterm libarchive13 ; \ + groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc ; \ + dpkg -i hpccsystems*.deb; \ + rm -rf hpccsystems*.deb + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["/usr/sbin/sshd", "-D"] diff --git a/dev/7/el7/bldsvr/Dockerfile b/dev/7/el7/bldsvr/Dockerfile new file mode 100644 index 0000000..572008d --- /dev/null +++ b/dev/7/el7/bldsvr/Dockerfile @@ -0,0 +1,89 @@ +From centos:7 + +RUN set -ex; \ + yum update -y; \ + yum install -y \ + epel-release \ + wget \ + zip \ + rpm-build \ + curl-devel \ + freetype6-devel \ + make \ + bison \ + flex \ + binutils-devel \ + libldap-devel \ + libcppunit-devel \ + libicu-devel \ + libxslt-devel \ + boost-devel \ + openssl-devel \ + libarchive-devel \ + v8-devel \ + java-1.8.0-openjdk-devel \ + apr-devel \ + apr-util-devel \ + hiredis-devel \ + tbb-devel \ + numactl-devel \ + libevent-devel \ + sqlite-devel \ + libmemcached-devel \ + memcached-devel \ + libtool \ + automake \ + make \ + m4 \ + gtk2-devel \ + atlas-devel \ + python-devel \ + gnutls \ + perl-ExtUtils-MakeMaker \ + libcouchbase-devel; \ + \ + yum install -y python34-devel; \ + export git_version=2.9.5; \ + wget https://mirrors.edge.kernel.org/pub/software/scm/git/git-${git_version}.tar.gz; \ + tar -zxvf git-${git_version}.tar.gz; \ + cd git-${git_version}; \ + ./configure; \ + make -j4; \ + make install; \ + cd .. ; \ + rm -rf git-${git_version} git-${git_version}.tar.gz; \ + \ + curl --silent --location https://rpm.nodesource.com/setup_8.x | bash -; \ + yum install -y nodejs; \ + \ + export cmake_version=3.15.2; \ + export cmake_name=cmake-${cmake_version}-Linux-x86_64; \ + wget https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_name}.tar.gz; \ + tar -zxf ${cmake_name}.tar.gz; \ + rm -rf ${cmake_name}.tar.gz; \ + cp -r ${cmake_name}/bin ${cmake_name}/share /usr/local/ ; \ + rm -rf ${cmake_name}; \ + \ + yum install -y centos-release-scl; \ + yum install -y devtoolset-7; \ + \ + wget http://mysql.mirrors.hoobly.com/Downloads/MySQL-5.6/MySQL-devel-5.6.45-1.el7.x86_64.rpm; \ + rpm -i MySQL-devel-5.6.45-1.el7.x86_64.rpm; \ + rm -rf MySQL-devel-5.6.45-1.el7.x86_64.rpm; \ + \ + yum install -y R-core-devel; \ + wget https://cran.r-project.org/src/contrib/Archive/Rcpp/Rcpp_0.12.19.tar.gz; \ + R CMD INSTALL Rcpp_0.12.19.tar.gz; \ + rm -rf Rcpp_0.12.19.tar.gz; \ + \ + wget https://cran.r-project.org/src/contrib/Archive/RInside/RInside_0.2.12.tar.gz; \ + R CMD INSTALL RInside_0.2.12.tar.gz; \ + rm -rf RInside_0.2.12.tar.gz + +# Import GPG key +RUN set -ex; \ + wget http://10.240.32.242/data3/build/gpg/HPCCSystems.priv; \ + #aws s3 cp s3://@S3_BUCKET@/HPCCSystems.priv . ; \ + #gpg --passphrase @GPG_PASSPHRASE@ --import HPCCSystems.priv; \ + gpg --passphrase icanspellthis --import HPCCSystems.priv; \ + rm -rf HPCCSystems.priv diff --git a/dev/7/el7/platform/ln/Dockerfile b/dev/7/el7/platform/ln/Dockerfile new file mode 100644 index 0000000..ea65c92 --- /dev/null +++ b/dev/7/el7/platform/ln/Dockerfile @@ -0,0 +1,99 @@ +From hpccsystems/hpcc-ln-dev:7-el7 AS el7-builder +# Usage: sudo docker build -t --build-arg branch= --build-arg owner= . +# or sudo docker build -t --build-arg branch= --build-arg owner= --build-arg user= --build-arg password= . +# or sudo docker build -t --build-arg branch= --build-arg ln_owner= --build-arg ln_user= --build-arg ln_password= --build-arg ln_branch=. +# The default branch is "master". The default github user is "hpcc-systems". +ARG branch= +ARG ln_branch= +ARG owner="hpcc-systems" +ARG user= +ARG password= +ARG ln_owner="hpcc-systems" +ARG ln_user= +ARG ln_password= +ARG passphrase= + +WORKDir /hpcc-dev +RUN set -ex; \ + if [ -z "${user}" ] ; \ + then \ + git clone https://github.com/${owner}/HPCC-Platform.git; \ + else \ + echo "machine github.com login ${user} password ${password}" > ~/.netrc ; \ + git clone https://github.com/${owner}/HPCC-Platform.git; \ + fi; \ + cd HPCC-Platform; \ + if [ -n "${branch}" ] ; \ + then \ + rc=$(git tag | grep ${branch}) || : ; \ + if [ "$rc" ]; \ + then \ + git fetch && git fetch --tags; \ + else : ; fi; \ + git checkout ${branch}; \ + else : ; fi; \ + git submodule update --init --recursive; \ + cd ..; \ + if [ -z "${ln_user}" ] ; \ + then \ + git clone https://github.com/${ln_owner}/LN.git; \ + else \ + echo "machine github.com login ${ln_user} password ${ln_password}" > ~/.netrc; \ + git clone https://github.com/${ln_owner}/LN.git; \ + fi; \ + if [ -n "${branch}" ] ; \ + then \ + cd LN; \ + git tag | grep ${branch}; \ + if [ $? -eq 0 ]; \ + then \ + git fetch && git fetch --tags; \ + fi; \ + git checkout ${branch}; \ + cd .. ; \ + else : ; fi; \ + mkdir -p build; \ + cd build; \ + rm -rf * ; \ + source /opt/rh/devtoolset-7/enable; \ + if [ -n "${passphrase}" ] ; \ + then \ + cmake -DBUILD_LEVEL=INTERNAL -DCMAKE_CXX_FLAGS=-D_DALIUSER_STACKTRACE -DUSE_LIBARCHIVE=false -DUSE_OPTIONAL=ON -DINCLUDE_PLUGINS=ON -DSUPPRESS_SPARK=ON -DUSE_SHLIBDEPS=ON -DUSE_MYSQL=ON -DUSE_MYSQL=ON -DSIGN_MODULES=ON -DSIGN_MODULES_PASSPHRASE=${passphrase} -DSIGN_MODULES_KEYID=8DAA417D ../LN; \ + else \ + cmake -DBUILD_LEVEL=INTERNAL -DCMAKE_CXX_FLAGS=-D_DALIUSER_STACKTRACE -DUSE_LIBARCHIVE=false -DUSE_OPTIONAL=ON -DINCLUDE_PLUGINS=ON -DSUPPRESS_SPARK=ON -DUSE_SHLIBDEPS=ON -DUSE_MYSQL=ON -DUSE_MYSQL=ON ../LN; \ + fi; \ + make -j 4 ; \ + make package + + +From hpccsystems/hpcc-base:7-el7 + +WORKDir /Downloads +COPY --from=0 /hpcc-dev/build/hpccsystems*.rpm . + +RUN yum update -y && yum install -y \ + java-1.8.0-openjdk \ + R-core-devel +RUN set -ex; \ + wget http://mysql.mirrors.hoobly.com/Downloads/MySQL-5.6/MySQL-devel-5.6.45-1.el7.x86_64.rpm; \ + rpm -i MySQL-devel-5.6.45-1.el7.x86_64.rpm; \ + rm -rf MySQL-devel-5.6.45-1.el7.x86_64.rpm; \ + \ + wget https://cran.r-project.org/src/contrib/Archive/Rcpp/Rcpp_0.12.19.tar.gz; \ + R CMD INSTALL Rcpp_0.12.19.tar.gz; \ + rm -rf Rcpp_0.12.19.tar.gz; \ + \ + wget https://cran.r-project.org/src/contrib/Archive/RInside/RInside_0.2.12.tar.gz; \ + R CMD INSTALL RInside_0.2.12.tar.gz; \ + rm -rf RInside_0.2.12.tar.gz; \ + \ + chmod u+s $(which ping); \ + groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc ; \ + yum install --nogpgcheck -y hpccsystems*.rpm; \ + rm -rf hpccsystems*.rpm + +EXPOSE 8010 8002 8015 9876 18010 18002 18015 19876 + +#RUN cp -r /home/hpcc/.ssh ~/ +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["/usr/sbin/sshd", "-D"] diff --git a/dev/7/eoan/bldsvr/7/Dockerfile b/dev/7/eoan/bldsvr/7/Dockerfile new file mode 100644 index 0000000..13a202a --- /dev/null +++ b/dev/7/eoan/bldsvr/7/Dockerfile @@ -0,0 +1,75 @@ +From ubuntu:19.10 + +# fails: curl -sL https://deb.nodesource.com/setup_12.x | bash - ; \ +## Your distribution, identified as "Ubuntu Eoan Ermine (development branch)", is a pre-release version of Ubuntu. NodeSource does not maintain official support for Ubuntu versions until they are formally released. You can try using the manual installation instructions available at https://github.com/nodesource/distributions and use the latest supported Ubuntu version name as the distribution identifier, although this is not guaranteed to work. + +RUN set -ex; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + wget \ + zip \ + build-essential \ + binutils-dev \ + libldap2-dev \ + libcppunit-dev \ + libboost-regex-dev \ + libarchive-dev \ + libv8-dev \ + libapr1-dev \ + libaprutil1-dev \ + libiberty-dev \ + libhiredis-dev \ + libtbb-dev \ + libxalan-c-dev \ + libnuma-dev \ + libmemcached-dev \ + libboost-thread-dev \ + libboost-filesystem-dev \ + bison \ + flex \ + libatlas-base-dev \ + python2.7-dev \ + python3-dev \ + libgtk2.0-dev \ + libcurl4-gnutls-dev \ + default-jdk \ + g++ \ + gcc \ + curl \ + libfreetype6-dev \ + make \ + git \ + libicu-dev \ + libxslt1-dev \ + zlib1g-dev \ + libevent-dev \ + libsqlite3-dev \ + libtool \ + autotools-dev \ + automake \ + m4 \ + libssl-dev \ + --fix-missing; \ + \ + curl -sL https://deb.nodesource.com/setup_12.x | bash - ; \ + apt-get install -y nodejs; \ + \ + export cmake_version=3.15.2; \ + export cmake_name=cmake-${cmake_version}-Linux-x86_64; \ + wget https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_name}.tar.gz; \ + tar -zxf ${cmake_name}.tar.gz; \ + rm -rf ${cmake_name}.tar.gz; \ + cp -r ${cmake_name}/bin ${cmake_name}/share /usr/local/ ; \ + rm -rf ${cmake_name} + +# Import GPG key +#RUN set -ex; \ +# wget http://10.240.32.242/data3/build/gpg/HPCCSystems.priv +# #aws s3 cp s3://@S3_BUCKET@/HPCCSystems.priv . +# gpg --passphrase @GPG_PASSPHRASE@ --import HPCCSystems.priv +# rm -rf HPCCSystems.priv + +# Documenttion build requisites +# libsaxonb-java +# fop +# xsltproc diff --git a/dev/7/gcc7/bldsvr/Dockerfile b/dev/7/gcc7/bldsvr/Dockerfile new file mode 100644 index 0000000..75001d3 --- /dev/null +++ b/dev/7/gcc7/bldsvr/Dockerfile @@ -0,0 +1,73 @@ +From gcc:7 + +RUN set -ex; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + zip \ + build-essential \ + binutils-dev \ + libldap2-dev \ + libcppunit-dev \ + libboost-regex-dev \ + libarchive-dev \ + libv8-dev \ + libapr1-dev \ + libaprutil1-dev \ + libiberty-dev \ + libhiredis-dev \ + libtbb-dev \ + libxalan-c-dev \ + libnuma-dev \ + libmemcached-dev \ + libboost-thread-dev \ + libboost-filesystem-dev \ + bison \ + flex \ + libatlas-base-dev \ + python2.7-dev \ + python3-dev \ + libgtk2.0-dev \ + libcurl4-gnutls-dev \ + default-jdk \ +# gcc already has following packages + #g++ \ + #gcc \ + #curl \ + #libfreetype6-dev \ + #make \ + #git \ + #libicu-dev \ + #libxslt1-dev \ + #zlib1g-dev \ + #libevent-dev \ + #libsqlite3-dev \ + #libtool \ + #autotools-dev \ + #automake \ + #m4 \ + #libssl-dev \ + --fix-missing; \ + \ + curl -sL https://deb.nodesource.com/setup_12.x | bash - ; \ + apt-get install -y nodejs; \ + \ + export cmake_version=3.15.2; \ + export cmake_name=cmake-${cmake_version}-Linux-x86_64; \ + wget https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_name}.tar.gz; \ + tar -zxf ${cmake_name}.tar.gz; \ + rm -rf ${cmake_name}.tar.gz; \ + cp -r ${cmake_name}/bin ${cmake_name}/share /usr/local/ ; \ + rm -rf ${cmake_name} + +# Import GPG key +#RUN set -ex; \ + #wget http://10.240.32.242/data3/build/gpg/HPCCSystems.priv; \ + #aws s3 cp s3://@S3_BUCKET@/HPCCSystems.priv . ; \ + #gpg --pinentry-mode=loopback --passphrase @GPG_PASSPHRASE@ --import HPCCSystems.priv; \ + #rm -rf HPCCSystems.priv + + +# Documenttion build requisites +# libsaxonb-java +# fop +# xsltproc diff --git a/dev/7/gcc7/platform/ce/Dockerfile b/dev/7/gcc7/platform/ce/Dockerfile new file mode 100644 index 0000000..8a03604 --- /dev/null +++ b/dev/7/gcc7/platform/ce/Dockerfile @@ -0,0 +1,47 @@ +From hpccsystems/hpcc-dev:7-gcc-7 AS gcc-7-builder +# Usage: sudo docker build -t --build-arg branch= --build-arg owner= . +# The default branch is "master". The default github repo owner is "hpcc-systems". +ARG branch= +ARG owner="hpcc-systems" +ARG user= +ARG password= +WORKDir /hpcc-dev +RUN set -ex; \ + if [ -n "${user}" ] ; \ + then \ + echo "machine github.com login ${user} password ${password}" > ~/.netrc ; \ + else : ; fi; \ + git clone https://github.com/${owner}/HPCC-Platform.git; \ + cd HPCC-Platform; \ + if [ -n "${branch}" ] ; \ + then \ + rc=$(git tag | grep ${branch}) || : ; \ + if [ $? -eq 0 ]; \ + then \ + git fetch && git fetch --tags; \ + else : ; fi; \ + git checkout ${branch}; \ + else : ; fi; \ + git submodule update --init --recursive; \ + cd ..; \ + mkdir -p build; \ + cd build; \ + rm -rf * ; \ + cmake ../HPCC-Platform; \ + make -j 4 ; \ + make package + + +From hpccsystems/hpcc-base:7-bionic + +WORKDir /Downloads +COPY --from=0 /hpcc-dev/build/hpccsystems*.deb . +RUN set -ex; \ + apt-get update -y; \ + apt-get install -y iputils-ping curl xterm libarchive13 ; \ + groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc ; \ + dpkg -i hpccsystems*.deb; \ + rm -rf hpccsystems*.deb + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["/usr/sbin/sshd", "-D"] diff --git a/dev/7/gcc8/bldsvr/Dockerfile b/dev/7/gcc8/bldsvr/Dockerfile new file mode 100644 index 0000000..f1be3fd --- /dev/null +++ b/dev/7/gcc8/bldsvr/Dockerfile @@ -0,0 +1,72 @@ +From gcc:8 + +RUN set -ex; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + zip \ + build-essential \ + binutils-dev \ + libldap2-dev \ + libcppunit-dev \ + libboost-regex-dev \ + libarchive-dev \ + libv8-dev \ + libapr1-dev \ + libaprutil1-dev \ + libiberty-dev \ + libhiredis-dev \ + libtbb-dev \ + libxalan-c-dev \ + libnuma-dev \ + libmemcached-dev \ + libboost-thread-dev \ + libboost-filesystem-dev \ + bison \ + flex \ + libatlas-base-dev \ + python2.7-dev \ + python3-dev \ + libgtk2.0-dev \ + libcurl4-gnutls-dev \ + default-jdk \ +# gcc already has following packages + #g++ \ + #gcc \ + #curl \ + #libfreetype6-dev \ + #make \ + #git \ + #libicu-dev \ + #libxslt1-dev \ + #zlib1g-dev \ + #libevent-dev \ + #libsqlite3-dev \ + #libtool \ + #autotools-dev \ + #automake \ + #m4 \ + #libssl-dev \ + --fix-missing; \ + \ + curl -sL https://deb.nodesource.com/setup_12.x | bash - ; \ + apt-get install -y nodejs; \ + \ + export cmake_version=3.15.2; \ + export cmake_name=cmake-${cmake_version}-Linux-x86_64; \ + wget https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_name}.tar.gz; \ + tar -zxf ${cmake_name}.tar.gz; \ + rm -rf ${cmake_name}.tar.gz; \ + cp -r ${cmake_name}/bin ${cmake_name}/share /usr/local/ ; \ + rm -rf ${cmake_name} + +# Import GPG key +#RUN set -ex; \ + #wget http://10.240.32.242/data3/build/gpg/HPCCSystems.priv ; \ + #aws s3 cp s3://@S3_BUCKET@/HPCCSystems.priv . ; \ + #gpg --pinentry-mode=loopback --passphrase @GPG_PASSPHRASE@ --import HPCCSystems.priv; \ + #rm -rf HPCCSystems.priv + +# Documenttion build requisites +# libsaxonb-java +# fop +# xsltproc diff --git a/dev/7/gcc8/platform/ce/Dockerfile b/dev/7/gcc8/platform/ce/Dockerfile new file mode 100644 index 0000000..5a10ade --- /dev/null +++ b/dev/7/gcc8/platform/ce/Dockerfile @@ -0,0 +1,47 @@ +From hpccsystems/hpcc-dev:7-gcc-8 AS gcc-8-builder +# Usage: sudo docker build -t --build-arg branch= --build-arg owner= . +# The default branch is "master". The default github repo owner is "hpcc-systems". +ARG branch= +ARG owner="hpcc-systems" +ARG user= +ARG password= +WORKDir /hpcc-dev +RUN set -ex; \ + if [ -n "${user}" ] ; \ + then \ + echo "machine github.com login ${user} password ${password}" > ~/.netrc ; \ + else : ; fi; \ + git clone https://github.com/${owner}/HPCC-Platform.git; \ + cd HPCC-Platform; \ + if [ -n "${branch}" ] ; \ + then \ + rc=$(git tag | grep ${branch}) || : ; \ + if [ $? -eq 0 ]; \ + then \ + git fetch && git fetch --tags; \ + else : ; fi; \ + git checkout ${branch}; \ + else : ; fi; \ + git submodule update --init --recursive; \ + cd ..; \ + mkdir -p build; \ + cd build; \ + rm -rf * ; \ + cmake ../HPCC-Platform; \ + make -j 4 ; \ + make package + + +From hpccsystems/hpcc-base:7-bionic + +WORKDir /Downloads +COPY --from=0 /hpcc-dev/build/hpccsystems*.deb . +RUN set -ex; \ + apt-get update -y; \ + apt-get install -y iputils-ping curl xterm libarchive13 ; \ + groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc ; \ + dpkg -i hpccsystems*.deb; \ + rm -rf hpccsystems*.deb + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["/usr/sbin/sshd", "-D"] diff --git a/dev/7/gcc9/bldsvr/Dockerfile b/dev/7/gcc9/bldsvr/Dockerfile new file mode 100644 index 0000000..32bb1a2 --- /dev/null +++ b/dev/7/gcc9/bldsvr/Dockerfile @@ -0,0 +1,73 @@ +From gcc:9 + +RUN set -ex; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + zip \ + build-essential \ + binutils-dev \ + libldap2-dev \ + libcppunit-dev \ + libboost-regex-dev \ + libarchive-dev \ + libv8-dev \ + libapr1-dev \ + libaprutil1-dev \ + libiberty-dev \ + libhiredis-dev \ + libtbb-dev \ + libxalan-c-dev \ + libnuma-dev \ + libmemcached-dev \ + libboost-thread-dev \ + libboost-filesystem-dev \ + bison \ + flex \ + libatlas-base-dev \ + python2.7-dev \ + python3-dev \ + libgtk2.0-dev \ + libcurl4-gnutls-dev \ + default-jdk \ +# gcc already has following packages + #g++ \ + #gcc \ + #curl \ + #libfreetype6-dev \ + #make \ + #git \ + #libicu-dev \ + #libxslt1-dev \ + #zlib1g-dev \ + #libevent-dev \ + #libsqlite3-dev \ + #libtool \ + #autotools-dev \ + #automake \ + #m4 \ + #libssl-dev \ + --fix-missing; \ + \ + curl -sL https://deb.nodesource.com/setup_12.x | bash - ; \ + apt-get install -y nodejs; \ + \ + export cmake_version=3.15.2; \ + export cmake_name=cmake-${cmake_version}-Linux-x86_64; \ + wget https://github.com/Kitware/CMake/releases/download/v${cmake_version}/${cmake_name}.tar.gz; \ + tar -zxf ${cmake_name}.tar.gz; \ + rm -rf ${cmake_name}.tar.gz; \ + cp -r ${cmake_name}/bin ${cmake_name}/share /usr/local/ ; \ + rm -rf ${cmake_name} + +# Import GPG key +RUN set -ex; \ + wget http://10.240.32.242/data3/build/gpg/HPCCSystems.priv; \ + #aws s3 cp s3://@S3_BUCKET@/HPCCSystems.priv . ; \ + #gpg --pinentry-mode=loopback --passphrase @GPG_PASSPHRASE@ --import HPCCSystems.priv; \ + gpg --pinentry-mode=loopback --passphrase icanspellthis --import HPCCSystems.priv; \ + rm -rf HPCCSystems.priv + +# Documenttion build requisites +# libsaxonb-java +# fop +# xsltproc diff --git a/dev/7/gcc9/platform/ce/Dockerfile b/dev/7/gcc9/platform/ce/Dockerfile new file mode 100644 index 0000000..aab9002 --- /dev/null +++ b/dev/7/gcc9/platform/ce/Dockerfile @@ -0,0 +1,47 @@ +From hpccsystems/hpcc-dev:7-gcc-9 AS gcc-9-builder +# Usage: sudo docker build -t --build-arg branch= --build-arg owner= . +# The default branch is "master". The default github repo owner is "hpcc-systems". +ARG branch= +ARG owner="hpcc-systems" +ARG user= +ARG password= +WORKDir /hpcc-dev +RUN set -ex; \ + if [ -n "${user}" ] ; \ + then \ + echo "machine github.com login ${user} password ${password}" > ~/.netrc ; \ + else : ; fi; \ + git clone https://github.com/${owner}/HPCC-Platform.git; \ + cd HPCC-Platform; \ + if [ -n "${branch}" ] ; \ + then \ + rc=$(git tag | grep ${branch}) || : ; \ + if [ "$rc" ]; \ + then \ + git fetch && git fetch --tags; \ + else : ; fi; \ + git checkout ${branch}; \ + else : ; fi; \ + git submodule update --init --recursive; \ + cd ..; \ + mkdir -p build; \ + cd build; \ + rm -rf * ; \ + cmake ../HPCC-Platform; \ + make -j 4 ; \ + make package + + +From hpccsystems/hpcc-base:7-bionic + +WORKDir /Downloads +COPY --from=0 /hpcc-dev/build/hpccsystems*.deb . +RUN set -ex; \ + apt-get update -y; \ + apt-get install -y iputils-ping curl xterm libarchive13 ; \ + groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc ; \ + dpkg -i hpccsystems*.deb; \ + rm -rf hpccsystems*.deb + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["/usr/sbin/sshd", "-D"] diff --git a/dev/README.md b/dev/README.md new file mode 100644 index 0000000..e69de29 diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 4343cf9..772d05d 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,8 +1,13 @@ #!/bin/sh set -e -/usr/bin/ssh-keygen -A -#/etc/init.d/hpcc-init start +if [ "${START_SSHD}" = "true" ] +then + /usr/bin/ssh-keygen -A + service ssh start +fi + + [ "${START_HPCC}" = "true" ] && /etc/init.d/hpcc-init start if [ "${EXEC_IN_LOOP}" = "true" ] then diff --git a/platform/ce/Dockerfile b/platform/ce/Dockerfile new file mode 100644 index 0000000..41059c7 --- /dev/null +++ b/platform/ce/Dockerfile @@ -0,0 +1,26 @@ +FROM hpccsystems/hpcc-base:7-bionic + +ARG version= +RUN if [ -z "${version}" ] ; then echo Missing HPCC version; exit 1; else : ; fi + +RUN set -ex; \ + apt-get update; \ + apt-get install -y iputils-ping curl xterm libarchive13 gpg ; \ + groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc ; \ + export VERSION_MMP=${version%-*}; \ + export PLATFORM_PACKAGE=hpccsystems-platform-community_${version}bionic_amd64.deb; \ + wget "http://d2wulyp08c6njk.cloudfront.net/releases/CE-Candidate-${VERSION_MMP}/bin/platform/${PLATFORM_PACKAGE}"; \ + #echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c -; \ + dpkg -i "${PLATFORM_PACKAGE}" ;\ + rm -rf "${PLATFORM_PACKAGE}" + + +EXPOSE 8010 8002 8015 9876 18010 18002 18015 19876 + +#COPY id_rsa* /home/hpcc/.ssh/ + +#RUN cp -r /home/hpcc/.ssh ~/ + +ENTRYPOINT ["/docker-entrypoint.sh"] + +CMD ["/usr/sbin/sshd", "-D"] diff --git a/platform/ce/Dockerfile-local b/platform/ce/Dockerfile-local new file mode 100644 index 0000000..717be62 --- /dev/null +++ b/platform/ce/Dockerfile-local @@ -0,0 +1,21 @@ +FROM hpccsystems/hpcc-base:7-bionic + +COPY hpccsystems-platform* . +RUN set -ex; \ + apt-get update; \ + apt-get install -y iputils-ping curl xterm libarchive13 gpg ; \ + groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc ; \ + export PLATFORM_PACKAGE=$(ls hpccsystems-platform*); \ + dpkg -i "${PLATFORM_PACKAGE}" ; \ + rm -rf "${PLATFORM_PACKAGE}" + + +EXPOSE 8010 8002 8015 9876 18010 18002 18015 19876 + +#COPY id_rsa* /home/hpcc/.ssh/ + +#RUN cp -r /home/hpcc/.ssh ~/ + +ENTRYPOINT ["/docker-entrypoint.sh"] + +CMD ["/usr/sbin/sshd", "-D"] diff --git a/platform/ce/README.md b/platform/ce/README.md new file mode 100644 index 0000000..13b45d5 --- /dev/null +++ b/platform/ce/README.md @@ -0,0 +1,57 @@ +# Supported tags and respective Dockerfile links +- [7.4.22-1, 7.4, 7](https://github.com/hpcc-systems/docker-hpcc/tree/platform/ce/Dockerfile) + +# Quick reference +- **Where to get help**: + [HPCCSystems Forum](https://hpccsystems.com/bb/) +- **Where to file issues**: + [JIRA HPCC](https://track.hpccsystems.com/projects/HPCC/summary) +- **Maintained by**: +builds@hpccsystems.com +- **Source of this project**: + [https://github.com/hpcc-systems/HPCC-Platform](https://github.com/hpcc-systems/HPCC-Platform) + + +# What is HPCC Systems(r) + +The HPCC Systems server platform is a free, open source, massively scalable platform for big data analytics. Download the HPCC Systems server platform now and take the reins of the same core technology that LexisNexis has used for over a decade to analyze massive data sets for its customers in industry, law enforcement, government, and science. + +For more information and related downloads for HPCC Systems products, please visit +https://hpccsystems.com + +![alt text](https://hpccsystems.com/sites/default/files/hpcc-systems-horiz.png "HPCC Systems Logo") + +# How to use this image +You can start the Docker HPCC Systems image in interactive (-i -t) or daemon mode (-d). You must start the HPCC Systems processes then go to ECLWatch to submit jobs, query, and explore your data with the HPCC Systems platform. + +To start Docker on in interactive mode : +```console +sudo docker run -t -i hpccsystems/platform /bin/bash +``` +For start CentOS Docker image +```console +sudo docker run -t -i --cap-add SYS_RESOURCE -e "container=docker" /bin/bash +``` + +To start Docker in daemon mode +```console +sudo docker run -d hpccsystems/platform +``` + + +# How is this image build +```console +sudo docker build -t : --build-arg version= . +``` +For example, +```console +sudo docker build -t hpccsystems/platform:7.4 --build-arg version=7.4.8-1 . +``` + +If you have local HPCC Systems Platform package you can use Dockerfile-local. It assumes that there is package with prefix hpccsystems-platform on local directory. + +```console +sudo docker build -t : -f Dockerfile-local . +``` +# License +Licensed under the Apache License, Version 2.0 (the "License") diff --git a/platform/ce/base/7/Dockerfile b/platform/ce/base/7/Dockerfile new file mode 100644 index 0000000..4a23ec5 --- /dev/null +++ b/platform/ce/base/7/Dockerfile @@ -0,0 +1,38 @@ +FROM ubuntu:bionic + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + wget \ + libaprutil1 \ + libboost-regex1.65.1 \ + libicu60 \ + libldap-2.4-2 \ + libtbb2 \ + libnuma1 \ + libxml2 \ + libxslt1.1 \ + g++ \ + openssh-client \ + openssh-server \ + expect \ + rsync \ + python \ + sudo \ + libblas3 \ + libatlas3-base \ + psmisc \ + libmemcached11 \ + libmemcachedutil2 \ + libpython2.7 \ + libpython3.6 \ + vim \ + net-tools \ + --fix-missing \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /var/run/dbus && mkdir -p /var/run/sshd + +COPY docker-entrypoint.sh / +COPY sleep.sh / + +CMD ["/sleep.sh"] diff --git a/platform/ln/Dockerfile b/platform/ln/Dockerfile new file mode 100644 index 0000000..747879a --- /dev/null +++ b/platform/ln/Dockerfile @@ -0,0 +1,45 @@ +FROM hpccsystems/hpcc-base:7-el7 + +ARG version= +RUN if [ -z "${version}" ] ; then echo Missing HPCC version; exit 1; else : ; fi + +RUN yum update -y && yum install -y \ +# curl \ +# git \ + java-1.8.0-openjdk \ + gpg \ + R-core-devel + +RUN set -ex; \ + wget http://mysql.mirrors.hoobly.com/Downloads/MySQL-5.6/MySQL-devel-5.6.45-1.el7.x86_64.rpm; \ + rpm -i MySQL-devel-5.6.45-1.el7.x86_64.rpm; \ + rm -rf MySQL-devel-5.6.45-1.el7.x86_64.rpm; \ + \ + wget https://cran.r-project.org/src/contrib/Archive/Rcpp/Rcpp_0.12.19.tar.gz; \ + R CMD INSTALL Rcpp_0.12.19.tar.gz; \ + rm -rf Rcpp_0.12.19.tar.gz; \ + \ + wget https://cran.r-project.org/src/contrib/Archive/RInside/RInside_0.2.12.tar.gz; \ + R CMD INSTALL RInside_0.2.12.tar.gz; \ + rm -rf RInside_0.2.12.tar.gz; \ + \ + chmod u+s $(which ping); \ + groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc; \ + \ + export VERSION_MMP=${version%-*}; \ + export PLATFORM_PACKAGE=hpccsystems-platform-internal_${version}.el7.x86_64.rpm; \ + wget "http://10.240.32.242/builds/LN-Candidate-${VERSION_MMP}/bin/platform-withplugins/${PLATFORM_PACKAGE}"; \ + #echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c -; \ + yum install --nogpgcheck -y "${PLATFORM_PACKAGE}"; \ + rm -rf "${PLATFORM_PACKAGE}" + +#RUN package-cleanup --leaves --all + +EXPOSE 8010 8002 8015 9876 18010 18002 18015 19876 + +#COPY id_rsa* /home/hpcc/.ssh/ +#RUN cp -r /home/hpcc/.ssh ~/ + +ENTRYPOINT ["/docker-entrypoint.sh"] + +CMD ["/usr/sbin/sshd", "-D"] diff --git a/platform/ln/Dockerfile-local b/platform/ln/Dockerfile-local new file mode 100644 index 0000000..a5de118 --- /dev/null +++ b/platform/ln/Dockerfile-local @@ -0,0 +1,46 @@ +FROM hpccsystems/hpcc-base:7-el7 + +ARG version= +RUN if [ -z "${version}" ] ; then echo Missing HPCC version; exit 1; else : ; fi + +COPY hpccsystems-platform* . + +RUN yum update -y && yum install -y \ +# curl \ +# git \ + java-1.8.0-openjdk \ + gpg \ + R-core-devel + +RUN set -ex; \ + wget http://mysql.mirrors.hoobly.com/Downloads/MySQL-5.6/MySQL-devel-5.6.45-1.el7.x86_64.rpm; \ + rpm -i MySQL-devel-5.6.45-1.el7.x86_64.rpm; \ + rm -rf MySQL-devel-5.6.45-1.el7.x86_64.rpm; \ + \ + wget https://cran.r-project.org/src/contrib/Archive/Rcpp/Rcpp_0.12.19.tar.gz; \ + R CMD INSTALL Rcpp_0.12.19.tar.gz; \ + rm -rf Rcpp_0.12.19.tar.gz; \ + \ + wget https://cran.r-project.org/src/contrib/Archive/RInside/RInside_0.2.12.tar.gz; \ + R CMD INSTALL RInside_0.2.12.tar.gz; \ + rm -rf RInside_0.2.12.tar.gz; \ + \ + chmod u+s $(which ping); \ + groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc; \ + \ + export VERSION_MMP=${version%-*}; \ + export PLATFORM_PACKAGE=hpccsystems-platform-internal_${version}.el7.x86_64.rpm; \ + export PLATFORM_PACKAGE=$(ls hpccsystems-platform*); \ + yum install --nogpgcheck -y "${PLATFORM_PACKAGE}"; \ + rm -rf "${PLATFORM_PACKAGE}" + +#RUN package-cleanup --leaves --all + +EXPOSE 8010 8002 8015 9876 18010 18002 18015 19876 + +#COPY id_rsa* /home/hpcc/.ssh/ +#RUN cp -r /home/hpcc/.ssh ~/ + +ENTRYPOINT ["/docker-entrypoint.sh"] + +CMD ["/usr/sbin/sshd", "-D"] diff --git a/7-base/platform/centos/Dockerfile b/platform/ln/base/7/Dockerfile similarity index 63% rename from 7-base/platform/centos/Dockerfile rename to platform/ln/base/7/Dockerfile index 164256c..3c77d01 100644 --- a/7-base/platform/centos/Dockerfile +++ b/platform/ln/base/7/Dockerfile @@ -1,6 +1,7 @@ FROM centos:7 -RUN yum update -y && yum install -y \ +RUN set -ex; \ + yum update -y && yum install -y \ epel-release \ wget \ gcc-c++ \ @@ -10,7 +11,7 @@ RUN yum update -y && yum install -y \ m4 \ libtool \ libxslt \ - libxml \ + libxml2 \ openssh-server \ openssh-clients \ expect \ @@ -23,16 +24,22 @@ RUN yum update -y && yum install -y \ net-tools \ sudo \ psmisc \ - blas.x86_64 \ - atlas.x86_64 \ + blas \ + atlas \ which \ - numa-libs \ + numactl-libs \ + numactl-devel \ + libmemcached \ vim-minimal \ - python34 \ - python34-libs + python \ + python-libs; \ + yum install -y python34 python34-libs -RUN /usr/bin/ssh-keygen -A +#RUN package-cleanup --leaves --all #RUN mkdir -p /var/run/dbus -RUN mkdir -p /var/run/sshd -CMD ["/usr/sbin/sshd", "-D"] +RUN /usr/bin/ssh-keygen -A && mkdir -p /var/run/sshd +COPY docker-entrypoint.sh / +COPY sleep.sh / + +CMD ["/sleep.sh"] diff --git a/plugins/ce/Dockerfile b/plugins/ce/Dockerfile new file mode 100644 index 0000000..88afd9b --- /dev/null +++ b/plugins/ce/Dockerfile @@ -0,0 +1,23 @@ +FROM hpccsystems/platform: + +#RUN apt-get update && apt-get install -y iputils-ping curl xterm +#RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc + +ENV VERSION +ENV FULL_VERSION +#ENV PLATFORM_DOWNLOAD_MD5 ef58e94dcc66a45cb0fc8202e68b50d9 +ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}bionic_amd64.deb + +RUN set -x \ + && wget "http://d2wulyp08c6njk.cloudfront.net/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \ + #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \ + && dpkg -i "${PLATFORM_PACKAGE}" \ + && rm -rf "${PLATFORM_PACKAGE}" + +EXPOSE 8010 8002 8015 9876 + +RUN cp -r /home/hpcc/.ssh ~/ + +ENTRYPOINT ["/docker-entrypoint.sh"] + +CMD ["/usr/sbin/sshd", "-D"] diff --git a/sleep.sh b/sleep.sh new file mode 100755 index 0000000..7599163 --- /dev/null +++ b/sleep.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +#trap : TERM INT +sleep infinity & wait diff --git a/stack/admin/Dockerfile b/stack/admin/Dockerfile deleted file mode 100644 index 0d80684..0000000 --- a/stack/admin/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM hpccsystems/hpcc-base:7-xenial - -RUN apt-get update -RUN apt-get install -y python3.5 ansible curl iputils-ping xterm -RUN ln -s /usr/bin/python3.5 /usr/bin/python3 - - -RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc - -ENV VERSION 7.0.0 -ENV FULL_VERSION 7.0.0-rc2 -#ENV PLATFORM_DOWNLOAD_MD5 a62b43a1ef1ecf7d8d702d9033e9cf39 -ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}xenial_amd64.deb - -RUN set -x \ - #&& wget "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \ - && wget "http://10.240.32.242/builds/custom/Spark/envgen2/${PLATFORM_PACKAGE}" \ - #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \ - && dpkg -i "${PLATFORM_PACKAGE}" \ - && rm -rf "${PLATFORM_PACKAGE}" - -EXPOSE 8010 8002 8015 9876 - -RUN cp -r /home/hpcc/.ssh ~/ -COPY hpcc-tools /opt/hpcc-tools -COPY docker-entrypoint.sh / -COPY spark/* /opt/HPCCSystems/componentfiles/configxml/ -ENTRYPOINT ["/docker-entrypoint.sh"] - -CMD ["/usr/sbin/sshd", "-D"] diff --git a/test-build.sh b/test-build.sh deleted file mode 100755 index 3efa098..0000000 --- a/test-build.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env bash -# -# Run a test build for all images. - -source functions - -usage() -{ - echo "" - echo "Usage: test-build.sh -v -p -l -t " - echo " -v major version list, comma delimited" - echo " -p package list, comma delimited. The default is platform." - echo " -l linux variant, comma delimited. The default is ubuntu. The other choice is centos" - echo " -r Docker Hub repository. The default is hpccsystems/hpcc" - echo " -t tag. This is only useful for single build. " - echo " By default a tag will be signed automatically base on version, package and linux variant." - echo "" - exit 1 -} - - -function build() -{ - local version - local tag - local variant - local package - local path - version="$1"; shift - package="$1"; shift - variant="$1"; shift - tag="$1"; shift - - - if [ "$variant" = "ubuntu" ] - then - if [ "$package" = "platform" ] - then - [ "$version" != "$latestRelease" ] && tag=${version} - path="$version" - else - [ -z "$tag" ] && tag="${package}-${version}" - path="$version/$package" - fi - else - if [ "$package" = "platform" ] - then - [ -z "$tag" ] && tag=${version}-${distro_tag["$variant"]} - path="$version/$package/$variant" - else - [ -z "$tag" ] && tag=${package}-${version}-${distro_tag["$variant"]} - path="$version/$package/$variant" - fi - fi - - [ -z "$tag" ] && tag="latest" - - - echo - echo - info "Building $tag ..." - - info "Docker build -t ${repo}:$tag $path" - logfile=/tmp/docker_hpcc_${version}_${package}_${variant}.log - info "Log file: $logfile" - if ! docker build -t "${repo}:$tag" "$path" > $logfile 2>&1 - - then - fatal "Build of $tag failed!. Log file: $logfile" - else - info "Building succeeded!" - fi - - echo "" - info "Testing hpcc:$tag" - if [ "$variant" = "centos" ] - then - info "docker run --rm --privileged -e \"container=docker\" -v \"$PWD/test-${package}.sh:/usr/local/bin/test.sh\" ${repo}:${tag} test.sh" - docker run --rm --privileged -e "container=docker" -v "$PWD/test-${package}.sh:/usr/local/bin/test.sh" ${repo}:${tag} test.sh - else - info "docker run --rm -v \"$PWD/test-${package}.sh:/usr/local/bin/test.sh\" ${repo}:${tag} test.sh" - docker run --rm -v "$PWD/test-${package}.sh:/usr/local/bin/test.sh" ${repo}:${tag} test.sh - fi - echo "" - -} - -versions=("6") -packages=("platform") -variants=("ubuntu") -inTag= -latestRelease=6 -repo="hpccsystems/hpcc" - -distro_tag["centos"]="el7" - -while getopts "*l:p:r:t:v:" arg -do - case "$arg" in - l) IFS=',' read -ra variants <<< "${OPTARG}" - ;; - p) IFS=',' read -ra packages <<< "${OPTARG}" - ;; - r) repo=${OPTARG} - ;; - t) inTag=${OPTARG} - ;; - v) IFS=',' read -ra versions <<< "${OPTARG}" - ;; - ?) usage - ;; - esac -done - - - -[ ${#variants[@]} -eq 0 ] && variants=("ubuntu") -[ ${#packages[@]} -eq 0 ] && packages=("platform") - -for version in "${versions[@]}" -do - for package in "${packages[@]}" - do - for variant in "${variants[@]}" - do - build "$version" "$package" "$variant" "$inTag" - done - done -done diff --git a/test/test-build.sh b/test/test-build.sh new file mode 100755 index 0000000..26c3140 --- /dev/null +++ b/test/test-build.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# +# Run a test build for all images. + +source functions + +usage() +{ + echo "" + echo "Usage: test-build.sh -l -r -t " + echo " -l linux variant, comma delimited. The default is ubuntu. The other choice is centos" + echo " -r Docker repository. The default is hpccsystems/platform. Repo for gitlab, for example ln + plugins:" + echo " gitlab.ins.risk.regn.net/docker-images/hpccsystems/ln-platform-wp" + echo " -t build tag. The default is latest. " + echo "" + exit 1 +} + + +function test() +{ + echo "" + info "Testing $repo:$tag" + if [ "$variant" = "centos" ] + then + info "docker run --rm --privileged -e \"container=docker\" -v \"$PWD/test-${package}.sh:/usr/local/bin/test.sh\" ${repo}:${tag} test.sh" + sudo docker run --rm --privileged -e "container=docker" -v "$PWD/test-${package}.sh:/usr/local/bin/test.sh" ${repo}:${tag} test.sh + else + info "docker run --rm -v \"$PWD/test-${package}.sh:/usr/local/bin/test.sh\" ${repo}:${tag} test.sh" + sudo docker run --rm -v "$PWD/test-${package}.sh:/usr/local/bin/test.sh" ${repo}:${tag} test.sh + fi + echo "" +} + +package="platform" +variants="ubuntu" +tag=latest +repo="hpccsystems/platform" + +while getopts "*l:r:t:" arg +do + case "$arg" in + l) IFS=',' read -ra variants <<< "${OPTARG}" + ;; + r) repo=${OPTARG} + ;; + t) tag=${OPTARG} + ;; + ?) usage + ;; + esac +done + +test diff --git a/test-platform.sh b/test/test-platform.sh similarity index 100% rename from test-platform.sh rename to test/test-platform.sh diff --git a/update.sh b/update.sh deleted file mode 100755 index 317d09e..0000000 --- a/update.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -#set -e -set -x - -cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" - -versions=( "$@" ) - -# Usage -# ./update.sh "6.4.14-1 6.4.16-rc1" -if [ ${#versions[@]} -eq 0 ]; then - versions=( */ ) -fi - -versions=( "${versions[@]%/}" ) - - -travisEnv= -for full_version in "${versions[@]}"; do - - major_version=$(echo "$full_version" | cut -d '.' -f1) - version=$(echo "$full_version" | cut -d '-' -f1) - maturity=$(echo "$full_version" | cut -d '-' -f2) - dir="${major_version}" - if [ "${maturity:0:2}" = "rc" ] - then - dir="${dir}-rc" - elif [ "${maturity:0:4}" = "beta" ] - then - dir="${dir}-beta" - fi - #echo "Major version: ${major_version}" - #echo "Version: ${version}" - ( - set -x - sed -ri \ - -e 's/^(ENV VERSION) .*/\1 '"${version}"'/' \ - -e 's/^(ENV FULL_VERSION) .*/\1 '"${full_version}"'/' \ - "$dir"/{,platform/centos/}Dockerfile - - echo "$dir"/{,platform/centos/} | xargs -n 1 cp docker-entrypoint.sh - ) - -done