WIP ESP and Dali containers

Signed-off-by: rpastrana <[email protected]>
hpcc-systems · May 10, 2019 · d26365f · d26365f
1 parent dd7b18b
commit d26365f
Show file tree

Hide file tree

Showing 9 changed files with 274 additions and 0 deletions.
diff --git a/dali/Dockerfile b/dali/Dockerfile
@@ -0,0 +1,28 @@
+FROM hpccsystems/hpcc-base:7-xenial
+
+RUN apt-get update && apt-get install -y iputils-ping curl xterm
+RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc
+
+ENV VERSION 7.0.18
+ENV FULL_VERSION 7.0.18-1
+#ENV PLATFORM_DOWNLOAD_MD5  ef58e94dcc66a45cb0fc8202e68b50d9
+ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}xenial_amd64.deb
+
+RUN set -x \
+   &&  wget  "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \
+   #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \
+   && dpkg -i "${PLATFORM_PACKAGE}" \
+   && rm -rf "${PLATFORM_PACKAGE}"
+
+EXPOSE 7070
+
+#RUN cp -r /home/hpcc/.ssh ~/
+
+COPY docker-entrypoint.sh /
+ENTRYPOINT ["/docker-entrypoint.sh"]
+
+#CMD ["/usr/sbin/sshd", "-D"]
+#CMD ["/etc/init.d/hpcc-init", "-c", "mydali", "start"]
+WORKDIR "/var/lib/HPCCSystems/mydali"
+#CMD ["/opt/HPCCSystems/bin/init_dali"]
+CMD ["/opt/HPCCSystems/bin/daserver"]
diff --git a/dali/docker-entrypoint.sh b/dali/docker-entrypoint.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+set -e
+
+/usr/bin/ssh-keygen -A
+#/etc/init.d/hpcc-init start
+
+if [ "${EXEC_IN_LOOP}" = "true" ]
+then
+  interval=5
+  [ -n EXEC_INTERVAL ] && interval=${EXEC_INTERVAL}
+  while [ 1 ]
+  do
+    [ -e "$1" ] && exec "$@"
+    sleep $interval
+  done
+else
+  exec "$@"
+fi
diff --git a/dali/platform/centos/Dockerfile b/dali/platform/centos/Dockerfile
@@ -0,0 +1,25 @@
+FROM hpccsystems/hpcc-base:7-el7
+
+RUN yum install -y curl
+#RUN yum install -y java-1.8.0-openjdk
+RUN chmod u+s $(which ping)
+#RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc
+
+ENV VERSION 7.0.0
+ENV FULL_VERSION 7.0.0-1
+#ENV PLATFORM_DOWNLOAD_MD5  c435045d4319880f141e8faf23555b83
+ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}.el7.x86_64.rpm
+
+RUN set -x \
+   &&  wget  "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \
+   #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \
+   && yum install --nogpgcheck -y "${PLATFORM_PACKAGE}" \
+   && rm -rf  "${PLATFORM_PACKAGE}"
+
+EXPOSE 8010 8002 8015 9876
+
+RUN cp -r /home/hpcc/.ssh ~/
+COPY docker-entrypoint.sh /
+ENTRYPOINT ["/docker-entrypoint.sh"]
+
+CMD ["/usr/sbin/sshd",  "-D"]
diff --git a/dali/platform/centos/docker-entrypoint.sh b/dali/platform/centos/docker-entrypoint.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+set -e
+
+/usr/bin/ssh-keygen -A
+#/etc/init.d/hpcc-init start
+
+if [ "${EXEC_IN_LOOP}" = "true" ]
+then
+  interval=5
+  [ -n EXEC_INTERVAL ] && interval=${EXEC_INTERVAL}
+  while [ 1 ]
+  do
+    [ -e "$1" ] && exec "$@"
+    sleep $interval
+  done
+else
+  exec "$@"
+fi
diff --git a/esp/Dockerfile b/esp/Dockerfile
@@ -0,0 +1,28 @@
+FROM hpccsystems/hpcc-base:7-xenial
+
+RUN apt-get update && apt-get install -y iputils-ping curl xterm
+RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc
+
+ENV VERSION 7.0.18
+ENV FULL_VERSION 7.0.18-1
+#ENV PLATFORM_DOWNLOAD_MD5  ef58e94dcc66a45cb0fc8202e68b50d9
+ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}xenial_amd64.deb
+
+RUN set -x \
+   &&  wget  "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \
+   #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \
+   && dpkg -i "${PLATFORM_PACKAGE}" \
+   && rm -rf "${PLATFORM_PACKAGE}"
+
+EXPOSE 8010
+
+#RUN cp -r /home/hpcc/.ssh ~/
+
+COPY docker-entrypoint.sh /
+COPY base-esp.xml /
+ENTRYPOINT ["/docker-entrypoint.sh"]
+
+#CMD ["/usr/sbin/sshd", "-D"]
+#WORKDIR "/var/lib/HPCCSystems/myesp"
+#CMD ["/opt/HPCCSystems/bin/init_dali"]
+#CMD ["/opt/HPCCSystems/bin/esp"]
diff --git a/esp/base-esp.xml b/esp/base-esp.xml
@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Environment>
+  <Software>
+    <EspProcess build="_" componentfilesDir="/opt/HPCCSystems/componentfiles" controlPort="8010" daliServers="[DALIIP]:7070" description="ESP server" enableSEHMapping="true" formOptionsAccess="false" httpConfigAccess="true" logLevel="1" logRequests="false" logResponses="false" maxBacklogQueueSize="200" maxConcurrentThreads="0" name="myesp" perfReportDelay="60" portalurl="http://hpccsystems.com/download" txSummaryLevel="1" txSummaryResourceReq="false" computer="localhost" directory="/var/lib/HPCCSystems/myesp"><Environment><EclWatch>http://172.17.0.3:8010</EclWatch></Environment><EspProtocol name="http" type="http_protocol" plugin="esphttp" maxRequestEntityLength="8000000"/>
+      <EspService name="WSESPControl_myesp" type="WSESPControl" plugin="ws_espcontrol"/>
+      <EspBinding name="WSESPControl_Binding_myesp" service="WSESPControl_myesp" protocol="http" type="ws_espcontrolSoapBinding" plugin="ws_espcontrol" netAddress="0.0.0.0" port="8010"/>
+      <EspService name="WsSMC_EclWatch_myesp" type="WsSMC" plugin="ws_smc"><ActivityInfoCacheSeconds>10</ActivityInfoCacheSeconds></EspService>
+      <EspBinding name="WsSMC_myespsmc_myesp" service="WsSMC_EclWatch_myesp" protocol="http" type="ws_smcSoapBinding" plugin="ws_smc" netAddress="0.0.0.0" port="8010" defaultBinding="true" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="WsWorkunits_EclWatch_myesp" type="WsWorkunits" plugin="ws_workunits"><ViewTimeout>1000</ViewTimeout><ClusterQueryStateThreadPoolSize>25</ClusterQueryStateThreadPoolSize><AWUsCacheMinutes>15</AWUsCacheMinutes><AllowNewRoxieOnDemandQuery>false</AllowNewRoxieOnDemandQuery><LayoutProgram>dot/dot -Tsvg -Gordering=out</LayoutProgram><StyleSheets><xslt name="atts">/esp/xslt/atts.xslt</xslt><xslt name="dot_update">/esp/xslt/dot_update.xslt</xslt><xslt name="dot">/esp/xslt/dot.xslt</xslt><xslt name="graphStats">/esp/xslt/graphStats.xslt</xslt><xslt name="graphupdate_gvc">/esp/xslt/graphupdate_gvc.xslt</xslt></StyleSheets></EspService>
+      <EspBinding name="WsWorkunits_myespsmc_myesp" service="WsWorkunits_EclWatch_myesp" protocol="http" type="ws_workunitsSoapBinding" plugin="ws_workunits" netAddress="0.0.0.0" port="8010" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="WsTopology_EclWatch_myesp" type="WsTopology" plugin="ws_topology" warnIfCpuLoadOver="95" warnIfFreeMemoryUnder="5" warnIfFreeStorageUnder="5"><AllowNewRoxieOnDemandQuery>false</AllowNewRoxieOnDemandQuery><SystemUseRewrite>false</SystemUseRewrite></EspService>
+      <EspBinding name="WsTopology_myespsmc_myesp" service="WsTopology_EclWatch_myesp" protocol="http" type="ws_topologySoapBinding" plugin="ws_topology" netAddress="0.0.0.0" port="8010" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="WsDfu_EclWatch_myesp" type="WsDfu" plugin="ws_dfu"><NodeGroupCacheMinutes>30</NodeGroupCacheMinutes><DisableUppercaseTranslation>false</DisableUppercaseTranslation><ViewTimeout>1000</ViewTimeout><StyleSheets><xslt name="def_file">./smc_xslt/def_file.xslt</xslt></StyleSheets><Plugins path="/opt/HPCCSystems/plugins"/></EspService>
+      <EspBinding name="WsDfu_myespsmc_myesp" service="WsDfu_EclWatch_myesp" protocol="http" type="ws_dfuSoapBinding" plugin="ws_dfu" netAddress="0.0.0.0" port="8010" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="WsDfuXRef_EclWatch_myesp" type="WsDfuXRef" plugin="ws_dfu"><ViewTimeout>1000</ViewTimeout><LayoutProgram>dot/dot -Tsvg -Gordering=out</LayoutProgram><StyleSheets><xslt name="def_file">./smc_xslt/def_file.xslt</xslt></StyleSheets></EspService>
+      <EspBinding name="WsDfuXRef_myespsmc_myesp" service="WsDfuXRef_EclWatch_myesp" protocol="http" type="ws_dfuxrefSoapBinding" plugin="ws_dfu" netAddress="0.0.0.0" port="8010" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="FileSpray_Serv_EclWatch_myesp" type="FileSpray_Serv" plugin="ws_fs"><QueueLabel>dfuserver_queue</QueueLabel><MonitorQueueLabel>dfuserver_monitor_queue</MonitorQueueLabel></EspService>
+      <EspBinding name="FileSpray_Serv_myespsmc_myesp" service="FileSpray_Serv_EclWatch_myesp" protocol="http" type="FileSpray_Bind" plugin="ws_fs" netAddress="0.0.0.0" port="8010" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="WsFileIO_EclWatch_myesp" type="WsFileIO" plugin="ws_fileio"/>
+      <EspBinding name="WsFileIO_myespsmc_myesp" service="WsFileIO_EclWatch_myesp" protocol="http" type="WsFileIO" plugin="ws_fileio" netAddress="0.0.0.0" port="8010" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="WsPackageProcess_EclWatch_myesp" type="WsPackageProcess" plugin="ws_packageprocess"/>
+      <EspBinding name="WsPackageProcess_myespsmc_myesp" service="WsPackageProcess_EclWatch_myesp" protocol="http" type="WsPackageProcessSoapBinding" plugin="ws_packageprocess" netAddress="0.0.0.0" port="8010" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="ws_machine_EclWatch_myesp" type="ws_machine" plugin="ws_machine" excludePartitions="/dev*,/sys,/proc/*" monitorDaliFileServer="false"><ProcessFilters>
+     <Platform name="Windows">
+      <ProcessFilter name="any">
+       <Process name="dafilesrv"/>
+      </ProcessFilter>
+      <ProcessFilter multipleInstances="true" name="DfuServerProcess"/>
+      <ProcessFilter multipleInstances="true" name="EclCCServerProcess"/>
+      <ProcessFilter multipleInstances="true" name="EspProcess">
+       <Process name="dafilesrv" remove="true"/>
+      </ProcessFilter>
+     </Platform>
+     <Platform name="Linux">
+      <ProcessFilter name="any">
+       <Process name="dafilesrv"/>
+      </ProcessFilter>
+      <ProcessFilter multipleInstances="true" name="DfuServerProcess"/>
+      <ProcessFilter multipleInstances="true" name="EclCCServerProcess"/>
+      <ProcessFilter multipleInstances="true" name="EspProcess">
+       <Process name="dafilesrv" remove="true"/>
+      </ProcessFilter>
+      <ProcessFilter name="GenesisServerProcess">
+       <Process name="httpd"/>
+       <Process name="atftpd"/>
+       <Process name="dhcpd"/>
+      </ProcessFilter>
+     </Platform>
+    </ProcessFilters></EspService>
+      <EspBinding name="ws_machine_myespsmc_myesp" service="ws_machine_EclWatch_myesp" protocol="http" type="ws_machineSoapBinding" plugin="ws_machine" netAddress="0.0.0.0" port="8010" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="ws_account_EclWatch_myesp" type="ws_account" plugin="ws_account"/>
+      <EspBinding name="ws_account_myespsmc_myesp" service="ws_account_EclWatch_myesp" protocol="http" type="ws_accountSoapBinding" plugin="ws_account" netAddress="0.0.0.0" port="8010" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="ws_access_EclWatch_myesp" type="ws_access" plugin="ws_access"><Resources><Binding name="myespsmc" service="EclWatch" port="8010" basedn="ou=SMC,ou=EspServices,ou=ecl" workunitsBasedn="ou=workunits,ou=ecl"/><Binding name="myws_ecl" service="ws_ecl" port="8002" basedn="ou=WsEcl,ou=EspServices,ou=ecl" workunitsBasedn="ou=workunits,ou=ecl"/><Binding name="myws_sqlbinding" service="ws_sql" port="8510" basedn="ou=WsSql,ou=EspServices,ou=ecl" workunitsBasedn="ou=workunits,ou=ecl"/><Binding name="DESDLBindingTemplate" service="DESDLServiceTemplate" port="0" basedn="ou=DynamicESDL,ou=EspServices,ou=ecl" workunitsBasedn="ou=workunits,ou=ecl"/></Resources></EspService>
+      <EspBinding name="ws_access_myespsmc_myesp" service="ws_access_EclWatch_myesp" protocol="http" type="ws_accessSoapBinding" plugin="ws_access" netAddress="0.0.0.0" port="8010" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="ws_config_EclWatch_myesp" type="ws_config" plugin="ws_config"/>
+      <EspBinding name="ws_config_myespsmc_myesp" service="ws_config_EclWatch_myesp" protocol="http" type="ws_configSoapBinding" plugin="ws_config" netAddress="0.0.0.0" port="8010" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="ws_esdlconfig_EclWatch_myesp" type="ws_esdlconfig" plugin="ws_esdlconfig"/>
+      <EspBinding name="ws_esdlconfig_myespsmc_myesp" service="ws_esdlconfig_EclWatch_myesp" protocol="http" type="ws_esdlconfigSoapBinding" plugin="ws_esdlconfig" netAddress="0.0.0.0" port="8010" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="ws_elk_EclWatch_myesp" type="ws_elk" plugin="ws_elk"><ELKIntegration><Kibana/><ElasticSearch/><LogStash/></ELKIntegration></EspService>
+      <EspBinding name="ws_elk_myespsmc_myesp" service="ws_elk_EclWatch_myesp" protocol="http" type="ws_elkSoapBinding" plugin="ws_elk" netAddress="0.0.0.0" port="8010" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="ws_ecl_ws_ecl_myesp" type="ws_ecl" plugin="ws_ecl">
+        <RoxieTimeout>300</RoxieTimeout>
+        <WorkunitTimeout>600</WorkunitTimeout>
+        <VIPS/>
+        <Targets/>
+      </EspService>
+      <EspBinding name="ws_ecl_myws_ecl_myesp" service="ws_ecl_ws_ecl_myesp" protocol="http" type="ws_eclSoapBinding" plugin="ws_ecl" netAddress="0.0.0.0" port="8002" defaultBinding="true" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="ws_sql_ws_sql_myesp" type="ws_sql" plugin="ws_sql"/>
+      <EspBinding name="ws_sql_myws_sqlbinding_myesp" service="ws_sql_ws_sql_myesp" protocol="http" type="ws_sqlSoapBinding" plugin="ws_sql" netAddress="0.0.0.0" port="8510" defaultBinding="true" wsdlServiceAddress="" defaultServiceVersion=""/>
+      <EspService name="DESDLServiceTemplate" type="DESDLServiceTemplate" plugin="esdl_svc_engine" namespaceBase="urn:hpccsystems:ws"/>
+      <EspBinding name="DESDLBindingTemplate" service="DESDLServiceTemplate" protocol="http" type="EsdlBinding" plugin="esdl_svc_engine" netAddress="0.0.0.0" port="0"/>
+
+
+
+   <Authentication checkViewPermissions="false" htpasswdFile="/etc/HPCCSystems/.htpasswd" ldapAuthMethod="kerberos" ldapConnections="10" ldapServer="" method="none" passwordExpirationWarningDays="10"/>
+
+
+
+
+  </EspProcess>
+    <Directories name="HPCCSystems">
+   <Category dir="/var/log/[NAME]/[INST]" name="log"/>
+   <Category dir="/var/lib/[NAME]/[INST]" name="run"/>
+   <Category dir="/etc/[NAME]/[INST]" name="conf"/>
+   <Category dir="/var/lib/[NAME]/[INST]/temp" name="temp"/>
+   <Category dir="/var/lib/[NAME]/hpcc-data/[COMPONENT]" name="data"/>
+   <Category dir="/var/lib/[NAME]/hpcc-data2/[COMPONENT]" name="data2"/>
+   <Category dir="/var/lib/[NAME]/hpcc-data3/[COMPONENT]" name="data3"/>
+   <Category dir="/var/lib/[NAME]/hpcc-mirror/[COMPONENT]" name="mirror"/>
+   <Category dir="/var/lib/[NAME]/queries/[INST]" name="query"/>
+   <Category dir="/var/lock/[NAME]/[INST]" name="lock"/>
+   <Category dir="/var/lib/[NAME]/hpcc-data4/[COMPONENT]" name="data4"/>
+   <Category dir="/var/lib/[NAME]/keys" name="keys"/>
+  </Directories>
+  </Software>
+</Environment>
diff --git a/esp/docker-entrypoint.sh b/esp/docker-entrypoint.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+set -e
+
+/usr/bin/ssh-keygen -A
+#/etc/init.d/hpcc-init start
+mkdir -p /var/lib/HPCCSystems/myesp
+mkdir -p /var/log/HPCCSystems/myesp
+cat /base-esp.xml | sed "s/\[DALIIP\]/$1/" > /var/lib/HPCCSystems/myesp/esp.xml
+chown -R hpcc:hpcc /var/lib/HPCCSystems
+chown -R hpcc:hpcc /var/log/HPCCSystems
+
+#cat /var/lib/HPCCSystems/myesp/esp.xml
+cd /var/lib/HPCCSystems/myesp
+/opt/HPCCSystems/bin/esp 
+
+
+
+
diff --git a/esp/platform/centos/Dockerfile b/esp/platform/centos/Dockerfile
@@ -0,0 +1,25 @@
+FROM hpccsystems/hpcc-base:7-el7
+
+RUN yum install -y curl
+#RUN yum install -y java-1.8.0-openjdk
+RUN chmod u+s $(which ping)
+#RUN groupadd hpcc && useradd -s /bin/bash -r -m -d /home/hpcc -g hpcc -c "hpcc Runtime User" hpcc
+
+ENV VERSION 7.0.0
+ENV FULL_VERSION 7.0.0-1
+#ENV PLATFORM_DOWNLOAD_MD5  c435045d4319880f141e8faf23555b83
+ENV PLATFORM_PACKAGE hpccsystems-platform-community_${FULL_VERSION}.el7.x86_64.rpm
+
+RUN set -x \
+   &&  wget  "http://wpc.423a.rhocdn.net/00423A/releases/CE-Candidate-${VERSION}/bin/platform/${PLATFORM_PACKAGE}" \
+   #&& echo "$PLATFORM_DOWNLOAD_MD5 ${PLATFORM_PACKAGE}" | md5sum -c - \
+   && yum install --nogpgcheck -y "${PLATFORM_PACKAGE}" \
+   && rm -rf  "${PLATFORM_PACKAGE}"
+
+EXPOSE 8010 8002 8015 9876
+
+RUN cp -r /home/hpcc/.ssh ~/
+COPY docker-entrypoint.sh /
+ENTRYPOINT ["/docker-entrypoint.sh"]
+
+CMD ["/usr/sbin/sshd",  "-D"]
diff --git a/esp/platform/centos/docker-entrypoint.sh b/esp/platform/centos/docker-entrypoint.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+set -e
+
+/usr/bin/ssh-keygen -A
+#/etc/init.d/hpcc-init start
+
+if [ "${EXEC_IN_LOOP}" = "true" ]
+then
+  interval=5
+  [ -n EXEC_INTERVAL ] && interval=${EXEC_INTERVAL}
+  while [ 1 ]
+  do
+    [ -e "$1" ] && exec "$@"
+    sleep $interval
+  done
+else
+  exec "$@"
+fi