From 806873d75b130d8f8790d0ee794fe447bfb5456d Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 14 Nov 2024 21:06:50 -0500 Subject: [PATCH] Support custom start-command/start-options on GHA container runner and openssl1.1.1 on AL2 docker images (#5187) Signed-off-by: Peter Zhu --- .github/workflows/get-ci-image-tag.yml | 26 +++++++++++++++-- .../build.al2.opensearch.x64.arm64.dockerfile | 29 +++++++++++++++++-- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/.github/workflows/get-ci-image-tag.yml b/.github/workflows/get-ci-image-tag.yml index ea5561905c..b1e7e41c15 100644 --- a/.github/workflows/get-ci-image-tag.yml +++ b/.github/workflows/get-ci-image-tag.yml @@ -25,25 +25,33 @@ on: ci-image-version-linux: description: The ci image version for linux build value: ${{ jobs.Get-CI-Image-Tag.outputs.output-ci-image-version-linux }} + ci-image-start-options: + description: The ci image start options to set when starting the container + value: ${{ jobs.Get-CI-Image-Tag.outputs.output-ci-image-start-options }} + ci-image-start-command: + description: The ci image start commands to run after initialization + value: ${{ jobs.Get-CI-Image-Tag.outputs.output-ci-image-start-command }} jobs: Get-CI-Image-Tag: runs-on: ubuntu-latest outputs: - output-ci-image-version-linux: ${{ steps.step-ci-image-version-linux.outputs.ci-image-version-linux }} + output-ci-image-version-linux: ${{ steps.step-ci-image-setups.outputs.ci-image-version-linux }} + output-ci-image-start-command: ${{ steps.step-ci-image-setups.outputs.ci-image-start-command }} + output-ci-image-start-options: ${{ steps.step-ci-image-setups.outputs.ci-image-start-options }} steps: - name: Install crane uses: iarekylew00t/crane-installer@v1 with: crane-release: v0.15.2 - name: Checkout opensearch-build repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: 'opensearch-project/opensearch-build' ref: ${{ inputs.build_ref }} path: 'opensearch-build' - name: Get ci image version from opensearch-build repository scripts - id: step-ci-image-version-linux + id: step-ci-image-setups run: | PRODUCT=${{ inputs.product }} PLATFORM=${{ inputs.platform }} @@ -55,6 +63,18 @@ jobs: PLATFORM="almalinux8" fi fi + + if [[ "$PLATFORM" = "al2" ]]; then + CI_IMAGE_CMD="cp -a /node_al2/* /node && /node/bin/node -v" + echo "ci-image-start-command=$CI_IMAGE_CMD" >> $GITHUB_OUTPUT + CI_IMAGE_OPTIONS="--user root -v /node:/node:rw,rshared -v /node:/__e/node20:ro,rshared" + echo "ci-image-start-options=$CI_IMAGE_OPTIONS" >> $GITHUB_OUTPUT + else + CI_IMAGE_CMD="echo pass" + echo "ci-image-start-command=$CI_IMAGE_CMD" >> $GITHUB_OUTPUT + CI_IMAGE_OPTIONS="--user root" + echo "ci-image-start-options=$CI_IMAGE_OPTIONS" >> $GITHUB_OUTPUT + fi crane version echo $PRODUCT $PLATFORM CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p $PLATFORM -u $PRODUCT -t $TYPE | head -1` diff --git a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile index c9dd94704c..4d1f2136f4 100644 --- a/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile @@ -76,11 +76,21 @@ ENV GEM_HOME=$CONTAINER_USER_HOME/.gem ENV GEM_PATH=$GEM_HOME ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH +# Installing openssl1.1.1 +# Support requests >= 2.28.1 version +ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/lib +RUN yum install -y curl libcurl-devel libfaketime perl-core pcre-devel && yum remove -y openssl-devel && yum clean all && \ + mkdir -p /tmp/openssl && cd /tmp/openssl && \ + curl -sSL -o- https://www.openssl.org/source/openssl-1.1.1g.tar.gz | tar -xz --strip-components 1 && \ + ./config --prefix=/usr --openssldir=/etc/ssl --libdir=lib shared zlib-dynamic && make && make install && \ + echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/lib" > /etc/profile.d/openssl.sh && openssl version + # Install Python binary RUN curl https://www.python.org/ftp/python/3.9.7/Python-3.9.7.tgz | tar xzvf - && \ cd Python-3.9.7 && \ - ./configure --enable-optimizations && \ - make altinstall + env LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/lib ./configure --enable-optimizations --with-openssl=/usr --prefix=/usr/local && \ + make altinstall && cd ../ && rm -rf Python-3.9.7.tgz Python-3.9.7 && \ + cp -v /etc/ssl/certs/ca-bundle.crt /etc/ssl/cert.pem # Setup Python links RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \ @@ -114,6 +124,21 @@ RUN git clone -b v0.3.27 --single-branch https://github.com/xianyi/OpenBLAS.git ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" RUN pip3 install cmake==3.26.4 +# NodeJS Unofficial Builds +# https://github.com/opensearch-project/opensearch-build/issues/5178 +# https://github.com/actions/runner/issues/2906 +# https://github.com/actions/runner/issues/3475 +# GitHub enforce nodejs 20 official build in runner 2.317.0 of their actions and CentOS7/AL2 would fail due to having older glibc versions +# Until https://github.com/actions/runner/pull/3128 is merged or AL2 is deprecated (2025/06) this is a quick fix with unofficial builds support glibc 2.17 +# With changes done similar to this PR (https://github.com/opensearch-project/job-scheduler/pull/702) alongside the image here +# Only linux x64 is supported in unofficial build until https://github.com/nodejs/unofficial-builds/pull/91 is merged +RUN if [ `uname -m` = "x86_64" ]; then \ + curl -SL https://unofficial-builds.nodejs.org/download/release/v20.10.0/node-v20.10.0-linux-x64-glibc-217.tar.xz -o /node20.tar.xz; \ + mkdir /node_al2; \ + tar -xf /node20.tar.xz --strip-components 1 -C /node_al2; \ + rm -v /node20.tar.xz; \ + fi + # Change User USER $CONTAINER_USER WORKDIR $CONTAINER_USER_HOME