From 9c7e2c879a35af3e5abe0a7dc751a3490b8ef291 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 8 May 2024 11:56:34 +0200 Subject: [PATCH 01/22] change(common): preliminary update of min node version This updates the minimum required node version to Node 20. The latest version of Node 18 still contains the npm bug (https://github.com/npm/cli/issues/7072) whereas Node 20 got updated to a npm version that contains a fix. Node 20 is known to work with our current code, so this change updates to that as an intermediate step before we investigate if we can update to Node 22 as discussed for Keyman 18. --- resources/build/minimum-versions.inc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/build/minimum-versions.inc.sh b/resources/build/minimum-versions.inc.sh index 17757d55848..d7aff76b94b 100644 --- a/resources/build/minimum-versions.inc.sh +++ b/resources/build/minimum-versions.inc.sh @@ -13,7 +13,7 @@ KEYMAN_MIN_TARGET_VERSION_UBUNTU=20.04 # Ubuntu 20.04 Focal KEYMAN_MIN_TARGET_VERSION_CHROME=95.0 # Final version that runs on Android 5.0 # Dependency versions -KEYMAN_MIN_VERSION_NODE_MAJOR=18 +KEYMAN_MIN_VERSION_NODE_MAJOR=20 # Latest Node 20 doesn't have the buggy npm (npm#7072) KEYMAN_MIN_VERSION_NPM=10.5.1 # 10.5.0 has bug, discussed in #10350 KEYMAN_MIN_VERSION_EMSCRIPTEN=3.1.44 # Warning: 3.1.45 is bad (#9529); newer versions work KEYMAN_MAX_VERSION_EMSCRIPTEN=3.1.58 # See #9529 From 2dffec366ce9c4fa24c094b05e0000f9099000bc Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 6 May 2024 15:07:37 +0200 Subject: [PATCH 02/22] docs(common): improve formatting of `builder.md` --- resources/build/builder.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/resources/build/builder.md b/resources/build/builder.md index 3855288392c..0d0c385c509 100644 --- a/resources/build/builder.md +++ b/resources/build/builder.md @@ -86,13 +86,12 @@ This somewhat unwieldy incantation handles all our build environments. The intent is to get a good solid consistent path for the script so that we can safely include the build script, no matter what `pwd` is when the script is run. - The only modification permissible in this block is the `` text which will be a series of `../` paths taking us to the repository root from the location of the script itself. It is essential to make the include relative to the repo root, even for scripts -under the resources/ folder. Doing this gives us significant performance +under the `resources/` folder. Doing this gives us significant performance benefits. Inclusion of other scripts should be kept outside this standard build script @@ -1030,4 +1029,4 @@ Note: it is recommended that you use `$(builder_term text)` instead of [`builder_echo`]: #builderecho-function [`builder_die`]: #builderdie-function [`builder_echo_debug`]: #builderechodebug-function -[`builder_is_debug_build`]: #builderisdebugbuild-function \ No newline at end of file +[`builder_is_debug_build`]: #builderisdebugbuild-function From d554415b1c7ddf7704bad9d6b95063014c06a124 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 6 May 2024 15:08:30 +0200 Subject: [PATCH 03/22] change(common): add readme and `build.sh` --- docs/build/linux-ubuntu.md | 90 ++++-------------- linux/Dockerfile | 90 ------------------ resources/docker-images/README.md | 148 ++++++++++++++++++++++++++++++ resources/docker-images/build.sh | 109 ++++++++++++++++++++++ 4 files changed, 273 insertions(+), 164 deletions(-) delete mode 100644 linux/Dockerfile create mode 100644 resources/docker-images/README.md create mode 100755 resources/docker-images/build.sh diff --git a/docs/build/linux-ubuntu.md b/docs/build/linux-ubuntu.md index 00aaeef0b30..5879e433371 100644 --- a/docs/build/linux-ubuntu.md +++ b/docs/build/linux-ubuntu.md @@ -205,83 +205,25 @@ Android projects. `JAVA_HOME_11` is mostly used by CI. ## Docker Builder The Docker builder allows you to perform a build from anywhere Docker is supported. - -To build the docker image: - -```shell -cd linux -docker pull ubuntu:latest # (to make sure you have an up-to-date image) -docker build . -t keymanapp/keyman-linux-builder:latest -``` - -Once the image is built, it may be used to build parts of Keyman. - -**Note** that it's not yet possible to run tests in the Docker container. - -- core - - ```shell - # build 'Keyman Core' in docker - # keep linux build artifacts separate - mkdir -p $(git rev-parse --show-toplevel)/core/build/linux - docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ - -v $(git rev-parse --show-toplevel)/core/build/linux:/home/build/build/core/build \ - keymanapp/keyman-linux-builder:latest \ - core/build.sh --debug - ``` - -- linux - - ```shell - # build 'Keyman for Linux' installation in docker - docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ - --entrypoint /bin/bash keymanapp/keyman-linux-builder:latest \ - -c 'DESTDIR=/home/build /usr/bin/bashwrapper linux/build.sh --debug build install' - ``` - -- Keyman Web - - ```shell - # build 'Keyman Web' in docker - docker run --privileged -it --rm \ - -v $(git rev-parse --show-toplevel):/home/build/build \ - keymanapp/keyman-linux-builder:latest \ - web/build.sh --debug - ``` - -- Keyman for Android - - ```shell - # build 'Keyman for Android' in docker - docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ - keymanapp/keyman-linux-builder:latest \ - android/build.sh --debug - ``` - -### Customizing the builder - -You can use Docker [build args](https://docs.docker.com/build/guide/build-args/) to customize the image build. As an example, the following will build an image explicitly with Ubuntu 23.04 and Node.js 20. Check the [Dockerfile](../../linux/Dockerfile) for `ARG` entries. - -```shell -cd linux -docker pull ubuntu:23.04 # (to make sure you have an up-to-date image) -docker build . -t keymanapp/keyman-linux-builder:u23.04-node20 --build-arg OS_VERSION=23.04 --build-arg NODE_MAJOR=20 -```` +See [this README.md](../../resources/docker-images/README.md) for details. ### Using the builder with VSCode [Dev Containers](https://code.visualstudio.com/docs/devcontainers/tutorial) -1. Save the following as `.devcontainer/devcontainer.json`, updating the `image` to match the Docker image built above. +1. Save the following as `.devcontainer/devcontainer.json`, updating the `image` + to match the Docker image built above. -```json -// file: .devcontainer/devcontainer.json -{ - "name": "Keyman Ubuntu 23.04", - "image": "keymanapp/keyman-linux-builder:u23.04-node18" -} -// For format details, see https://aka.ms/devcontainer.json. For config options, see the -// README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu -``` + ```json + // file: .devcontainer/devcontainer.json + { + "name": "Keyman Ubuntu 23.04", + "image": "keymanapp/keyman-linux-builder:u23.04-node18" + } + // For format details, see https://aka.ms/devcontainer.json. For config options, + // see the README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu + ``` -2. in VSCode, use the "Dev Containers: Open Folder In Container…" option and choose the Keyman directory. +2. in VSCode, use the "Dev Containers: Open Folder In Container…" option and + choose the Keyman directory. -3. You will be given a window which is running VSCode inside this builder image, regardless of your host OS. +3. You will be given a window which is running VSCode inside this builder + image, regardless of your host OS. diff --git a/linux/Dockerfile b/linux/Dockerfile deleted file mode 100644 index ea07ec699ad..00000000000 --- a/linux/Dockerfile +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (c) 2022-2023 SIL International. All rights reserved. -# -# builder image for a linux build -# see ../docs/build/linux-ubuntu.md - -ARG OS_VERSION=latest -ARG OS_PLATFORM=amd64 - -FROM --platform=${OS_PLATFORM} ubuntu:${OS_VERSION} -LABEL org.opencontainers.image.authors="SIL International." -LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" -LABEL org.opencontainers.image.title="Keyman Linux Build Image" - -# We will switch to a build user after some installation -USER root -ENV HOME /home/build -RUN useradd -c "Build user" --home-dir $HOME --create-home --shell /usr/bin/bashwrapper build -VOLUME /home/build/build -WORKDIR /home/build/build -ENV DEBIAN_FRONTEND noninteractive -ENV DEBIAN_PRIORITY critical -ENV DEBCONF_NOWARNINGS yes - -# Update to the latest -RUN apt-get -q -y update && \ - apt-get -q -y install devscripts equivs meson python3 python3-setuptools software-properties-common curl && \ - add-apt-repository ppa:keymanapp/keyman && \ - add-apt-repository ppa:keymanapp/keyman-alpha -RUN apt-get -q -y update && \ - apt-get -q -y upgrade - -# Install dependencies -ADD debian/control /tmp/control -# Answer 'yes' to install questions -RUN (yes | mk-build-deps --install /tmp/control) || true && \ - rm /tmp/control - -# Install Node -ARG NODE_MAJOR=18 -RUN apt-get install -q -y ca-certificates curl gnupg && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_"${NODE_MAJOR}".x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && apt-get update && apt-get install nodejs -y - -ARG EMSCRIPTEN_VERSION=3.1.44 -# Install emscripten -RUN cd /usr/share && \ - git clone https://github.com/emscripten-core/emsdk.git && \ - cd emsdk && \ - ./emsdk install ${EMSCRIPTEN_VERSION} && \ - ./emsdk activate ${EMSCRIPTEN_VERSION} && \ - echo "#!/bin/bash" > /usr/bin/bashwrapper && \ - echo "export EMSCRIPTEN_BASE=/usr/share/emsdk/upstream/emscripten" >> /usr/bin/bashwrapper - -# Keyman Web -RUN curl --output google-chrome-stable_current_amd64.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ - apt-get -q -y install ./google-chrome-stable_current_amd64.deb && \ - rm google-chrome-stable_current_amd64.deb && \ - echo "export CHROME_BIN=/opt/google/chrome/chrome" >> /usr/bin/bashwrapper - -# Keyman for Android -RUN apt-get -q -y install gradle maven pandoc sdkmanager jq && \ - sdkmanager platform-tools && \ - yes | sdkmanager --licenses && \ - chown -R build:build /opt/android-sdk/ && \ - echo "export ANDROID_HOME=/opt/android-sdk" >> /usr/bin/bashwrapper && \ - echo "export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64" >> /usr/bin/bashwrapper - -# Finish bashwrapper script and adjust permissions -RUN echo "\${@:-bash}" >> /usr/bin/bashwrapper && \ - chmod +x /usr/bin/bashwrapper && \ - chown -R build:build $HOME - -# now, switch to build user -USER build - -# Pre-install gradle. This will put files in ~/.gradle which will speed up builds. -RUN mkdir -p $HOME/tmp/gradle/wrapper && \ - # KMEA uses gradle-7.5.1-bin - curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradle/wrapper/gradle-wrapper.jar && \ - curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.properties https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradle/wrapper/gradle-wrapper.properties && \ - curl --location --output $HOME/tmp/gradlew https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradlew && \ - chmod +x $HOME/tmp/gradlew && \ - $HOME/tmp/gradlew --quiet && \ - # Some projects use gradle-7.5.1-all, so we pre-install that as well - curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.jar && \ - curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.properties https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.properties && \ - curl --location --output $HOME/tmp/gradlew https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradlew && \ - chmod +x $HOME/tmp/gradlew && \ - $HOME/tmp/gradlew --quiet && \ - rm -rf $HOME/tmp - -ENTRYPOINT [ "/usr/bin/bashwrapper" ] diff --git a/resources/docker-images/README.md b/resources/docker-images/README.md new file mode 100644 index 00000000000..5a241af91f3 --- /dev/null +++ b/resources/docker-images/README.md @@ -0,0 +1,148 @@ +# Container + +Docker containers that can be used to build Keyman on the respective +platforms. They contain everything that a CI build agent needs to +build for the platform. + +## Prerequisites + +You'll need Docker Buildx installed to successfully be able to build the +container images. This is easiest achieved by installing the [official +Docker version](https://docs.docker.com/engine/install/ubuntu/). + +Currently it is not possible to use Podman instead of Docker due to a number +of bugs and incompatibilities in the Podman implementation. + +## Building the images + +To build the docker images: + +```shell +resources/docker-images/build.sh +``` + +By default this will create 64-bit images for building +Keyman for Android, Keyman for Linux and Keyman for Web. These images +are based on the Ubuntu 24.04 with Node 20 and Emscripten +3.1.44 (for the exact versions, see [`minimum-versions.inc.sh`](../build/minimum-versions.inc.sh)) +and are named e.g. `keyman-core-ci:default`. + +The versions can be changed, e.g. + +```shell +resources/docker-images/build.sh --ubuntu-version jammy --node 20 +``` + +This will create an image named e.g. `keyman-core-ci:jammy-node20`. + +Once the image is built, it may be used to build parts of Keyman. + +## Building locally + +It is possible to build locally with these images: + +- Keyman Core + + ```shell + # build 'Keyman Core' in docker + # keep build artifacts separate + mkdir -p $(git rev-parse --show-toplevel)/core/build/docker-core + docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ + -v $(git rev-parse --show-toplevel)/core/build/docker-core:/home/build/build/core/build \ + keymanapp/keyman-core-ci:default \ + core/build.sh --debug build + ``` + + Note: Since the generated binaries are platform dependent we put them in a container + specific directory. + +- Keyman for Linux + + ```shell + # build 'Keyman for Linux' installation in docker + # keep build artifacts separate + mkdir -p $(git rev-parse --show-toplevel)/linux/build/docker-linux + docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ + -v $(git rev-parse --show-toplevel)/linux/build/docker-linux:/home/build/build/linux/build \ + -e DESTDIR=/tmp \ + keymanapp/keyman-linux-ci:default \ + linux/build.sh --debug build install + ``` + + Note: Since the generated binaries are platform dependent we put them in a container + specific directory. + +- Keyman Web + + ```shell + # build 'Keyman Web' in docker + docker run --privileged -it --rm \ + -v $(git rev-parse --show-toplevel):/home/build/build \ + keymanapp/keyman-web-ci:default \ + web/build.sh --debug + ``` + +- Keyman for Android + + ```shell + # build 'Keyman for Android' in docker + docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ + keymanapp/keyman-android-ci:default \ + android/build.sh --debug + ``` + +## Running tests locally + +- Keyman Core + + ```shell + # build 'Keyman Core' in docker + # keep build artifacts separate + mkdir -p $(git rev-parse --show-toplevel)/core/build/docker-core + docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ + -v $(git rev-parse --show-toplevel)/core/build/docker-core:/home/build/build/core/build \ + keymanapp/keyman-core-ci:default \ + core/build.sh --debug test + ``` + + Note: Since the generated binaries are platform dependent we put them in a container + specific directory. + +- Keyman for Linux + + ```shell + # build 'Keyman for Linux' installation in docker + # keep build artifacts separate + mkdir -p $(git rev-parse --show-toplevel)/linux/build/docker-linux + docker run --privileged -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ + -v $(git rev-parse --show-toplevel)/linux/build/docker-linux:/home/build/build/linux/build \ + -e DESTDIR=/tmp \ + keymanapp/keyman-linux-ci:default \ + linux/build.sh --debug test + ``` + + Note: this requires the `--privileged` parameter in order for all tests to pass! + + Note: Since the generated binaries are platform dependent we put them in a container + specific directory. + +- Keyman Web + + ```shell + # build 'Keyman Web' in docker + docker run --privileged -it --rm \ + -v $(git rev-parse --show-toplevel):/home/build/build \ + keymanapp/keyman-web-ci:default \ + web/build.sh --debug test + ``` + + Note: this requires the `--privileged` parameter in order for all tests to pass! + +- Keyman for Android + + ```shell + # build 'Keyman for Android' in docker + docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ + keymanapp/keyman-android-ci:default \ + android/build.sh --debug test + ``` diff --git a/resources/docker-images/build.sh b/resources/docker-images/build.sh new file mode 100755 index 00000000000..294d5f463aa --- /dev/null +++ b/resources/docker-images/build.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +################################ Main script ################################ + +. "${KEYMAN_ROOT}/resources/build/minimum-versions.inc.sh" + +builder_describe \ + "Build docker images" \ + ":android" \ + ":base" \ + ":core" \ + ":linux" \ + ":web" \ + "--ubuntu-version=UBUNTU_VERSION The Ubuntu version (default: ${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER})" \ + "--node=NODE_MAJOR Node version (default: ${KEYMAN_MIN_VERSION_NODE_MAJOR})" \ + "--emscripten=EMSCRIPTEN_VERSION Emscripten version (default: ${KEYMAN_MIN_VERSION_EMSCRIPTEN})" \ + "--no-cache Force rebuild of docker images" \ + "build" + +builder_parse "$@" + +_add_build_args() { + local var=$1 + local default_var=$2 + local name=$3 + local value + + if [[ -n "${!var:-}" ]]; then + value="${!var}" + else + value="${!default_var:-}" + fi + + build_args+=(--build-arg="${var}=${value}") + + if [[ -n "${build_version:-}" ]]; then + build_version="${build_version}-${name:-}${value}" + else + build_version="${name}${value}" + fi +} + +_convert_parameters_to_build_args() { + build_args=() + build_version= + + _add_build_args UBUNTU_VERSION KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER "" + _add_build_args JAVA_VERSION KEYMAN_VERSION_JAVA java + _add_build_args NODE_MAJOR KEYMAN_MIN_VERSION_NODE_MAJOR node + _add_build_args EMSCRIPTEN_VERSION KEYMAN_MIN_VERSION_EMSCRIPTEN emscr + + if [[ -n "${BASE_VERSION:-}" ]]; then + build_args+=(--build-arg="BASE_VERSION=${BASE_VERSION}") + fi +} + +_is_default_values() { + [[ -z "${UBUNTU_VERSION:-}" ]] && [[ -z "${JAVA_VERSION:-}" ]] && \ + [[ -z "${NODE_MAJOR:-}" ]] && [[ -z "${EMSCRIPTEN_VERSION:-}" ]] +} + +build_action() { + local platform=$1 + + builder_echo debug "Building image for ${platform}" + + _convert_parameters_to_build_args + + if [[ "${platform}" == "base" ]]; then + docker pull --platform "amd64" "ubuntu:${UBUNTU_VERSION:-${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER}}" + ### TMP code + elif [[ "${platform}" == "core" ]]; then + cp "${KEYMAN_ROOT}/linux/debian/control" "${platform}" + ### TMP END + elif [[ "${platform}" == "linux" ]]; then + cp "${KEYMAN_ROOT}/linux/debian/control" "${platform}" + fi + + if builder_has_option --no-cache; then + OPTION_NO_CACHE="--no-cache" + fi + + cd "${platform}" || true + # shellcheck disable=SC2248 + docker build ${OPTION_NO_CACHE:-} -t "keymanapp/keyman-${platform}-ci:${build_version}" "${build_args[@]}" . + # If the user didn't specify particular versions we will additionaly create an image + # with the tag 'default'. + if _is_default_values; then + builder_echo debug "Setting default tag for ${platform}" + docker build . -t "keymanapp/keyman-${platform}-ci:default" "${build_args[@]}" + fi + cd - || true + builder_echo success "Docker image 'keymanapp/keyman-${platform}-ci:${build_version}' built" +} + +if builder_has_action build; then + build_action base + BASE_VERSION="${build_version}" + builder_run_action build:android build_action android + builder_run_action build:core build_action core + builder_run_action build:linux build_action linux + builder_run_action build:web build_action web +fi From 5c337bec8e2427bedbf16d5b822c9d18f9402b54 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Fri, 3 May 2024 17:47:48 +0200 Subject: [PATCH 04/22] feat(common): add base CI image --- resources/docker-images/base/Dockerfile | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 resources/docker-images/base/Dockerfile diff --git a/resources/docker-images/base/Dockerfile b/resources/docker-images/base/Dockerfile new file mode 100644 index 00000000000..116d4b80846 --- /dev/null +++ b/resources/docker-images/base/Dockerfile @@ -0,0 +1,30 @@ +# Copyright (c) 2024 SIL International. All rights reserved. + +ARG UBUNTU_VERSION=latest +FROM --platform=amd64 ubuntu:${UBUNTU_VERSION} + +LABEL org.opencontainers.image.authors="SIL International." +LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" +LABEL org.opencontainers.image.title="Keyman Build Base Image" + +# We will switch to a build user after some installation +USER root +ENV HOME=/home/build +RUN grep ubuntu /etc/passwd && userdel ubuntu || true && \ + rm -rf /home/ubuntu && \ + useradd -c "Build user" --uid 1000 --home-dir $HOME --create-home --shell /usr/bin/bashwrapper build + +ENV DEBIAN_FRONTEND=noninteractive +ENV DEBIAN_PRIORITY=critical +ENV DEBCONF_NOWARNINGS=yes + +# Update to the latest +RUN apt-get -q -y update && \ + apt-get -q -y install ca-certificates curl gnupg meson software-properties-common sudo && \ + add-apt-repository ppa:keymanapp/keyman && \ + add-apt-repository ppa:keymanapp/keyman-alpha +RUN apt-get -q -y update && \ + apt-get -q -y upgrade + +# Allow build user to use `sudo` +RUN echo "build ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers From 72bc6c7b8bb9ce0acd55a8303d0cc4449068ecf6 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Thu, 25 Apr 2024 18:04:46 +0200 Subject: [PATCH 05/22] feat(linux): add CI image for building Keyman for Linux This change allows to build a docker image that can build Keyman for Linux with test coverage reports, and installs the necessary dependencies for running integration tests. --- resources/docker-images/linux/.gitignore | 1 + resources/docker-images/linux/Dockerfile | 56 ++++++++++++++++++++++ resources/docker-images/linux/run-tests.sh | 19 ++++++++ 3 files changed, 76 insertions(+) create mode 100644 resources/docker-images/linux/.gitignore create mode 100644 resources/docker-images/linux/Dockerfile create mode 100755 resources/docker-images/linux/run-tests.sh diff --git a/resources/docker-images/linux/.gitignore b/resources/docker-images/linux/.gitignore new file mode 100644 index 00000000000..4db28ac4954 --- /dev/null +++ b/resources/docker-images/linux/.gitignore @@ -0,0 +1 @@ +control diff --git a/resources/docker-images/linux/Dockerfile b/resources/docker-images/linux/Dockerfile new file mode 100644 index 00000000000..6766aa08591 --- /dev/null +++ b/resources/docker-images/linux/Dockerfile @@ -0,0 +1,56 @@ +# Copyright (c) 2024 SIL International. All rights reserved. + +ARG BASE_VERSION +FROM --platform=amd64 keymanapp/keyman-base-ci:${BASE_VERSION} +LABEL org.opencontainers.image.authors="SIL International." +LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" +LABEL org.opencontainers.image.title="Keyman Linux Build Image" + +# Install dependencies +ADD control /tmp/control +# Answer 'yes' to install questions +RUN apt-get install -qy python3 python3-setuptools python3-coverage \ + devscripts equivs libdatetime-perl lcov gcovr xvfb \ + xserver-xephyr metacity mutter dbus-x11 weston xwayland && \ + (yes | mk-build-deps --install /tmp/control) || true && \ + rm /tmp/control + +# Install lcov for code coverage +# Update to the latest and install packages needed for Linux coverage reporting +# and integration tests. We need at least version 2.0 of lcov. However, +# version 2.0-4 from Noble doesn't work either on Jammy. So we use +# version 2.0-1 from Mantic. +RUN LCOV_VERSION=$(dpkg -s lcov | grep Version | cut -d' ' -f2) && \ + if dpkg --compare-versions "${LCOV_VERSION}" lt 2.0; then \ + curl -sS -o /tmp/lcov.deb --location http://mirrors.kernel.org/ubuntu/pool/universe/l/lcov/lcov_2.0-1_all.deb && \ + apt-get -qy install /tmp/lcov.deb && \ + rm /tmp/lcov.deb ; \ + fi + +RUN mkdir -p /var/run/1000 && \ + chown build:build /var/run/1000 && \ + echo "#!/bin/bash" > /usr/bin/bashwrapper && \ + echo "export XDG_RUNTIME_DIR=/var/run/1000" >> /usr/bin/bashwrapper + +COPY run-tests.sh /usr/bin/run-tests.sh + +# Finish bashwrapper script and adjust permissions +RUN <> /usr/bin/bashwrapper + +if [[ "\$@" =~ test ]] && [ -f /usr/bin/run-tests.sh ]; then + /usr/bin/run-tests.sh "\${@:-bash}" +else + "\${@:-bash}" +fi +EOF + +RUN chmod +x /usr/bin/bashwrapper && \ + chown -R build:build $HOME + +# now, switch to build user +USER build + +VOLUME /home/build/build +WORKDIR /home/build/build + +ENTRYPOINT [ "/usr/bin/bashwrapper" ] diff --git a/resources/docker-images/linux/run-tests.sh b/resources/docker-images/linux/run-tests.sh new file mode 100755 index 00000000000..962418467ef --- /dev/null +++ b/resources/docker-images/linux/run-tests.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -e + +# Start system dbus +sudo dbus-daemon --system --fork + +# Start session dbus +# shellcheck disable=SC2046 # SC2046: quote this to prevent word-splitting +export $(dbus-launch) + +# Start Wayland +weston --no-config --socket=wayland-0 --backend=headless & +export WAYLAND_DISPLAY=wayland-0 + +# Start X11 (on Wayland) +Xwayland & +export DISPLAY=:0 + +"${@:-bash}" From 2767765c5a92cfb0d95b51fee3e2bee50a48312e Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Thu, 25 Apr 2024 19:11:37 +0200 Subject: [PATCH 06/22] feat(web): add CI image for building Keyman Web --- resources/docker-images/web/Dockerfile | 71 ++++++++++++++++++++++++ resources/docker-images/web/run-tests.sh | 12 ++++ 2 files changed, 83 insertions(+) create mode 100644 resources/docker-images/web/Dockerfile create mode 100755 resources/docker-images/web/run-tests.sh diff --git a/resources/docker-images/web/Dockerfile b/resources/docker-images/web/Dockerfile new file mode 100644 index 00000000000..83aa5432d9b --- /dev/null +++ b/resources/docker-images/web/Dockerfile @@ -0,0 +1,71 @@ +# Copyright (c) 2024 SIL International. All rights reserved. + +ARG BASE_VERSION +FROM --platform=amd64 keymanapp/keyman-base-ci:${BASE_VERSION} + +LABEL org.opencontainers.image.authors="SIL International." +LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" +LABEL org.opencontainers.image.title="Keyman for Web Build Image" + +USER root +RUN apt-get install -qy git jq xvfb xserver-xephyr metacity + +COPY run-tests.sh /usr/bin/run-tests.sh + +# Install node +ARG NODE_MAJOR +RUN set -eu; \ + NODE_VERSION=$(curl -sSL https://unofficial-builds.nodejs.org/download/release/ | cut -d'>' -f2 | cut -d'/' -f1 | grep v${NODE_MAJOR} | sort -V | tail -1) && \ + echo "Installing node version ${NODE_VERSION}" && \ + curl -fsSLO --compressed "https://unofficial-builds.nodejs.org/download/release/${NODE_VERSION}/node-${NODE_VERSION}-linux-x64-glibc-217.tar.xz" && \ + tar -xJf "node-${NODE_VERSION}-linux-x64-glibc-217.tar.xz" -C /usr/local --strip-components=1 --no-same-owner && \ + ln -s /usr/local/bin/node /usr/local/bin/nodejs + +# Install emscripten +ARG EMSCRIPTEN_VERSION +RUN echo "Installing emscripten version ${EMSCRIPTEN_VERSION}" && \ + cd /usr/share && \ + git clone https://github.com/emscripten-core/emsdk.git && \ + cd emsdk && \ + ./emsdk install ${EMSCRIPTEN_VERSION} && \ + ./emsdk activate ${EMSCRIPTEN_VERSION} && \ + echo "#!/bin/bash" > /usr/bin/bashwrapper && \ + echo "export EMSCRIPTEN_BASE=/usr/share/emsdk/upstream/emscripten" >> /usr/bin/bashwrapper + +# Keyman Web +RUN curl --output google-chrome-stable_current_amd64.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ + apt-get -qy install ./google-chrome-stable_current_amd64.deb && \ + rm google-chrome-stable_current_amd64.deb && \ + echo "export CHROME_BIN=/opt/google/chrome/chrome" >> /usr/bin/bashwrapper + +RUN < /etc/apt/preferences.d/mozilla +Package: * +Pin: origin packages.mozilla.org +Pin-Priority: 1000 +EOF +RUN curl https://packages.mozilla.org/apt/repo-signing-key.gpg > /etc/apt/keyrings/packages.mozilla.org.asc && \ + echo "deb [signed-by=/etc/apt/keyrings/packages.mozilla.org.asc] https://packages.mozilla.org/apt mozilla main" >> /etc/apt/sources.list.d/mozilla.list && \ + apt-get update && \ + apt-get -qy install firefox && \ + echo "export FIREFOX_BIN=/usr/bin/firefox" >> /usr/bin/bashwrapper + +# Finish bashwrapper script and adjust permissions +RUN <> /usr/bin/bashwrapper + +if [[ "\$@" =~ test ]] && [ -f /usr/bin/run-tests.sh ]; then + /usr/bin/run-tests.sh "\${@:-bash}" +else + "\${@:-bash}" +fi +EOF + +RUN chmod +x /usr/bin/bashwrapper && \ + chown -R build:build $HOME + +# now, switch to build user +USER build + +VOLUME /home/build/build +WORKDIR /home/build/build + +ENTRYPOINT [ "/usr/bin/bashwrapper" ] diff --git a/resources/docker-images/web/run-tests.sh b/resources/docker-images/web/run-tests.sh new file mode 100755 index 00000000000..1a3100159f8 --- /dev/null +++ b/resources/docker-images/web/run-tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e +echo "Starting Xvfb..." +Xvfb -screen 0 1024x768x24 :33 &> /dev/null & +sleep 1 +echo "Starting Xephyr..." +DISPLAY=:33 Xephyr :32 -screen 1024x768 &> /dev/null & +sleep 1 +echo "Starting metacity" +metacity --display=:32 &> /dev/null & +export DISPLAY=:32 +"${@:-bash}" From 23d8a014fd0a03fedd89d59fd2b10cdd6a086123 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Fri, 3 May 2024 10:30:38 +0200 Subject: [PATCH 07/22] feat(android): add CI image for building Keyman for Android --- resources/docker-images/android/Dockerfile | 68 ++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 resources/docker-images/android/Dockerfile diff --git a/resources/docker-images/android/Dockerfile b/resources/docker-images/android/Dockerfile new file mode 100644 index 00000000000..5cd8fb1e638 --- /dev/null +++ b/resources/docker-images/android/Dockerfile @@ -0,0 +1,68 @@ +# Copyright (c) 2024 SIL International. All rights reserved. + +ARG BASE_VERSION=latest +FROM --platform=amd64 keymanapp/keyman-base-ci:${BASE_VERSION} +LABEL org.opencontainers.image.authors="SIL International." +LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" +LABEL org.opencontainers.image.title="Keyman Android Build Image" + +# Keyman for Android +SHELL ["/bin/bash", "-c"] +# Starting with Ubuntu 24.04 sdkmanager is no longer available, instead +# a version dependent package allows to install the cmdline tools +ARG JAVA_VERSION=11 +RUN </dev/null) + echo "OS_VER=${OS_VER}" + if (( ${OS_VER%%.*} > 22 )); then + PKG_SDKMANAGER=google-android-cmdline-tools-13.0-installer + DIR_SDK=/usr/lib/android-sdk + else + PKG_SDKMANAGER=sdkmanager + DIR_SDK=/opt/android-sdk + fi + apt-get -q -y install gradle maven pandoc $PKG_SDKMANAGER jq openjdk-${JAVA_VERSION}-jdk + sdkmanager platform-tools + yes | sdkmanager --licenses + chown -R build:build $DIR_SDK + echo "#!/bin/bash" > /usr/bin/bashwrapper + echo "export ANDROID_HOME=$DIR_SDK" >> /usr/bin/bashwrapper + echo "export JAVA_HOME_${JAVA_VERSION}=/usr/lib/jvm/java-${JAVA_VERSION}-openjdk-amd64" >> /usr/bin/bashwrapper +EOF + +# Finish bashwrapper script and adjust permissions +RUN <> /usr/bin/bashwrapper + +if [[ "\$@" =~ test ]] && [ -f /usr/bin/run-tests.sh ]; then + /usr/bin/run-tests.sh "\${@:-bash}" +else + "\${@:-bash}" +fi +EOF + +RUN chmod +x /usr/bin/bashwrapper && \ + chown -R build:build $HOME + +# now, switch to build user +USER build + +VOLUME /home/build/build +WORKDIR /home/build/build + +# Pre-install gradle. This will put files in ~/.gradle which will speed up builds. +RUN mkdir -p $HOME/tmp/gradle/wrapper && \ + # KMEA uses gradle-7.5.1-bin + curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradle/wrapper/gradle-wrapper.jar && \ + curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.properties https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradle/wrapper/gradle-wrapper.properties && \ + curl --location --output $HOME/tmp/gradlew https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradlew && \ + chmod +x $HOME/tmp/gradlew && \ + $HOME/tmp/gradlew --quiet && \ + # Some projects use gradle-7.5.1-all, so we pre-install that as well + curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.jar && \ + curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.properties https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.properties && \ + curl --location --output $HOME/tmp/gradlew https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradlew && \ + chmod +x $HOME/tmp/gradlew && \ + $HOME/tmp/gradlew --quiet && \ + rm -rf $HOME/tmp + +ENTRYPOINT [ "/usr/bin/bashwrapper" ] From 8a4cdcedf411a85c3c1fffa58265ce70dfa268c3 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 6 May 2024 18:22:08 +0200 Subject: [PATCH 08/22] feat(core): add CI image for building Keyman Core --- resources/docker-images/core/.gitignore | 1 + resources/docker-images/core/Dockerfile | 64 +++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 resources/docker-images/core/.gitignore create mode 100644 resources/docker-images/core/Dockerfile diff --git a/resources/docker-images/core/.gitignore b/resources/docker-images/core/.gitignore new file mode 100644 index 00000000000..4db28ac4954 --- /dev/null +++ b/resources/docker-images/core/.gitignore @@ -0,0 +1 @@ +control diff --git a/resources/docker-images/core/Dockerfile b/resources/docker-images/core/Dockerfile new file mode 100644 index 00000000000..9b7073b7a27 --- /dev/null +++ b/resources/docker-images/core/Dockerfile @@ -0,0 +1,64 @@ +# Copyright (c) 2024 SIL International. All rights reserved. + +ARG BASE_VERSION +FROM --platform=amd64 keymanapp/keyman-base-ci:${BASE_VERSION} + +LABEL org.opencontainers.image.authors="SIL International." +LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" +LABEL org.opencontainers.image.title="Keyman Core Build Image" + +USER root +RUN apt-get install -qy git jq llvm meson pkgconf \ + xvfb xserver-xephyr metacity + +#### TMP until we properly figure out the dependencies needed for Core +#### We should not need to install /tmp/control +# Install dependencies +ADD control /tmp/control +RUN apt-get install -qy python3 python3-setuptools python3-coverage \ + devscripts equivs libdatetime-perl meson pkgconf lcov gcovr xvfb \ + xserver-xephyr metacity mutter dbus-x11 weston xwayland && \ + (yes | mk-build-deps --install /tmp/control) || true && \ + rm /tmp/control +#### TMP END + +# Install node +ARG NODE_MAJOR +RUN set -eu; \ + NODE_VERSION=$(curl -sSL https://unofficial-builds.nodejs.org/download/release/ | cut -d'>' -f2 | cut -d'/' -f1 | grep v${NODE_MAJOR} | sort -V | tail -1) && \ + echo "Installing node version ${NODE_VERSION}" && \ + curl -fsSLO --compressed "https://unofficial-builds.nodejs.org/download/release/${NODE_VERSION}/node-${NODE_VERSION}-linux-x64-glibc-217.tar.xz" && \ + tar -xJf "node-${NODE_VERSION}-linux-x64-glibc-217.tar.xz" -C /usr/local --strip-components=1 --no-same-owner && \ + ln -s /usr/local/bin/node /usr/local/bin/nodejs + +# Install emscripten +ARG EMSCRIPTEN_VERSION +RUN echo "Installing emscripten version ${EMSCRIPTEN_VERSION}" && \ + cd /usr/share && \ + git clone https://github.com/emscripten-core/emsdk.git && \ + cd emsdk && \ + ./emsdk install ${EMSCRIPTEN_VERSION} && \ + ./emsdk activate ${EMSCRIPTEN_VERSION} && \ + echo "#!/bin/bash" > /usr/bin/bashwrapper && \ + echo "export EMSCRIPTEN_BASE=/usr/share/emsdk/upstream/emscripten" >> /usr/bin/bashwrapper + +# Finish bashwrapper script and adjust permissions +RUN <> /usr/bin/bashwrapper + +if [[ "\$@" =~ test ]] && [ -f /usr/bin/run-tests.sh ]; then + /usr/bin/run-tests.sh "\${@:-bash}" +else + "\${@:-bash}" +fi +EOF + +RUN chmod +x /usr/bin/bashwrapper && \ + chown -R build:build $HOME + +# now, switch to build user +USER build + +VOLUME /home/build/build +WORKDIR /home/build/build + +ENTRYPOINT [ "/usr/bin/bashwrapper" ] From ad4bbdf0470f33daad62ab3b500a18e0d803f596 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 28 Aug 2024 14:15:19 +0700 Subject: [PATCH 09/22] feat(common): add script to run build and tests in Docker container Addresses code review comments. --- resources/docker-images/README.md | 124 +++++------------------------- resources/docker-images/run.sh | 56 ++++++++++++++ 2 files changed, 74 insertions(+), 106 deletions(-) create mode 100755 resources/docker-images/run.sh diff --git a/resources/docker-images/README.md b/resources/docker-images/README.md index 5a241af91f3..8c5593e2fb4 100644 --- a/resources/docker-images/README.md +++ b/resources/docker-images/README.md @@ -7,7 +7,7 @@ build for the platform. ## Prerequisites You'll need Docker Buildx installed to successfully be able to build the -container images. This is easiest achieved by installing the [official +container images. This is most easily achieved by installing the [official Docker version](https://docs.docker.com/engine/install/ubuntu/). Currently it is not possible to use Podman instead of Docker due to a number @@ -22,9 +22,10 @@ resources/docker-images/build.sh ``` By default this will create 64-bit images for building -Keyman for Android, Keyman for Linux and Keyman for Web. These images -are based on the Ubuntu 24.04 with Node 20 and Emscripten -3.1.44 (for the exact versions, see [`minimum-versions.inc.sh`](../build/minimum-versions.inc.sh)) +Keyman Core, Keyman for Android, Keyman for Linux and +Keyman for Web. These images are based on the Ubuntu 24.04 +with Node 20 and Emscripten 3.1.58 (for the exact versions, +see [`minimum-versions.inc.sh`](../build/minimum-versions.inc.sh)) and are named e.g. `keyman-core-ci:default`. The versions can be changed, e.g. @@ -41,108 +42,19 @@ Once the image is built, it may be used to build parts of Keyman. It is possible to build locally with these images: -- Keyman Core - - ```shell - # build 'Keyman Core' in docker - # keep build artifacts separate - mkdir -p $(git rev-parse --show-toplevel)/core/build/docker-core - docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ - -v $(git rev-parse --show-toplevel)/core/build/docker-core:/home/build/build/core/build \ - keymanapp/keyman-core-ci:default \ - core/build.sh --debug build - ``` - - Note: Since the generated binaries are platform dependent we put them in a container - specific directory. - -- Keyman for Linux - - ```shell - # build 'Keyman for Linux' installation in docker - # keep build artifacts separate - mkdir -p $(git rev-parse --show-toplevel)/linux/build/docker-linux - docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ - -v $(git rev-parse --show-toplevel)/linux/build/docker-linux:/home/build/build/linux/build \ - -e DESTDIR=/tmp \ - keymanapp/keyman-linux-ci:default \ - linux/build.sh --debug build install - ``` - - Note: Since the generated binaries are platform dependent we put them in a container - specific directory. - -- Keyman Web - - ```shell - # build 'Keyman Web' in docker - docker run --privileged -it --rm \ - -v $(git rev-parse --show-toplevel):/home/build/build \ - keymanapp/keyman-web-ci:default \ - web/build.sh --debug - ``` - -- Keyman for Android - - ```shell - # build 'Keyman for Android' in docker - docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ - keymanapp/keyman-android-ci:default \ - android/build.sh --debug - ``` +```shell +# build 'Keyman Core' in docker +resources/docker-images/run.sh core -- core/build.sh --debug build +``` + +Note: For Core and Linux we put the generated binaries in a +container specific directory because they are platform dependent. ## Running tests locally -- Keyman Core - - ```shell - # build 'Keyman Core' in docker - # keep build artifacts separate - mkdir -p $(git rev-parse --show-toplevel)/core/build/docker-core - docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ - -v $(git rev-parse --show-toplevel)/core/build/docker-core:/home/build/build/core/build \ - keymanapp/keyman-core-ci:default \ - core/build.sh --debug test - ``` - - Note: Since the generated binaries are platform dependent we put them in a container - specific directory. - -- Keyman for Linux - - ```shell - # build 'Keyman for Linux' installation in docker - # keep build artifacts separate - mkdir -p $(git rev-parse --show-toplevel)/linux/build/docker-linux - docker run --privileged -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ - -v $(git rev-parse --show-toplevel)/linux/build/docker-linux:/home/build/build/linux/build \ - -e DESTDIR=/tmp \ - keymanapp/keyman-linux-ci:default \ - linux/build.sh --debug test - ``` - - Note: this requires the `--privileged` parameter in order for all tests to pass! - - Note: Since the generated binaries are platform dependent we put them in a container - specific directory. - -- Keyman Web - - ```shell - # build 'Keyman Web' in docker - docker run --privileged -it --rm \ - -v $(git rev-parse --show-toplevel):/home/build/build \ - keymanapp/keyman-web-ci:default \ - web/build.sh --debug test - ``` - - Note: this requires the `--privileged` parameter in order for all tests to pass! - -- Keyman for Android - - ```shell - # build 'Keyman for Android' in docker - docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ - keymanapp/keyman-android-ci:default \ - android/build.sh --debug test - ``` +To run the tests locally, use the `run.sh` script: + +```shell +# Run common/web tests +resources/docker-images/run.sh web -- common/web/build.sh --debug test +``` diff --git a/resources/docker-images/run.sh b/resources/docker-images/run.sh new file mode 100755 index 00000000000..ffe136efff6 --- /dev/null +++ b/resources/docker-images/run.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +. "${KEYMAN_ROOT}/resources/build/minimum-versions.inc.sh" + +################################ Main script ################################ + +builder_describe \ + "Run build.sh script inside of a docker image. Pass the build script and parameters after --." \ + "android" \ + "core" \ + "linux" \ + "web" \ + "--ubuntu-version=UBUNTU_VERSION The Ubuntu version (default: ${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER})" + +builder_parse "$@" + +run_android() { + docker run -it --rm -v ${KEYMAN_ROOT}:/home/build/build \ + -v ${KEYMAN_ROOT}/core/build/docker-core:/home/build/build/core/build \ + keymanapp/keyman-android-ci:default \ + "${builder_extra_params[@]}" +} + +run_core() { + mkdir -p ${KEYMAN_ROOT}/core/build/docker-core + docker run -it --rm -v ${KEYMAN_ROOT}:/home/build/build \ + -v ${KEYMAN_ROOT}/core/build/docker-core:/home/build/build/core/build \ + keymanapp/keyman-core-ci:default \ + "${builder_extra_params[@]}" +} + +run_linux() { + mkdir -p ${KEYMAN_ROOT}/linux/build/docker-linux + docker run -it --privileged --rm -v ${KEYMAN_ROOT}:/home/build/build \ + -v ${KEYMAN_ROOT}/linux/build/docker-linux:/home/build/build/linux/build \ + -e DESTDIR=/tmp \ + keymanapp/keyman-linux-ci:default \ + "${builder_extra_params[@]}" +} + +run_web() { + docker run -it --privileged --rm -v ${KEYMAN_ROOT}:/home/build/build \ + keymanapp/keyman-web-ci:default \ + "${builder_extra_params[@]}" +} + +builder_run_action android run_android +builder_run_action core run_core +builder_run_action linux run_linux +builder_run_action web run_web From d6c532032539bb60cbbd4da3c6297c213c41f6e0 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Thu, 29 Aug 2024 08:45:47 +0700 Subject: [PATCH 10/22] chore(linux): address code review comments - use KEYMAN_USE_NVM and KEYMAN_USE_EMSDK - pre-install node (where necessary) to prevent having to do it on each build - adjust to current `master` - fix a few bugs in the Dockerfile - get required node version from package.json (through `shellHelperFunctions.sh`) --- resources/docker-images/android/Dockerfile | 5 +-- resources/docker-images/base/Dockerfile | 8 +++- resources/docker-images/build.sh | 20 +++++----- resources/docker-images/core/Dockerfile | 42 ++++++++++----------- resources/docker-images/linux/Dockerfile | 5 +-- resources/docker-images/web/Dockerfile | 44 +++++++++++----------- resources/shellHelperFunctions.sh | 6 ++- 7 files changed, 68 insertions(+), 62 deletions(-) diff --git a/resources/docker-images/android/Dockerfile b/resources/docker-images/android/Dockerfile index 5cd8fb1e638..1e0dd089b72 100644 --- a/resources/docker-images/android/Dockerfile +++ b/resources/docker-images/android/Dockerfile @@ -1,7 +1,7 @@ # Copyright (c) 2024 SIL International. All rights reserved. -ARG BASE_VERSION=latest -FROM --platform=amd64 keymanapp/keyman-base-ci:${BASE_VERSION} +ARG BASE_VERSION=default +FROM keymanapp/keyman-base-ci:${BASE_VERSION} LABEL org.opencontainers.image.authors="SIL International." LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" LABEL org.opencontainers.image.title="Keyman Android Build Image" @@ -25,7 +25,6 @@ RUN < /usr/bin/bashwrapper echo "export ANDROID_HOME=$DIR_SDK" >> /usr/bin/bashwrapper echo "export JAVA_HOME_${JAVA_VERSION}=/usr/lib/jvm/java-${JAVA_VERSION}-openjdk-amd64" >> /usr/bin/bashwrapper EOF diff --git a/resources/docker-images/base/Dockerfile b/resources/docker-images/base/Dockerfile index 116d4b80846..a22328d5654 100644 --- a/resources/docker-images/base/Dockerfile +++ b/resources/docker-images/base/Dockerfile @@ -1,7 +1,7 @@ # Copyright (c) 2024 SIL International. All rights reserved. ARG UBUNTU_VERSION=latest -FROM --platform=amd64 ubuntu:${UBUNTU_VERSION} +FROM ubuntu:${UBUNTU_VERSION} LABEL org.opencontainers.image.authors="SIL International." LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" @@ -28,3 +28,9 @@ RUN apt-get -q -y update && \ # Allow build user to use `sudo` RUN echo "build ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +RUN < /usr/bin/bashwrapper +#!/bin/bash +export KEYMAN_USE_NVM=1 +export KEYMAN_USE_EMSDK=1 +EOF diff --git a/resources/docker-images/build.sh b/resources/docker-images/build.sh index 294d5f463aa..d1fdc41eeb4 100755 --- a/resources/docker-images/build.sh +++ b/resources/docker-images/build.sh @@ -9,6 +9,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" ################################ Main script ################################ . "${KEYMAN_ROOT}/resources/build/minimum-versions.inc.sh" +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" builder_describe \ "Build docker images" \ @@ -18,8 +19,6 @@ builder_describe \ ":linux" \ ":web" \ "--ubuntu-version=UBUNTU_VERSION The Ubuntu version (default: ${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER})" \ - "--node=NODE_MAJOR Node version (default: ${KEYMAN_MIN_VERSION_NODE_MAJOR})" \ - "--emscripten=EMSCRIPTEN_VERSION Emscripten version (default: ${KEYMAN_MIN_VERSION_EMSCRIPTEN})" \ "--no-cache Force rebuild of docker images" \ "build" @@ -49,11 +48,11 @@ _add_build_args() { _convert_parameters_to_build_args() { build_args=() build_version= + local required_node_version="$(_print_expected_node_version)" - _add_build_args UBUNTU_VERSION KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER "" - _add_build_args JAVA_VERSION KEYMAN_VERSION_JAVA java - _add_build_args NODE_MAJOR KEYMAN_MIN_VERSION_NODE_MAJOR node - _add_build_args EMSCRIPTEN_VERSION KEYMAN_MIN_VERSION_EMSCRIPTEN emscr + _add_build_args UBUNTU_VERSION KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER "" + _add_build_args JAVA_VERSION KEYMAN_VERSION_JAVA java + _add_build_args REQUIRED_NODE_VERSION required_node_version "" if [[ -n "${BASE_VERSION:-}" ]]; then build_args+=(--build-arg="BASE_VERSION=${BASE_VERSION}") @@ -61,8 +60,7 @@ _convert_parameters_to_build_args() { } _is_default_values() { - [[ -z "${UBUNTU_VERSION:-}" ]] && [[ -z "${JAVA_VERSION:-}" ]] && \ - [[ -z "${NODE_MAJOR:-}" ]] && [[ -z "${EMSCRIPTEN_VERSION:-}" ]] + [[ -z "${UBUNTU_VERSION:-}" ]] && [[ -z "${JAVA_VERSION:-}" ]] } build_action() { @@ -86,14 +84,14 @@ build_action() { OPTION_NO_CACHE="--no-cache" fi - cd "${platform}" || true + cd "${platform}" # shellcheck disable=SC2248 - docker build ${OPTION_NO_CACHE:-} -t "keymanapp/keyman-${platform}-ci:${build_version}" "${build_args[@]}" . + docker build ${OPTION_NO_CACHE:-} --platform amd64 -t "keymanapp/keyman-${platform}-ci:${build_version}" "${build_args[@]}" . # If the user didn't specify particular versions we will additionaly create an image # with the tag 'default'. if _is_default_values; then builder_echo debug "Setting default tag for ${platform}" - docker build . -t "keymanapp/keyman-${platform}-ci:default" "${build_args[@]}" + docker build . --platform amd64 -t "keymanapp/keyman-${platform}-ci:default" "${build_args[@]}" fi cd - || true builder_echo success "Docker image 'keymanapp/keyman-${platform}-ci:${build_version}' built" diff --git a/resources/docker-images/core/Dockerfile b/resources/docker-images/core/Dockerfile index 9b7073b7a27..d4a74eda4ef 100644 --- a/resources/docker-images/core/Dockerfile +++ b/resources/docker-images/core/Dockerfile @@ -1,7 +1,10 @@ # Copyright (c) 2024 SIL International. All rights reserved. +# ARGS used in this file: +# - ARG BASE_VERSION=default +# - ARG REQUIRED_NODE_VERSION=18 -ARG BASE_VERSION -FROM --platform=amd64 keymanapp/keyman-base-ci:${BASE_VERSION} +ARG BASE_VERSION=default +FROM keymanapp/keyman-base-ci:${BASE_VERSION} LABEL org.opencontainers.image.authors="SIL International." LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" @@ -22,25 +25,14 @@ RUN apt-get install -qy python3 python3-setuptools python3-coverage \ rm /tmp/control #### TMP END -# Install node -ARG NODE_MAJOR -RUN set -eu; \ - NODE_VERSION=$(curl -sSL https://unofficial-builds.nodejs.org/download/release/ | cut -d'>' -f2 | cut -d'/' -f1 | grep v${NODE_MAJOR} | sort -V | tail -1) && \ - echo "Installing node version ${NODE_VERSION}" && \ - curl -fsSLO --compressed "https://unofficial-builds.nodejs.org/download/release/${NODE_VERSION}/node-${NODE_VERSION}-linux-x64-glibc-217.tar.xz" && \ - tar -xJf "node-${NODE_VERSION}-linux-x64-glibc-217.tar.xz" -C /usr/local --strip-components=1 --no-same-owner && \ - ln -s /usr/local/bin/node /usr/local/bin/nodejs - -# Install emscripten -ARG EMSCRIPTEN_VERSION -RUN echo "Installing emscripten version ${EMSCRIPTEN_VERSION}" && \ - cd /usr/share && \ - git clone https://github.com/emscripten-core/emsdk.git && \ - cd emsdk && \ - ./emsdk install ${EMSCRIPTEN_VERSION} && \ - ./emsdk activate ${EMSCRIPTEN_VERSION} && \ - echo "#!/bin/bash" > /usr/bin/bashwrapper && \ - echo "export EMSCRIPTEN_BASE=/usr/share/emsdk/upstream/emscripten" >> /usr/bin/bashwrapper +# Install NVM +RUN NVM_RELEASE=$(curl -s https://api.github.com/repos/nvm-sh/nvm/releases/latest | grep tag_name | cut -d : -f 2 | cut -d '"' -f 2) && \ + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_RELEASE}/install.sh | bash +RUN <> /usr/bin/bashwrapper +PATH=/home/build/.keyman/node:\$PATH +export NVM_DIR="$HOME/.nvm" +. /home/build/.nvm/nvm.sh +EOF # Finish bashwrapper script and adjust permissions RUN <> /usr/bin/bashwrapper @@ -58,6 +50,14 @@ RUN chmod +x /usr/bin/bashwrapper && \ # now, switch to build user USER build +# Pre-install node +ARG REQUIRED_NODE_VERSION=18 +RUN echo "HOME=\${HOME}; REQUIRED_NODE_VERSION=${REQUIRED_NODE_VERSION}" && \ + export NVM_DIR="/home/build/.nvm" && \ + . /home/build/.nvm/nvm.sh && \ + nvm install "${REQUIRED_NODE_VERSION}" && \ + nvm use "${REQUIRED_NODE_VERSION}" + VOLUME /home/build/build WORKDIR /home/build/build diff --git a/resources/docker-images/linux/Dockerfile b/resources/docker-images/linux/Dockerfile index 6766aa08591..134bc1d1a52 100644 --- a/resources/docker-images/linux/Dockerfile +++ b/resources/docker-images/linux/Dockerfile @@ -1,7 +1,7 @@ # Copyright (c) 2024 SIL International. All rights reserved. -ARG BASE_VERSION -FROM --platform=amd64 keymanapp/keyman-base-ci:${BASE_VERSION} +ARG BASE_VERSION=default +FROM keymanapp/keyman-base-ci:${BASE_VERSION} LABEL org.opencontainers.image.authors="SIL International." LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" LABEL org.opencontainers.image.title="Keyman Linux Build Image" @@ -29,7 +29,6 @@ RUN LCOV_VERSION=$(dpkg -s lcov | grep Version | cut -d' ' -f2) && \ RUN mkdir -p /var/run/1000 && \ chown build:build /var/run/1000 && \ - echo "#!/bin/bash" > /usr/bin/bashwrapper && \ echo "export XDG_RUNTIME_DIR=/var/run/1000" >> /usr/bin/bashwrapper COPY run-tests.sh /usr/bin/run-tests.sh diff --git a/resources/docker-images/web/Dockerfile b/resources/docker-images/web/Dockerfile index 83aa5432d9b..d438ca86ff3 100644 --- a/resources/docker-images/web/Dockerfile +++ b/resources/docker-images/web/Dockerfile @@ -1,36 +1,28 @@ # Copyright (c) 2024 SIL International. All rights reserved. +# ARGS used in this file: +# - ARG BASE_VERSION=default +# - ARG REQUIRED_NODE_VERSION=18 -ARG BASE_VERSION -FROM --platform=amd64 keymanapp/keyman-base-ci:${BASE_VERSION} +ARG BASE_VERSION=default +FROM keymanapp/keyman-base-ci:${BASE_VERSION} LABEL org.opencontainers.image.authors="SIL International." LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" LABEL org.opencontainers.image.title="Keyman for Web Build Image" USER root -RUN apt-get install -qy git jq xvfb xserver-xephyr metacity +RUN apt-get install -qy git jq xvfb xserver-xephyr metacity libevent-2.1-7t64 COPY run-tests.sh /usr/bin/run-tests.sh -# Install node -ARG NODE_MAJOR -RUN set -eu; \ - NODE_VERSION=$(curl -sSL https://unofficial-builds.nodejs.org/download/release/ | cut -d'>' -f2 | cut -d'/' -f1 | grep v${NODE_MAJOR} | sort -V | tail -1) && \ - echo "Installing node version ${NODE_VERSION}" && \ - curl -fsSLO --compressed "https://unofficial-builds.nodejs.org/download/release/${NODE_VERSION}/node-${NODE_VERSION}-linux-x64-glibc-217.tar.xz" && \ - tar -xJf "node-${NODE_VERSION}-linux-x64-glibc-217.tar.xz" -C /usr/local --strip-components=1 --no-same-owner && \ - ln -s /usr/local/bin/node /usr/local/bin/nodejs - -# Install emscripten -ARG EMSCRIPTEN_VERSION -RUN echo "Installing emscripten version ${EMSCRIPTEN_VERSION}" && \ - cd /usr/share && \ - git clone https://github.com/emscripten-core/emsdk.git && \ - cd emsdk && \ - ./emsdk install ${EMSCRIPTEN_VERSION} && \ - ./emsdk activate ${EMSCRIPTEN_VERSION} && \ - echo "#!/bin/bash" > /usr/bin/bashwrapper && \ - echo "export EMSCRIPTEN_BASE=/usr/share/emsdk/upstream/emscripten" >> /usr/bin/bashwrapper +# Install NVM +RUN NVM_RELEASE=$(curl -s https://api.github.com/repos/nvm-sh/nvm/releases/latest | grep tag_name | cut -d : -f 2 | cut -d '"' -f 2) && \ + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_RELEASE}/install.sh | bash +RUN <> /usr/bin/bashwrapper +PATH=/home/build/.keyman/node:\$PATH +export NVM_DIR="$HOME/.nvm" +. /home/build/.nvm/nvm.sh +EOF # Keyman Web RUN curl --output google-chrome-stable_current_amd64.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ @@ -65,6 +57,14 @@ RUN chmod +x /usr/bin/bashwrapper && \ # now, switch to build user USER build +# Pre-install node +ARG REQUIRED_NODE_VERSION=18 +RUN echo "HOME=\${HOME}; REQUIRED_NODE_VERSION=${REQUIRED_NODE_VERSION}" && \ + export NVM_DIR="/home/build/.nvm" && \ + . /home/build/.nvm/nvm.sh && \ + nvm install "${REQUIRED_NODE_VERSION}" && \ + nvm use "${REQUIRED_NODE_VERSION}" + VOLUME /home/build/build WORKDIR /home/build/build diff --git a/resources/shellHelperFunctions.sh b/resources/shellHelperFunctions.sh index 7ded208d448..68d498cb4d7 100755 --- a/resources/shellHelperFunctions.sh +++ b/resources/shellHelperFunctions.sh @@ -272,10 +272,14 @@ verify_npm_setup() { popd > /dev/null } +_print_expected_node_version() { +"$JQ" -r '.engines.node' "$KEYMAN_ROOT/package.json" +} + # Use nvm to select a node version according to package.json # see /docs/build/node.md _select_node_version_with_nvm() { - local REQUIRED_NODE_VERSION="$("$JQ" -r '.engines.node' "$KEYMAN_ROOT/package.json")" + local REQUIRED_NODE_VERSION="$(_print_expected_node_version)" if [[ $BUILDER_OS != win ]]; then # launch nvm in a sub process, see _builder_nvm.sh for details From 1772fdf05581390da4156ad23623cad41ddb2235 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Sat, 31 Aug 2024 23:06:28 +0700 Subject: [PATCH 11/22] chore(linux): pre-install EMSCRIPTEN Also add test action to build.sh which builds all images and then tests them by running configure,build,test on each. Also add new dependencies to web image which Playwright requires. --- resources/docker-images/base/Dockerfile | 1 - resources/docker-images/build.sh | 22 ++++++++++++++++++---- resources/docker-images/core/Dockerfile | 18 +++++++++++++++++- resources/docker-images/web/Dockerfile | 25 +++++++++++++++++++++++-- 4 files changed, 58 insertions(+), 8 deletions(-) diff --git a/resources/docker-images/base/Dockerfile b/resources/docker-images/base/Dockerfile index a22328d5654..b7fa3dce72e 100644 --- a/resources/docker-images/base/Dockerfile +++ b/resources/docker-images/base/Dockerfile @@ -32,5 +32,4 @@ RUN echo "build ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers RUN < /usr/bin/bashwrapper #!/bin/bash export KEYMAN_USE_NVM=1 -export KEYMAN_USE_EMSDK=1 EOF diff --git a/resources/docker-images/build.sh b/resources/docker-images/build.sh index d1fdc41eeb4..a9c68c4106a 100755 --- a/resources/docker-images/build.sh +++ b/resources/docker-images/build.sh @@ -20,7 +20,8 @@ builder_describe \ ":web" \ "--ubuntu-version=UBUNTU_VERSION The Ubuntu version (default: ${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER})" \ "--no-cache Force rebuild of docker images" \ - "build" + "build" \ + "test" builder_parse "$@" @@ -50,9 +51,10 @@ _convert_parameters_to_build_args() { build_version= local required_node_version="$(_print_expected_node_version)" - _add_build_args UBUNTU_VERSION KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER "" - _add_build_args JAVA_VERSION KEYMAN_VERSION_JAVA java - _add_build_args REQUIRED_NODE_VERSION required_node_version "" + _add_build_args UBUNTU_VERSION KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER "" + _add_build_args JAVA_VERSION KEYMAN_VERSION_JAVA java + _add_build_args REQUIRED_NODE_VERSION required_node_version node + _add_build_args REQUIRED_EMSCRIPTEN_VERSION KEYMAN_MIN_VERSION_EMSCRIPTEN emsdk if [[ -n "${BASE_VERSION:-}" ]]; then build_args+=(--build-arg="BASE_VERSION=${BASE_VERSION}") @@ -97,6 +99,13 @@ build_action() { builder_echo success "Docker image 'keymanapp/keyman-${platform}-ci:${build_version}' built" } +test_action() { + local platform=$1 + + builder_echo debug "Testing image for ${platform}" + ./run.sh ${platform} -- ./build.sh configure,build,test:${platform} +} + if builder_has_action build; then build_action base BASE_VERSION="${build_version}" @@ -105,3 +114,8 @@ if builder_has_action build; then builder_run_action build:linux build_action linux builder_run_action build:web build_action web fi + +builder_run_action test:android test_action android +builder_run_action test:core test_action core +builder_run_action test:linux test_action linux +builder_run_action test:web test_action web diff --git a/resources/docker-images/core/Dockerfile b/resources/docker-images/core/Dockerfile index d4a74eda4ef..54f5b92b667 100644 --- a/resources/docker-images/core/Dockerfile +++ b/resources/docker-images/core/Dockerfile @@ -34,6 +34,20 @@ export NVM_DIR="$HOME/.nvm" . /home/build/.nvm/nvm.sh EOF +# Pre-install emscripten +USER build +ARG REQUIRED_EMSCRIPTEN_VERSION=1.0 +RUN echo "Installing emscripten version ${REQUIRED_EMSCRIPTEN_VERSION}" && \ + export EMSDK_KEEP_DOWNLOADS=1 && \ + cd /home/build/ && \ + git clone https://github.com/emscripten-core/emsdk.git && \ + cd emsdk && \ + ./emsdk install ${REQUIRED_EMSCRIPTEN_VERSION} && \ + ./emsdk activate ${REQUIRED_EMSCRIPTEN_VERSION} +USER root +RUN echo "export EMSCRIPTEN_BASE=/home/build/emsdk/upstream/emscripten" >> /usr/bin/bashwrapper && \ + echo "export KEYMAN_USE_EMSDK=1" >> /usr/bin/bashwrapper + # Finish bashwrapper script and adjust permissions RUN <> /usr/bin/bashwrapper @@ -56,7 +70,9 @@ RUN echo "HOME=\${HOME}; REQUIRED_NODE_VERSION=${REQUIRED_NODE_VERSION}" && \ export NVM_DIR="/home/build/.nvm" && \ . /home/build/.nvm/nvm.sh && \ nvm install "${REQUIRED_NODE_VERSION}" && \ - nvm use "${REQUIRED_NODE_VERSION}" + nvm use "${REQUIRED_NODE_VERSION}" && \ + cd /home/build/emsdk/upstream/emscripten && \ + npm install VOLUME /home/build/build WORKDIR /home/build/build diff --git a/resources/docker-images/web/Dockerfile b/resources/docker-images/web/Dockerfile index d438ca86ff3..7adbae6106a 100644 --- a/resources/docker-images/web/Dockerfile +++ b/resources/docker-images/web/Dockerfile @@ -11,7 +11,12 @@ LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" LABEL org.opencontainers.image.title="Keyman for Web Build Image" USER root -RUN apt-get install -qy git jq xvfb xserver-xephyr metacity libevent-2.1-7t64 +RUN apt-get install -qy git jq xvfb xserver-xephyr metacity +# For playwright: +RUN apt-get install -qy libevent-2.1-7t64 libxslt1.1 libwoff1 libvpx9 \ + libgstreamer-plugins-bad1.0-0 libwebpdemux2 libharfbuzz-icu0 \ + libenchant-2-2 libsecret-1-0 libhyphen0 libmanette-0.2-0 libflite1 \ + gstreamer1.0-libav COPY run-tests.sh /usr/bin/run-tests.sh @@ -24,6 +29,20 @@ export NVM_DIR="$HOME/.nvm" . /home/build/.nvm/nvm.sh EOF +# Pre-install emscripten +USER build +ARG REQUIRED_EMSCRIPTEN_VERSION=1.0 +RUN echo "Installing emscripten version ${REQUIRED_EMSCRIPTEN_VERSION}" && \ + export EMSDK_KEEP_DOWNLOADS=1 && \ + cd /home/build/ && \ + git clone https://github.com/emscripten-core/emsdk.git && \ + cd emsdk && \ + ./emsdk install ${REQUIRED_EMSCRIPTEN_VERSION} && \ + ./emsdk activate ${REQUIRED_EMSCRIPTEN_VERSION} +USER root +RUN echo "export EMSCRIPTEN_BASE=/home/build/emsdk/upstream/emscripten" >> /usr/bin/bashwrapper && \ + echo "export KEYMAN_USE_EMSDK=1" >> /usr/bin/bashwrapper + # Keyman Web RUN curl --output google-chrome-stable_current_amd64.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ apt-get -qy install ./google-chrome-stable_current_amd64.deb && \ @@ -63,7 +82,9 @@ RUN echo "HOME=\${HOME}; REQUIRED_NODE_VERSION=${REQUIRED_NODE_VERSION}" && \ export NVM_DIR="/home/build/.nvm" && \ . /home/build/.nvm/nvm.sh && \ nvm install "${REQUIRED_NODE_VERSION}" && \ - nvm use "${REQUIRED_NODE_VERSION}" + nvm use "${REQUIRED_NODE_VERSION}" && \ + cd /home/build/emsdk/upstream/emscripten && \ + npm install VOLUME /home/build/build WORKDIR /home/build/build From 0d6f655d498964da4a2c2d732af63bf946775fb6 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 3 Sep 2024 15:44:08 +0200 Subject: [PATCH 12/22] chore(linux): fix build order --- resources/docker-images/README.md | 5 +++++ resources/docker-images/build.sh | 3 ++- resources/docker-images/run.sh | 5 ++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/resources/docker-images/README.md b/resources/docker-images/README.md index 8c5593e2fb4..c36503ad6a5 100644 --- a/resources/docker-images/README.md +++ b/resources/docker-images/README.md @@ -50,6 +50,11 @@ resources/docker-images/run.sh core -- core/build.sh --debug build Note: For Core and Linux we put the generated binaries in a container specific directory because they are platform dependent. +If you build both with Docker and directly with the build scripts, it is +advisable to run a `git clean -dxf` before switching between the two. The +reason is that the Docker images use a different user, so that paths +will be different. + ## Running tests locally To run the tests locally, use the `run.sh` script: diff --git a/resources/docker-images/build.sh b/resources/docker-images/build.sh index a9c68c4106a..5090193237f 100755 --- a/resources/docker-images/build.sh +++ b/resources/docker-images/build.sh @@ -115,7 +115,8 @@ if builder_has_action build; then builder_run_action build:web build_action web fi -builder_run_action test:android test_action android builder_run_action test:core test_action core builder_run_action test:linux test_action linux builder_run_action test:web test_action web +# Android uses artifacts from web, so it has to come after web +builder_run_action test:android test_action android diff --git a/resources/docker-images/run.sh b/resources/docker-images/run.sh index ffe136efff6..c85c15e873e 100755 --- a/resources/docker-images/run.sh +++ b/resources/docker-images/run.sh @@ -28,7 +28,6 @@ run_android() { } run_core() { - mkdir -p ${KEYMAN_ROOT}/core/build/docker-core docker run -it --rm -v ${KEYMAN_ROOT}:/home/build/build \ -v ${KEYMAN_ROOT}/core/build/docker-core:/home/build/build/core/build \ keymanapp/keyman-core-ci:default \ @@ -38,6 +37,7 @@ run_core() { run_linux() { mkdir -p ${KEYMAN_ROOT}/linux/build/docker-linux docker run -it --privileged --rm -v ${KEYMAN_ROOT}:/home/build/build \ + -v ${KEYMAN_ROOT}/core/build/docker-core:/home/build/build/core/build \ -v ${KEYMAN_ROOT}/linux/build/docker-linux:/home/build/build/linux/build \ -e DESTDIR=/tmp \ keymanapp/keyman-linux-ci:default \ @@ -46,10 +46,13 @@ run_linux() { run_web() { docker run -it --privileged --rm -v ${KEYMAN_ROOT}:/home/build/build \ + -v ${KEYMAN_ROOT}/core/build/docker-core:/home/build/build/core/build \ keymanapp/keyman-web-ci:default \ "${builder_extra_params[@]}" } +mkdir -p ${KEYMAN_ROOT}/core/build/docker-core + builder_run_action android run_android builder_run_action core run_core builder_run_action linux run_linux From 475fd734e6ba6b18b6f0aa1aed85b1ce4c926d11 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 11 Sep 2024 14:29:01 +0200 Subject: [PATCH 13/22] docs(android): document gradle version in `minimum-versions` --- docs/minimum-versions.md | 1 + resources/build/minimum-versions.inc.sh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/minimum-versions.md b/docs/minimum-versions.md index f5826320070..1f0040975ed 100644 --- a/docs/minimum-versions.md +++ b/docs/minimum-versions.md @@ -63,6 +63,7 @@ https://help.keyman.com/developer/engine/android/latest-version/ | KEYMAN_MIN_VERSION_NPM | 10.5.1 | | KEYMAN_MIN_VERSION_VISUAL_STUDIO | 2019 | | KEYMAN_VERSION_CLDR | 45 | +| KEYMAN_VERSION_GRADLE | 7.6.4 | | KEYMAN_VERSION_ICU | 73.1 | | KEYMAN_VERSION_ISO639_3 | 2024-05-22 | | KEYMAN_VERSION_JAVA | 11 | diff --git a/resources/build/minimum-versions.inc.sh b/resources/build/minimum-versions.inc.sh index 97f25094067..3b5f74a8792 100644 --- a/resources/build/minimum-versions.inc.sh +++ b/resources/build/minimum-versions.inc.sh @@ -24,7 +24,8 @@ KEYMAN_MIN_VERSION_EMSCRIPTEN=3.1.58 # Use KEYMAN_USE_EMSDK to automati KEYMAN_MIN_VERSION_VISUAL_STUDIO=2019 KEYMAN_MIN_VERSION_MESON=1.0.0 -KEYMAN_VERSION_ICU=73.1 # See /core/subprojects/icu-minimal.wrap +KEYMAN_VERSION_GRADLE=7.6.4 # See /android/KMEA/gradle/wrapper/gradle-wrapper.properties +KEYMAN_VERSION_ICU=73.1 # See /core/subprojects/icu-minimal.wrap # Language and runtime versions KEYMAN_VERSION_JAVA=11 # We're using Java/OpenJDK 11 From caba09a8d4c4671fc0bcfc6c5d691e4536d159da Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 11 Sep 2024 14:30:22 +0200 Subject: [PATCH 14/22] chore(linux): attempt to fix Android build in Docker These changes bring us one step further, but the build is still failing. --- resources/docker-images/android/Dockerfile | 5 +++-- resources/docker-images/base/Dockerfile | 23 ++++++++++++++++++++++ resources/docker-images/build.sh | 2 +- resources/docker-images/core/Dockerfile | 15 +------------- resources/docker-images/web/Dockerfile | 15 +------------- 5 files changed, 29 insertions(+), 31 deletions(-) diff --git a/resources/docker-images/android/Dockerfile b/resources/docker-images/android/Dockerfile index 1e0dd089b72..8c6833c0c1d 100644 --- a/resources/docker-images/android/Dockerfile +++ b/resources/docker-images/android/Dockerfile @@ -8,6 +8,7 @@ LABEL org.opencontainers.image.title="Keyman Android Build Image" # Keyman for Android SHELL ["/bin/bash", "-c"] + # Starting with Ubuntu 24.04 sdkmanager is no longer available, instead # a version dependent package allows to install the cmdline tools ARG JAVA_VERSION=11 @@ -50,13 +51,13 @@ WORKDIR /home/build/build # Pre-install gradle. This will put files in ~/.gradle which will speed up builds. RUN mkdir -p $HOME/tmp/gradle/wrapper && \ - # KMEA uses gradle-7.5.1-bin + # KMEA uses gradle-7.6.4-bin curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradle/wrapper/gradle-wrapper.jar && \ curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.properties https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradle/wrapper/gradle-wrapper.properties && \ curl --location --output $HOME/tmp/gradlew https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradlew && \ chmod +x $HOME/tmp/gradlew && \ $HOME/tmp/gradlew --quiet && \ - # Some projects use gradle-7.5.1-all, so we pre-install that as well + # Some projects use gradle-7.6.4-all, so we pre-install that as well curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.jar && \ curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.properties https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.properties && \ curl --location --output $HOME/tmp/gradlew https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradlew && \ diff --git a/resources/docker-images/base/Dockerfile b/resources/docker-images/base/Dockerfile index b7fa3dce72e..14bc0b36ccb 100644 --- a/resources/docker-images/base/Dockerfile +++ b/resources/docker-images/base/Dockerfile @@ -33,3 +33,26 @@ RUN < /usr/bin/bashwrapper #!/bin/bash export KEYMAN_USE_NVM=1 EOF + +# Install NVM +RUN NVM_RELEASE=$(curl -s https://api.github.com/repos/nvm-sh/nvm/releases/latest | grep tag_name | cut -d : -f 2 | cut -d '"' -f 2) && \ + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_RELEASE}/install.sh | bash +RUN <> /usr/bin/bashwrapper +PATH=/home/build/.keyman/node:\$PATH +export NVM_DIR="$HOME/.nvm" +. /home/build/.nvm/nvm.sh +EOF + +RUN chmod +x /usr/bin/bashwrapper && \ + chown -R build:build $HOME + +USER build +# Pre-install node +ARG REQUIRED_NODE_VERSION=unset +RUN echo "HOME=\${HOME}; REQUIRED_NODE_VERSION=${REQUIRED_NODE_VERSION}" && \ + export NVM_DIR="/home/build/.nvm" && \ + . /home/build/.nvm/nvm.sh && \ + nvm install "${REQUIRED_NODE_VERSION}" && \ + nvm use "${REQUIRED_NODE_VERSION}" + +USER root diff --git a/resources/docker-images/build.sh b/resources/docker-images/build.sh index 5090193237f..a3254a2f526 100755 --- a/resources/docker-images/build.sh +++ b/resources/docker-images/build.sh @@ -93,7 +93,7 @@ build_action() { # with the tag 'default'. if _is_default_values; then builder_echo debug "Setting default tag for ${platform}" - docker build . --platform amd64 -t "keymanapp/keyman-${platform}-ci:default" "${build_args[@]}" + docker build --platform amd64 -t "keymanapp/keyman-${platform}-ci:default" "${build_args[@]}" . fi cd - || true builder_echo success "Docker image 'keymanapp/keyman-${platform}-ci:${build_version}' built" diff --git a/resources/docker-images/core/Dockerfile b/resources/docker-images/core/Dockerfile index 54f5b92b667..4a678da31cd 100644 --- a/resources/docker-images/core/Dockerfile +++ b/resources/docker-images/core/Dockerfile @@ -25,15 +25,6 @@ RUN apt-get install -qy python3 python3-setuptools python3-coverage \ rm /tmp/control #### TMP END -# Install NVM -RUN NVM_RELEASE=$(curl -s https://api.github.com/repos/nvm-sh/nvm/releases/latest | grep tag_name | cut -d : -f 2 | cut -d '"' -f 2) && \ - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_RELEASE}/install.sh | bash -RUN <> /usr/bin/bashwrapper -PATH=/home/build/.keyman/node:\$PATH -export NVM_DIR="$HOME/.nvm" -. /home/build/.nvm/nvm.sh -EOF - # Pre-install emscripten USER build ARG REQUIRED_EMSCRIPTEN_VERSION=1.0 @@ -65,12 +56,8 @@ RUN chmod +x /usr/bin/bashwrapper && \ USER build # Pre-install node -ARG REQUIRED_NODE_VERSION=18 -RUN echo "HOME=\${HOME}; REQUIRED_NODE_VERSION=${REQUIRED_NODE_VERSION}" && \ - export NVM_DIR="/home/build/.nvm" && \ +RUN export NVM_DIR="/home/build/.nvm" && \ . /home/build/.nvm/nvm.sh && \ - nvm install "${REQUIRED_NODE_VERSION}" && \ - nvm use "${REQUIRED_NODE_VERSION}" && \ cd /home/build/emsdk/upstream/emscripten && \ npm install diff --git a/resources/docker-images/web/Dockerfile b/resources/docker-images/web/Dockerfile index 7adbae6106a..f7ad7ee3d95 100644 --- a/resources/docker-images/web/Dockerfile +++ b/resources/docker-images/web/Dockerfile @@ -20,15 +20,6 @@ RUN apt-get install -qy libevent-2.1-7t64 libxslt1.1 libwoff1 libvpx9 \ COPY run-tests.sh /usr/bin/run-tests.sh -# Install NVM -RUN NVM_RELEASE=$(curl -s https://api.github.com/repos/nvm-sh/nvm/releases/latest | grep tag_name | cut -d : -f 2 | cut -d '"' -f 2) && \ - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_RELEASE}/install.sh | bash -RUN <> /usr/bin/bashwrapper -PATH=/home/build/.keyman/node:\$PATH -export NVM_DIR="$HOME/.nvm" -. /home/build/.nvm/nvm.sh -EOF - # Pre-install emscripten USER build ARG REQUIRED_EMSCRIPTEN_VERSION=1.0 @@ -77,12 +68,8 @@ RUN chmod +x /usr/bin/bashwrapper && \ USER build # Pre-install node -ARG REQUIRED_NODE_VERSION=18 -RUN echo "HOME=\${HOME}; REQUIRED_NODE_VERSION=${REQUIRED_NODE_VERSION}" && \ - export NVM_DIR="/home/build/.nvm" && \ +RUN export NVM_DIR="/home/build/.nvm" && \ . /home/build/.nvm/nvm.sh && \ - nvm install "${REQUIRED_NODE_VERSION}" && \ - nvm use "${REQUIRED_NODE_VERSION}" && \ cd /home/build/emsdk/upstream/emscripten && \ npm install From e23e112e7a4bdabe24b115d03d287ec853421b86 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 11 Sep 2024 15:54:20 +0200 Subject: [PATCH 15/22] chore(linux): remove temporary code for Core image --- resources/docker-images/build.sh | 4 ---- resources/docker-images/core/.gitignore | 1 - resources/docker-images/core/Dockerfile | 11 ----------- 3 files changed, 16 deletions(-) delete mode 100644 resources/docker-images/core/.gitignore diff --git a/resources/docker-images/build.sh b/resources/docker-images/build.sh index a3254a2f526..6e81d8a27e0 100755 --- a/resources/docker-images/build.sh +++ b/resources/docker-images/build.sh @@ -74,10 +74,6 @@ build_action() { if [[ "${platform}" == "base" ]]; then docker pull --platform "amd64" "ubuntu:${UBUNTU_VERSION:-${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER}}" - ### TMP code - elif [[ "${platform}" == "core" ]]; then - cp "${KEYMAN_ROOT}/linux/debian/control" "${platform}" - ### TMP END elif [[ "${platform}" == "linux" ]]; then cp "${KEYMAN_ROOT}/linux/debian/control" "${platform}" fi diff --git a/resources/docker-images/core/.gitignore b/resources/docker-images/core/.gitignore deleted file mode 100644 index 4db28ac4954..00000000000 --- a/resources/docker-images/core/.gitignore +++ /dev/null @@ -1 +0,0 @@ -control diff --git a/resources/docker-images/core/Dockerfile b/resources/docker-images/core/Dockerfile index 4a678da31cd..55ca0b1ef2d 100644 --- a/resources/docker-images/core/Dockerfile +++ b/resources/docker-images/core/Dockerfile @@ -14,17 +14,6 @@ USER root RUN apt-get install -qy git jq llvm meson pkgconf \ xvfb xserver-xephyr metacity -#### TMP until we properly figure out the dependencies needed for Core -#### We should not need to install /tmp/control -# Install dependencies -ADD control /tmp/control -RUN apt-get install -qy python3 python3-setuptools python3-coverage \ - devscripts equivs libdatetime-perl meson pkgconf lcov gcovr xvfb \ - xserver-xephyr metacity mutter dbus-x11 weston xwayland && \ - (yes | mk-build-deps --install /tmp/control) || true && \ - rm /tmp/control -#### TMP END - # Pre-install emscripten USER build ARG REQUIRED_EMSCRIPTEN_VERSION=1.0 From e54f8227860d2c65f190cdbbca455cc9ad370c37 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 11 Sep 2024 16:16:18 +0200 Subject: [PATCH 16/22] chore(linux): cleanup Setting permissions is now done in the base image. --- resources/docker-images/android/Dockerfile | 3 --- resources/docker-images/core/Dockerfile | 3 --- resources/docker-images/linux/Dockerfile | 3 --- resources/docker-images/web/Dockerfile | 3 --- 4 files changed, 12 deletions(-) diff --git a/resources/docker-images/android/Dockerfile b/resources/docker-images/android/Dockerfile index 8c6833c0c1d..6ebafb496ae 100644 --- a/resources/docker-images/android/Dockerfile +++ b/resources/docker-images/android/Dockerfile @@ -40,9 +40,6 @@ else fi EOF -RUN chmod +x /usr/bin/bashwrapper && \ - chown -R build:build $HOME - # now, switch to build user USER build diff --git a/resources/docker-images/core/Dockerfile b/resources/docker-images/core/Dockerfile index 55ca0b1ef2d..990143f6e22 100644 --- a/resources/docker-images/core/Dockerfile +++ b/resources/docker-images/core/Dockerfile @@ -38,9 +38,6 @@ else fi EOF -RUN chmod +x /usr/bin/bashwrapper && \ - chown -R build:build $HOME - # now, switch to build user USER build diff --git a/resources/docker-images/linux/Dockerfile b/resources/docker-images/linux/Dockerfile index 134bc1d1a52..3fe18c94b9a 100644 --- a/resources/docker-images/linux/Dockerfile +++ b/resources/docker-images/linux/Dockerfile @@ -43,9 +43,6 @@ else fi EOF -RUN chmod +x /usr/bin/bashwrapper && \ - chown -R build:build $HOME - # now, switch to build user USER build diff --git a/resources/docker-images/web/Dockerfile b/resources/docker-images/web/Dockerfile index f7ad7ee3d95..986212fb11b 100644 --- a/resources/docker-images/web/Dockerfile +++ b/resources/docker-images/web/Dockerfile @@ -61,9 +61,6 @@ else fi EOF -RUN chmod +x /usr/bin/bashwrapper && \ - chown -R build:build $HOME - # now, switch to build user USER build From 7fd16dc93f6294c283d4b45cd761051fa6d8769c Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 8 Jan 2025 12:34:53 +0100 Subject: [PATCH 17/22] chore(linux): improve help for build.sh --- resources/docker-images/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/docker-images/build.sh b/resources/docker-images/build.sh index 6e81d8a27e0..01af657bcdc 100755 --- a/resources/docker-images/build.sh +++ b/resources/docker-images/build.sh @@ -20,8 +20,8 @@ builder_describe \ ":web" \ "--ubuntu-version=UBUNTU_VERSION The Ubuntu version (default: ${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER})" \ "--no-cache Force rebuild of docker images" \ - "build" \ - "test" + "build Build docker images" \ + "test Test the docker images by running configure,build,test for all or the specified platforms" builder_parse "$@" From 93caf9b782a5e4b7fdd91ae73e828abc906d69da Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 8 Jan 2025 12:42:05 +0100 Subject: [PATCH 18/22] chore(linux): change from SIL International to SIL Global --- resources/docker-images/android/Dockerfile | 4 ++-- resources/docker-images/base/Dockerfile | 4 ++-- resources/docker-images/core/Dockerfile | 4 ++-- resources/docker-images/linux/Dockerfile | 4 ++-- resources/docker-images/web/Dockerfile | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/resources/docker-images/android/Dockerfile b/resources/docker-images/android/Dockerfile index 6ebafb496ae..98c76d332b4 100644 --- a/resources/docker-images/android/Dockerfile +++ b/resources/docker-images/android/Dockerfile @@ -1,8 +1,8 @@ -# Copyright (c) 2024 SIL International. All rights reserved. +# Copyright (c) 2024 SIL Global. All rights reserved. ARG BASE_VERSION=default FROM keymanapp/keyman-base-ci:${BASE_VERSION} -LABEL org.opencontainers.image.authors="SIL International." +LABEL org.opencontainers.image.authors="SIL Global." LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" LABEL org.opencontainers.image.title="Keyman Android Build Image" diff --git a/resources/docker-images/base/Dockerfile b/resources/docker-images/base/Dockerfile index 14bc0b36ccb..fb3bf1dfc9c 100644 --- a/resources/docker-images/base/Dockerfile +++ b/resources/docker-images/base/Dockerfile @@ -1,9 +1,9 @@ -# Copyright (c) 2024 SIL International. All rights reserved. +# Copyright (c) 2024 SIL Global. All rights reserved. ARG UBUNTU_VERSION=latest FROM ubuntu:${UBUNTU_VERSION} -LABEL org.opencontainers.image.authors="SIL International." +LABEL org.opencontainers.image.authors="SIL Global." LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" LABEL org.opencontainers.image.title="Keyman Build Base Image" diff --git a/resources/docker-images/core/Dockerfile b/resources/docker-images/core/Dockerfile index 990143f6e22..a3bdd93f9b0 100644 --- a/resources/docker-images/core/Dockerfile +++ b/resources/docker-images/core/Dockerfile @@ -1,4 +1,4 @@ -# Copyright (c) 2024 SIL International. All rights reserved. +# Copyright (c) 2024 SIL Global. All rights reserved. # ARGS used in this file: # - ARG BASE_VERSION=default # - ARG REQUIRED_NODE_VERSION=18 @@ -6,7 +6,7 @@ ARG BASE_VERSION=default FROM keymanapp/keyman-base-ci:${BASE_VERSION} -LABEL org.opencontainers.image.authors="SIL International." +LABEL org.opencontainers.image.authors="SIL Global." LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" LABEL org.opencontainers.image.title="Keyman Core Build Image" diff --git a/resources/docker-images/linux/Dockerfile b/resources/docker-images/linux/Dockerfile index 3fe18c94b9a..8518a64cc24 100644 --- a/resources/docker-images/linux/Dockerfile +++ b/resources/docker-images/linux/Dockerfile @@ -1,8 +1,8 @@ -# Copyright (c) 2024 SIL International. All rights reserved. +# Copyright (c) 2024 SIL Global. All rights reserved. ARG BASE_VERSION=default FROM keymanapp/keyman-base-ci:${BASE_VERSION} -LABEL org.opencontainers.image.authors="SIL International." +LABEL org.opencontainers.image.authors="SIL Global." LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" LABEL org.opencontainers.image.title="Keyman Linux Build Image" diff --git a/resources/docker-images/web/Dockerfile b/resources/docker-images/web/Dockerfile index 986212fb11b..93b50cb417f 100644 --- a/resources/docker-images/web/Dockerfile +++ b/resources/docker-images/web/Dockerfile @@ -1,4 +1,4 @@ -# Copyright (c) 2024 SIL International. All rights reserved. +# Copyright (c) 2024 SIL Global. All rights reserved. # ARGS used in this file: # - ARG BASE_VERSION=default # - ARG REQUIRED_NODE_VERSION=18 @@ -6,7 +6,7 @@ ARG BASE_VERSION=default FROM keymanapp/keyman-base-ci:${BASE_VERSION} -LABEL org.opencontainers.image.authors="SIL International." +LABEL org.opencontainers.image.authors="SIL Global." LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" LABEL org.opencontainers.image.title="Keyman for Web Build Image" From 9c8ad2e7832916701d4ff6e9c7ca02b9dc30557f Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 8 Jan 2025 17:08:57 +0100 Subject: [PATCH 19/22] fix(core): fix compile error Compiling the tests failed in the container. --- core/tests/unit/km_core_keyboard_api.tests.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/tests/unit/km_core_keyboard_api.tests.cpp b/core/tests/unit/km_core_keyboard_api.tests.cpp index 2898f9e7cd7..06e604c023a 100644 --- a/core/tests/unit/km_core_keyboard_api.tests.cpp +++ b/core/tests/unit/km_core_keyboard_api.tests.cpp @@ -56,10 +56,8 @@ TEST_F(KmCoreKeyboardApiTests, LoadFromBlobNull) { // Setup km::core::path kmxfile = ""; - std::unique_ptr data(new uint8_t[0]); - // Execute - auto status = km_core_keyboard_load_from_blob(kmxfile.stem().c_str(), data.get(), 0, &this->keyboard); + auto status = km_core_keyboard_load_from_blob(kmxfile.stem().c_str(), nullptr, 0, &this->keyboard); // Verify EXPECT_EQ(status, KM_CORE_STATUS_INVALID_ARGUMENT); From d425f2a1a4dccbbec8ad3beaf78095900c8ef9b4 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 8 Jan 2025 18:40:08 +0100 Subject: [PATCH 20/22] chore(linux): use standard header --- resources/docker-images/android/Dockerfile | 2 +- resources/docker-images/base/Dockerfile | 2 +- resources/docker-images/core/Dockerfile | 3 ++- resources/docker-images/linux/Dockerfile | 2 +- resources/docker-images/web/Dockerfile | 3 ++- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/resources/docker-images/android/Dockerfile b/resources/docker-images/android/Dockerfile index 98c76d332b4..e86d496ab84 100644 --- a/resources/docker-images/android/Dockerfile +++ b/resources/docker-images/android/Dockerfile @@ -1,4 +1,4 @@ -# Copyright (c) 2024 SIL Global. All rights reserved. +# Keyman is copyright (C) SIL Global. MIT License. ARG BASE_VERSION=default FROM keymanapp/keyman-base-ci:${BASE_VERSION} diff --git a/resources/docker-images/base/Dockerfile b/resources/docker-images/base/Dockerfile index fb3bf1dfc9c..4c24e89bb09 100644 --- a/resources/docker-images/base/Dockerfile +++ b/resources/docker-images/base/Dockerfile @@ -1,4 +1,4 @@ -# Copyright (c) 2024 SIL Global. All rights reserved. +# Keyman is copyright (C) SIL Global. MIT License. ARG UBUNTU_VERSION=latest FROM ubuntu:${UBUNTU_VERSION} diff --git a/resources/docker-images/core/Dockerfile b/resources/docker-images/core/Dockerfile index a3bdd93f9b0..cefeb6a2ca8 100644 --- a/resources/docker-images/core/Dockerfile +++ b/resources/docker-images/core/Dockerfile @@ -1,4 +1,5 @@ -# Copyright (c) 2024 SIL Global. All rights reserved. +# Keyman is copyright (C) SIL Global. MIT License. +# # ARGS used in this file: # - ARG BASE_VERSION=default # - ARG REQUIRED_NODE_VERSION=18 diff --git a/resources/docker-images/linux/Dockerfile b/resources/docker-images/linux/Dockerfile index 8518a64cc24..3f680efe3ea 100644 --- a/resources/docker-images/linux/Dockerfile +++ b/resources/docker-images/linux/Dockerfile @@ -1,4 +1,4 @@ -# Copyright (c) 2024 SIL Global. All rights reserved. +# Keyman is copyright (C) SIL Global. MIT License. ARG BASE_VERSION=default FROM keymanapp/keyman-base-ci:${BASE_VERSION} diff --git a/resources/docker-images/web/Dockerfile b/resources/docker-images/web/Dockerfile index 93b50cb417f..8c708e6a475 100644 --- a/resources/docker-images/web/Dockerfile +++ b/resources/docker-images/web/Dockerfile @@ -1,4 +1,5 @@ -# Copyright (c) 2024 SIL Global. All rights reserved. +# Keyman is copyright (C) SIL Global. MIT License. +# # ARGS used in this file: # - ARG BASE_VERSION=default # - ARG REQUIRED_NODE_VERSION=18 From 3251168e6b7a5accf6321a2df32fe9ffb6256933 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 8 Jan 2025 18:40:22 +0100 Subject: [PATCH 21/22] chore(linux): address code review comments --- resources/docker-images/core/Dockerfile | 6 +++--- resources/docker-images/web/Dockerfile | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/resources/docker-images/core/Dockerfile b/resources/docker-images/core/Dockerfile index cefeb6a2ca8..924e8f1e390 100644 --- a/resources/docker-images/core/Dockerfile +++ b/resources/docker-images/core/Dockerfile @@ -1,8 +1,8 @@ # Keyman is copyright (C) SIL Global. MIT License. # # ARGS used in this file: -# - ARG BASE_VERSION=default -# - ARG REQUIRED_NODE_VERSION=18 +# - ARG BASE_VERSION +# - ARG REQUIRED_EMSCRIPTEN_VERSION ARG BASE_VERSION=default FROM keymanapp/keyman-base-ci:${BASE_VERSION} @@ -17,7 +17,7 @@ RUN apt-get install -qy git jq llvm meson pkgconf \ # Pre-install emscripten USER build -ARG REQUIRED_EMSCRIPTEN_VERSION=1.0 +ARG REQUIRED_EMSCRIPTEN_VERSION=unset RUN echo "Installing emscripten version ${REQUIRED_EMSCRIPTEN_VERSION}" && \ export EMSDK_KEEP_DOWNLOADS=1 && \ cd /home/build/ && \ diff --git a/resources/docker-images/web/Dockerfile b/resources/docker-images/web/Dockerfile index 8c708e6a475..5a27c9724de 100644 --- a/resources/docker-images/web/Dockerfile +++ b/resources/docker-images/web/Dockerfile @@ -2,7 +2,7 @@ # # ARGS used in this file: # - ARG BASE_VERSION=default -# - ARG REQUIRED_NODE_VERSION=18 +# - ARG REQUIRED_EMSCRIPTEN_VERSION=unset ARG BASE_VERSION=default FROM keymanapp/keyman-base-ci:${BASE_VERSION} @@ -23,7 +23,7 @@ COPY run-tests.sh /usr/bin/run-tests.sh # Pre-install emscripten USER build -ARG REQUIRED_EMSCRIPTEN_VERSION=1.0 +ARG REQUIRED_EMSCRIPTEN_VERSION=unset RUN echo "Installing emscripten version ${REQUIRED_EMSCRIPTEN_VERSION}" && \ export EMSDK_KEEP_DOWNLOADS=1 && \ cd /home/build/ && \ From 5176dd8a0ee5e30f03cf531cebb63c78bf9ba67c Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Thu, 9 Jan 2025 12:07:10 +0100 Subject: [PATCH 22/22] refactor(linux): address code review comments Also some cleanup. --- resources/docker-images/android/Dockerfile | 3 +++ resources/docker-images/base/Dockerfile | 1 + resources/docker-images/build.sh | 12 ++++++++---- resources/docker-images/linux/run-tests.sh | 5 +++++ resources/docker-images/run.sh | 22 +++++++++++----------- resources/docker-images/web/run-tests.sh | 5 +++++ 6 files changed, 33 insertions(+), 15 deletions(-) diff --git a/resources/docker-images/android/Dockerfile b/resources/docker-images/android/Dockerfile index e86d496ab84..91cd095a69d 100644 --- a/resources/docker-images/android/Dockerfile +++ b/resources/docker-images/android/Dockerfile @@ -47,6 +47,9 @@ VOLUME /home/build/build WORKDIR /home/build/build # Pre-install gradle. This will put files in ~/.gradle which will speed up builds. +# Note it would be safer to copy these files directly from our repo rather than +# getting it over the Internet, but Docker doesn't allow us to copy files +# from outside the current directory when building the image. RUN mkdir -p $HOME/tmp/gradle/wrapper && \ # KMEA uses gradle-7.6.4-bin curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradle/wrapper/gradle-wrapper.jar && \ diff --git a/resources/docker-images/base/Dockerfile b/resources/docker-images/base/Dockerfile index 4c24e89bb09..87516b3f966 100644 --- a/resources/docker-images/base/Dockerfile +++ b/resources/docker-images/base/Dockerfile @@ -32,6 +32,7 @@ RUN echo "build ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers RUN < /usr/bin/bashwrapper #!/bin/bash export KEYMAN_USE_NVM=1 +export DOCKER_RUNNING=true EOF # Install NVM diff --git a/resources/docker-images/build.sh b/resources/docker-images/build.sh index 01af657bcdc..8100459d472 100755 --- a/resources/docker-images/build.sh +++ b/resources/docker-images/build.sh @@ -49,7 +49,9 @@ _add_build_args() { _convert_parameters_to_build_args() { build_args=() build_version= - local required_node_version="$(_print_expected_node_version)" + local required_node_version + # shellcheck disable=SC2034 + required_node_version="$(_print_expected_node_version)" _add_build_args UBUNTU_VERSION KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER "" _add_build_args JAVA_VERSION KEYMAN_VERSION_JAVA java @@ -82,8 +84,9 @@ build_action() { OPTION_NO_CACHE="--no-cache" fi + # shellcheck disable=SC2164 cd "${platform}" - # shellcheck disable=SC2248 + # shellcheck disable=SC2248,SC2086 docker build ${OPTION_NO_CACHE:-} --platform amd64 -t "keymanapp/keyman-${platform}-ci:${build_version}" "${build_args[@]}" . # If the user didn't specify particular versions we will additionaly create an image # with the tag 'default'. @@ -91,7 +94,8 @@ build_action() { builder_echo debug "Setting default tag for ${platform}" docker build --platform amd64 -t "keymanapp/keyman-${platform}-ci:default" "${build_args[@]}" . fi - cd - || true + # shellcheck disable=SC2164,SC2103 + cd - builder_echo success "Docker image 'keymanapp/keyman-${platform}-ci:${build_version}' built" } @@ -99,7 +103,7 @@ test_action() { local platform=$1 builder_echo debug "Testing image for ${platform}" - ./run.sh ${platform} -- ./build.sh configure,build,test:${platform} + ./run.sh "${platform}" -- ./build.sh configure,build,test:"${platform}" } if builder_has_action build; then diff --git a/resources/docker-images/linux/run-tests.sh b/resources/docker-images/linux/run-tests.sh index 962418467ef..fbc5fb3b67b 100755 --- a/resources/docker-images/linux/run-tests.sh +++ b/resources/docker-images/linux/run-tests.sh @@ -1,6 +1,11 @@ #!/usr/bin/env bash set -e +if [[ -z "${DOCKER_RUNNING:-}" ]]; then + echo "This script is intended to be run inside a docker container." + exit 0 +fi + # Start system dbus sudo dbus-daemon --system --fork diff --git a/resources/docker-images/run.sh b/resources/docker-images/run.sh index c85c15e873e..c42d6c2a028 100755 --- a/resources/docker-images/run.sh +++ b/resources/docker-images/run.sh @@ -21,37 +21,37 @@ builder_describe \ builder_parse "$@" run_android() { - docker run -it --rm -v ${KEYMAN_ROOT}:/home/build/build \ - -v ${KEYMAN_ROOT}/core/build/docker-core:/home/build/build/core/build \ + docker run -it --rm -v "${KEYMAN_ROOT}":/home/build/build \ + -v "${KEYMAN_ROOT}/core/build/docker-core":/home/build/build/core/build \ keymanapp/keyman-android-ci:default \ "${builder_extra_params[@]}" } run_core() { - docker run -it --rm -v ${KEYMAN_ROOT}:/home/build/build \ - -v ${KEYMAN_ROOT}/core/build/docker-core:/home/build/build/core/build \ + docker run -it --rm -v "${KEYMAN_ROOT}":/home/build/build \ + -v "${KEYMAN_ROOT}/core/build/docker-core":/home/build/build/core/build \ keymanapp/keyman-core-ci:default \ "${builder_extra_params[@]}" } run_linux() { - mkdir -p ${KEYMAN_ROOT}/linux/build/docker-linux - docker run -it --privileged --rm -v ${KEYMAN_ROOT}:/home/build/build \ - -v ${KEYMAN_ROOT}/core/build/docker-core:/home/build/build/core/build \ - -v ${KEYMAN_ROOT}/linux/build/docker-linux:/home/build/build/linux/build \ + mkdir -p "${KEYMAN_ROOT}/linux/build/docker-linux" + docker run -it --privileged --rm -v "${KEYMAN_ROOT}":/home/build/build \ + -v "${KEYMAN_ROOT}/core/build/docker-core":/home/build/build/core/build \ + -v "${KEYMAN_ROOT}/linux/build/docker-linux":/home/build/build/linux/build \ -e DESTDIR=/tmp \ keymanapp/keyman-linux-ci:default \ "${builder_extra_params[@]}" } run_web() { - docker run -it --privileged --rm -v ${KEYMAN_ROOT}:/home/build/build \ - -v ${KEYMAN_ROOT}/core/build/docker-core:/home/build/build/core/build \ + docker run -it --privileged --rm -v "${KEYMAN_ROOT}":/home/build/build \ + -v "${KEYMAN_ROOT}/core/build/docker-core":/home/build/build/core/build \ keymanapp/keyman-web-ci:default \ "${builder_extra_params[@]}" } -mkdir -p ${KEYMAN_ROOT}/core/build/docker-core +mkdir -p "${KEYMAN_ROOT}/core/build/docker-core" builder_run_action android run_android builder_run_action core run_core diff --git a/resources/docker-images/web/run-tests.sh b/resources/docker-images/web/run-tests.sh index 1a3100159f8..84871d8b19f 100755 --- a/resources/docker-images/web/run-tests.sh +++ b/resources/docker-images/web/run-tests.sh @@ -1,4 +1,9 @@ #!/usr/bin/env bash +if [[ -z "${DOCKER_RUNNING:-}" ]]; then + echo "This script is intended to be run inside a docker container." + exit 0 +fi + set -e echo "Starting Xvfb..." Xvfb -screen 0 1024x768x24 :33 &> /dev/null &