diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml index 7ef2fda0ba..e2f427c066 100644 --- a/.github/workflows/binaries.yml +++ b/.github/workflows/binaries.yml @@ -1,7 +1,8 @@ -name: Passenger generic binaries tests CI tests +name: Passenger generic binaries CI env: - ENTERPRISE: 0 + FORCE_COLOR: 1 + RUST_LOG_STYLE: always on: push: {} @@ -9,59 +10,211 @@ on: jobs: build_linux: - name: "Binary automation Linux-${{ matrix.arch }}" + name: Binaries Linux ${{ matrix.arch }} strategy: fail-fast: false matrix: include: - - arch: x86_64 - runner: ubuntu-24.04 - - arch: aarch64 - runner: passenger-ubuntu-24.04-arm64-2cpu + - runner: ubuntu-24.04 + arch: x86_64 + docker_arch: amd64 + - runner: passenger-ubuntu-24.04-arm64-4cpu + arch: aarch64 + docker_arch: arm64 runs-on: ${{ matrix.runner }} - env: - WORKSPACE: ${{ github.workspace }} - OUTPUT_DIR: ${{ github.workspace }}/output-linux-${{ matrix.arch }} - ARCHITECTURE: ${{ matrix.arch }} - CACHE_DIR: ${{ github.workspace }}/cache/linux-${{ matrix.arch }}/executor-${{ github.run_id }} - RUNTIME_DIR: ${{ github.workspace }}/cache/linux-${{ matrix.arch }}/executor-${{ github.run_id }}/runtime steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: docker/setup-buildx-action@v3 - - run: ./dev/ci/tests/binaries/build-linux - - uses: actions/upload-artifact@v4 - with: - name: binaries-linux-${{ matrix.arch }} - path: 'output-linux-${{ matrix.arch }}/**/*' - - run: ./dev/ci/tests/binaries/test-linux + - uses: actions/checkout@v4 + with: + submodules: true + + - run: mkdir cache output + working-directory: packaging/binaries/linux + + - name: Fetch cache + run: ./dev/ci/fetch-cache-az-blob-storage + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + BLOB_NAME: "binaries/linux-binaries-cache-${{ hashFiles('packaging/binaries/linux/docker_image/Dockerfile') }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd" + CACHE_PATH: packaging/binaries/linux/cache + + - name: Build + run: > + ./build + -p "$WORKSPACE" + -c cache + -o output + -A "$DOCKER_ARCH" + -j 2 + passenger nginx + env: + WORKSPACE: ${{ github.workspace }} + DOCKER_ARCH: ${{ matrix.docker_arch }} + working-directory: packaging/binaries/linux + + - uses: actions/upload-artifact@v4 + with: + name: binaries-linux-${{ matrix.arch }} + path: packages/binaries/linux/output/**/* + + - name: Update cache + run: ./dev/ci/update-cache-az-blob-storage + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + BLOB_NAME: "binaries/linux-binaries-cache-${{ hashFiles('packaging/binaries/linux/docker_image/Dockerfile') }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd" + CACHE_PATH: packaging/binaries/linux/cache + if: '!cancelled()' + + - name: Package + run: ./package -i output -o output -a "$ARCHITECTURE" + env: + ARCHITECTURE: ${{ matrix.arch }} + working-directory: packaging/binaries/linux + + - name: Test + run: > + ./test + -p "$WORKSPACE" + -i output + -I output + -A "$DOCKER_ARCH" + env: + WORKSPACE: ${{ github.workspace }} + DOCKER_ARCH: ${{ matrix.docker_arch }} + working-directory: packaging/binaries/linux build_macos: - name: "Binary automation macOS-${{ matrix.arch }}" + name: Binaries macOS ${{ matrix.arch }} strategy: fail-fast: false matrix: - arch: - - x86_64 - - aarch64 - runs-on: macos-${{ matrix.arch == 'x86_64' && '13' || 'latest' }} + include: + - runner: macos-13 + arch: x86_64 + sccache_arch_and_os: x86_64-apple-darwin + sccache_azure_key_prefix: sccache/cxx-macos-13-x86_64 + - runner: macos-14 + arch: arm64 + sccache_arch_and_os: aarch64-apple-darwin + sccache_azure_key_prefix: sccache/cxx-macos-14-arm + runs-on: ${{ matrix.runner }} env: - WORKSPACE: ${{ github.workspace }} - OUTPUT_DIR: ${{ github.workspace }}/output-macos-${{ matrix.arch }} - ARCHITECTURE: ${{ matrix.arch }} - CACHE_DIR: ${{ github.workspace }}/cache/macos-${{ matrix.arch }}/executor-${{ github.run_id }} - RUNTIME_DIR: ${{ github.workspace }}/cache/macos-${{ matrix.arch }}/executor-${{ github.run_id }}/runtime + CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + SCCACHE_AZURE_BLOB_CONTAINER: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + SCCACHE_AZURE_KEY_PREFIX: ${{ matrix.sccache_azure_key_prefix }} steps: - - uses: actions/checkout@v4 - with: - submodules: true - - run: \curl -sSL https://get.rvm.io | bash - - run: xargs -I{} "$HOME/.rvm/bin/rvm" install ruby-{} < packaging/binaries/shared/definitions/ruby_versions - - run: ./dev/ci/tests/binaries/prepare-macos - - run: ./dev/ci/tests/binaries/build-macos - - uses: actions/upload-artifact@v4 - with: - name: binaries-macos-${{ matrix.arch }} - path: 'output-macos-${{ matrix.arch }}/**/*' - - run: ./dev/ci/tests/binaries/test-macos + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Install dependencies + run: brew install zlib automake libtool coreutils + + - run: mkdir work runtime-output output + working-directory: packaging/binaries/macos + + - name: Determine runtime version + id: determine_runtime_version + run: | + RUNTIME_VERSION=$(cat ../shared/definitions/macos_runtime_version) + echo "version=$RUNTIME_VERSION" >> "$GITHUB_OUTPUT" + working-directory: packaging/binaries/macos + + + - name: Fetch RVM cache + run: CACHE_PATH="$HOME/.rvm" ./dev/ci/fetch-cache-az-blob-storage + id: fetch_rvm_cache + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + BLOB_NAME: "binaries/rvm-${{ hashFiles('packaging/binaries/shared/definitions/ruby_versions') }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd" + + - name: Install RVM + run: set -o pipefail && curl -fsSL https://get.rvm.io/stable | bash + + - name: Install Rubies + run: xargs -I{} "$HOME/.rvm/bin/rvm" install ruby-{} < packaging/binaries/shared/definitions/ruby_versions + if: steps.fetch_rvm_cache.outputs.cache-hit != 'true' + + - name: Cleanup RVM + run: ~/.rvm/bin/rvm cleanup all + if: steps.fetch_rvm_cache.outputs.cache-hit != 'true' + + - name: Update RVM cache + run: CACHE_PATH="$HOME/.rvm" ./dev/ci/update-cache-az-blob-storage + if: steps.fetch_rvm_cache.outputs.cache-hit != 'true' + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + BLOB_NAME: "binaries/rvm-${{ hashFiles('packaging/binaries/shared/definitions/ruby_versions') }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd" + + + - name: Setup sccache + run: ./dev/ci/setup-sccache + timeout-minutes: 1 + env: + ARCH_AND_OS: ${{ matrix.sccache_arch_and_os }} + SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + + + - name: Fetch runtime cache + run: ./dev/ci/fetch-cache-az-blob-storage + id: fetch_runtime_cache + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + BLOB_NAME: binaries/macos-runtime-${{ steps.determine_runtime_version.outputs.version }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd + CACHE_PATH: packaging/binaries/macos/runtime-output + + - name: Compile runtime + run: ./setup-runtime -w work -o runtime-output -j 4 + if: steps.fetch_runtime_cache.outputs.cache-hit != 'true' + working-directory: packaging/binaries/macos + + - name: Update runtime cache + run: ./dev/ci/update-cache-az-blob-storage + if: steps.fetch_runtime_cache.outputs.cache-hit != 'true' + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + BLOB_NAME: binaries/macos-runtime-${{ steps.determine_runtime_version.outputs.version }}-${{ matrix.runner }}-${{ matrix.arch }}.tar.zstd + CACHE_PATH: packaging/binaries/macos/runtime-output + + + - name: Build binaries + run: ./build -p "$WORKSPACE" -r runtime-output -w work -o output -j 4 passenger nginx + env: + WORKSPACE: ${{ github.workspace }} + working-directory: packaging/binaries/macos + + - uses: actions/upload-artifact@v4 + with: + name: binaries-macos-${{ matrix.arch }} + path: packaging/binaries/macos/output/**/* + + + - name: Package binaries + run: ./package -i output -o output + working-directory: packaging/binaries/macos + + - name: Cache test suite gem bundle + uses: actions/cache@v4 + with: + path: packaging/binaries/macos/runtime-output/gems + key: test-gems-${{ hashFiles('packaging/binaries/shared/Gemfile.lock', 'packaging/binaries/shared/definitions/ruby_versions') }}-${{ matrix.runner }}-${{ matrix.arch }} + + - name: Test + run: ./test -p "$WORKSPACE" -r runtime-output -i output -I output + env: + WORKSPACE: ${{ github.workspace }} + working-directory: packaging/binaries/macos + + + - name: Teardown sccache + run: ./dev/ci/teardown-sccache + if: '!cancelled()' + + - name: Archive configure logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: generic-binaries-logs-macos + path: packaging/binaries/macos/work/*/config.log + if-no-files-found: ignore diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 2c3f442308..f9e63a5875 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -1,17 +1,12 @@ name: Passenger Debian packaging tests env: - ENTERPRISE: 0 + FORCE_COLOR: 1 + RUST_LOG_STYLE: always on: - push: - branches: - - 'stable-*' - - 'feature/*' - pull_request: - branches: - - 'stable-*' - - 'feature/*' + push: {} + pull_request: {} jobs: define-matrix: @@ -27,29 +22,73 @@ jobs: run: echo "distros=[$(awk -F= '/DEFAULT_DISTROS/{print $2}' packaging/debian/internal/lib/distro_info.sh | sed -e 's/ /", "/g')]" >> "$GITHUB_OUTPUT" test: - name: "Test ${{ matrix.distro }} ${{ matrix.arch }} packages" + name: ${{ matrix.distro }} ${{ matrix.arch.name }} needs: define-matrix - runs-on: ubuntu-latest strategy: fail-fast: false matrix: distro: ${{ fromJSON(needs.define-matrix.outputs.distros) }} arch: - - amd64 - - arm64 + - name: amd64 + runner: ubuntu-24.04 + - name: arm64 + runner: passenger-ubuntu-24.04-arm64-4cpu + runs-on: ${{ matrix.arch.runner }} env: WORKSPACE: ${{ github.workspace }} - ARCHITECTURE: ${{ matrix.arch }} - CACHE_DIR: ${{ github.workspace }}/cache/debian-test/${{ matrix.distro }}-${{ matrix.arch }} + ARCHITECTURE: ${{ matrix.arch.name }} + CACHE_DIR: ${{ github.workspace }}/cache/debian-test/${{ matrix.distro }}-${{ matrix.arch.name }} DISTRIBUTION: ${{ matrix.distro }} steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - run: ./dev/ci/tests/debian/run - - uses: actions/upload-artifact@v4 - with: - name: debian-${{ matrix.distro }}-${{ matrix.arch }} - path: 'output/${{ matrix.distro }}/*' + - uses: actions/checkout@v4 + with: + submodules: true + + - run: mkdir work cache output + + - name: Fetch cache + run: ./dev/ci/fetch-cache-az-blob-storage + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + BLOB_NAME: "binaries/debian-cache-${{ matrix.distro }}-${{ matrix.arch.name }}.tar.zstd" + CACHE_PATH: packaging/debian/cache + SUDO: true + + - name: Build + run: ./build -w work -c cache -o output -p "$WORKSPACE" -d "$DISTRIBUTION" -a "$ARCHITECTURE" -R pkg:all + env: + WORKSPACE: ${{ github.workspace }} + DISTRIBUTION: ${{ matrix.distro }} + ARCHITECTURE: ${{ matrix.arch.name }} + working-directory: packaging/debian + + - name: Test + run: > + ./test + -p "$WORKSPACE" + -d "output/$DISTRIBUTION" + -c cache + -x "$DISTRIBUTION" + -a "$ARCHITECTURE" + -j + env: + WORKSPACE: ${{ github.workspace }} + DISTRIBUTION: ${{ matrix.distro }} + ARCHITECTURE: ${{ matrix.arch.name }} + working-directory: packaging/debian + + - name: Update cache + run: ./dev/ci/update-cache-az-blob-storage + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + BLOB_NAME: "binaries/debian-cache-${{ matrix.distro }}-${{ matrix.arch.name }}.tar.zstd" + CACHE_PATH: packaging/debian/cache + SUDO: true + if: '!cancelled()' + + - uses: actions/upload-artifact@v4 + with: + name: debian-${{ matrix.distro }}-${{ matrix.arch.name }} + path: packaging/debian/output/${{ matrix.distro }}/* diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b52d0ae2d6..a836e69c6c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,152 +1,661 @@ name: Passenger CI tests env: - DEFAULT_RUBY_VERSION: 3.1.6 - COMPILE_CONCURRENCY: 4 + FORCE_COLOR: 1 + RUST_LOG_STYLE: always + DEFAULT_RUBY_VERSION: 3.1 + BUNDLE_CLEAN: true + USE_ASAN: true on: push: {} pull_request: {} jobs: - cxx: - name: "C++ unit tests on ${{ matrix.os }}, as ${{ matrix.user }}" + cxx-linux: + name: C++ tests on Linux + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + EXTRA_CFLAGS: '-fdiagnostics-color' + EXTRA_CXXFLAGS: '-fdiagnostics-color' + SCCACHE_AZURE_BLOB_CONTAINER: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + SCCACHE_AZURE_KEY_PREFIX: sccache/cxx-ubuntu + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ env.DEFAULT_RUBY_VERSION }} + bundler-cache: true + + - name: Setup bundle gem path + run: ./dev/ci/setup-bundle-gem-path + + - name: Setup sccache + run: ./dev/ci/setup-sccache + timeout-minutes: 1 + env: + ARCH_AND_OS: x86_64-unknown-linux-musl + SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + + - name: Setup library dependencies + run: > + sudo apt update && + sudo apt install -y libcurl4-openssl-dev + + - name: Setup misc + run: | + set -x + sudo chmod 755 "$HOME" + cp test/config.json.github-ci-linux test/config.json + + - name: Build C++ tests + run: bundle exec drake -j4 test:cxx:build + + - name: Run C++ tests + run: > + sudo env + PATH="$PATH" + GEM_PATH="$GEM_PATH" + ../buildout/test/cxx/main + working-directory: test + + - name: Archive logs + uses: actions/upload-artifact@v4 + with: + name: cxx-test-logs-macos + path: 'buildout/testlogs/*' + + - name: Teardown sccache + run: ./dev/ci/teardown-sccache + if: '!cancelled()' + + cxx-macos: + name: C++ tests on macOS + runs-on: macos-14 + timeout-minutes: 30 + env: + EXTRA_CFLAGS: '-I/opt/homebrew/opt/openssl@3/include -fcolor-diagnostics' + EXTRA_CXXFLAGS: '-I/opt/homebrew/opt/openssl@3/include -fcolor-diagnostics' + SCCACHE_AZURE_BLOB_CONTAINER: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + SCCACHE_AZURE_KEY_PREFIX: sccache/cxx-macos-14-arm + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ env.DEFAULT_RUBY_VERSION }} + bundler-cache: true + + - name: Setup bundle gem path + run: ./dev/ci/setup-bundle-gem-path + + - name: Setup sccache + run: ./dev/ci/setup-sccache + timeout-minutes: 1 + env: + ARCH_AND_OS: aarch64-apple-darwin + SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + + - name: Setup misc + run: | + sudo chmod 755 "$HOME" + sed "s|_USER_|$(whoami)|" test/config.json.github-ci-macos > test/config.json + + - name: Build C++ tests + run: bundle exec drake -j4 test:cxx:build + + - name: Run C++ tests + # The AbortHandler doesn't work so well on Github's macOS runners: + # - crash-watch invocation freezes the runner, so we disable this. + # - resuming the parent process after SIGSTOP doesn't work, so we force kill the parent. + run: > + sudo env + PATH="$PATH" + GEM_PATH="$GEM_PATH" + PASSENGER_DUMP_WITH_CRASH_WATCH=false + PASSENGER_FORCE_TERMINATE_ON_ABORT=true + ../buildout/test/cxx/main + working-directory: test + # The AbortHandler may not be able to forcefully kill the parent. We set a low timeout + # to prevent the test from hanging for a long time. + timeout-minutes: 10 + + - name: Archive logs + uses: actions/upload-artifact@v4 + with: + name: cxx-test-logs-macos + path: 'buildout/testlogs/*' + + - name: Teardown sccache + run: ./dev/ci/teardown-sccache + if: '!cancelled()' + + apache2: + name: "Apache2 tests on ${{ matrix.name }}" + strategy: + fail-fast: false + matrix: + include: + - name: Linux + os: ubuntu-24.04 + config_file: test/config.json.github-ci-linux + extra_cflags: '-fdiagnostics-color' + address_sanitizer: true + sccache_arch_and_os: x86_64-unknown-linux-musl + sccache_azure_key_prefix: sccache/cxx-ubuntu-24.04-x86_64 + - name: macOS + os: macos-14 + config_file: test/config.json.github-ci-macos + extra_cflags: '-I/opt/homebrew/opt/openssl@3/include -fcolor-diagnostics' + # Using AddressSanitizer on dynamically loaded modules (like Apache modules) + # requires loading Apache with the AddressSanitizer runtime using DYLD_INSERT_LIBRARIES. + # That in turn requires disabling SIP, which is not possible on Github Actions. + address_sanitizer: false + sccache_arch_and_os: aarch64-apple-darwin + sccache_azure_key_prefix: sccache/cxx-macos-14-arm + runs-on: ${{ matrix.os }} + env: + EXTRA_CFLAGS: ${{ matrix.extra_cflags }} + EXTRA_CXXFLAGS: ${{ matrix.extra_cflags }} + USE_ASAN: ${{ matrix.address_sanitizer }} + SCCACHE_AZURE_BLOB_CONTAINER: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + SCCACHE_AZURE_KEY_PREFIX: ${{ matrix.sccache_azure_key_prefix }} + SCCACHE_CACHE_MULTIARCH: true + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ env.DEFAULT_RUBY_VERSION }} + bundler-cache: true + + - uses: actions/setup-node@v4 + with: + node-version: 18 + cache: npm + + - name: Setup bundle gem path + run: ./dev/ci/setup-bundle-gem-path + + - name: Setup misc + run: | + sudo chmod 755 "$HOME" + sed "s|_USER_|$(whoami)|; s|_SOURCE_ROOT_|$(pwd)|" ${{ matrix.config_file }} > test/config.json + + - name: Setup Linux dependencies + run: > + sudo apt update && + sudo apt install -y libcurl4-openssl-dev apache2-dev libapr1-dev libaprutil1-dev + if: matrix.name == 'Linux' + + - name: Setup macOS code signing + run: | + exec 2>&1 + set -x + + echo "$OPENSSL_CONFIG" > cert_config.cnf + openssl req -new -newkey rsa:2048 -nodes -keyout github-ci.key -out github-ci.csr -config cert_config.cnf + openssl x509 -req -in github-ci.csr -signkey github-ci.key -out github-ci.crt -days 3650 -extensions req_ext -extfile cert_config.cnf + + sudo security authorizationdb write com.apple.trust-settings.admin allow + security create-keychain -p mypassword mykeychain.keychain + security set-keychain-settings -lut 21600 mykeychain.keychain + security unlock-keychain -p mypassword mykeychain.keychain + + security import github-ci.key -k mykeychain.keychain -A + security set-key-partition-list -S apple-tool:,apple: -k mypassword mykeychain.keychain + security add-trusted-cert -d -r trustRoot -k mykeychain.keychain github-ci.crt + security list-keychain -d user -s mykeychain.keychain + if: matrix.name == 'macOS' + env: + OPENSSL_CONFIG: | + [ req ] + default_bits = 2048 + prompt = no + default_md = sha256 + distinguished_name = req_distinguished_name + req_extensions = req_ext + + [ req_distinguished_name ] + CN = Github CI + + [ req_ext ] + keyUsage = critical, digitalSignature, keyEncipherment + extendedKeyUsage = codeSigning + + - name: Setup hosts entries + run: | + echo 127.0.0.1 passenger.test | sudo tee -a /etc/hosts + echo 127.0.0.1 1.passenger.test 2.passenger.test 3.passenger.test | sudo tee -a /etc/hosts + echo 127.0.0.1 4.passenger.test 5.passenger.test 6.passenger.test | sudo tee -a /etc/hosts + echo 127.0.0.1 7.passenger.test 8.passenger.test 9.passenger.test | sudo tee -a /etc/hosts + + - run: npm ci + + + - name: Setup sccache + run: ./dev/ci/setup-sccache + timeout-minutes: 1 + env: + ARCH_AND_OS: ${{ matrix.sccache_arch_and_os }} + SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + + - name: Test module installation as normal user + run: ./bin/passenger-install-apache2-module --auto --force-colors + + - name: Teardown sccache + run: ./dev/ci/teardown-sccache + if: '!cancelled()' + + + - name: Setup sccache as root + run: ./dev/ci/setup-sccache + timeout-minutes: 1 + env: + SUDO: true + ARCH_AND_OS: ${{ matrix.sccache_arch_and_os }} + SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + + - name: Test module installation as root + run: sudo -E ./bin/passenger-install-apache2-module --auto --force-colors --no-compile + + - name: Teardown sccache as root + run: ./dev/ci/teardown-sccache + if: '!cancelled()' + env: + SUDO: true + + + - name: Integration tests + run: bundle exec rake test:integration:apache2 + + - name: Archive logs + uses: actions/upload-artifact@v4 + with: + name: apache-test-logs-${{ matrix.os }} + path: 'buildout/testlogs/*' + + nginx: + name: "Nginx tests on ${{ matrix.name }}" + strategy: + fail-fast: false + matrix: + include: + - name: Linux + os: ubuntu-24.04 + config_file: test/config.json.github-ci-linux + extra_cflags: '-fdiagnostics-color' + sccache_arch_and_os: x86_64-unknown-linux-musl + sccache_azure_key_prefix: sccache/cxx-ubuntu-24.04-x86_64 + - name: macOS + os: macos-14 + config_file: test/config.json.github-ci-macos + extra_cflags: '-I/opt/homebrew/opt/openssl@3/include -fcolor-diagnostics' + sccache_arch_and_os: aarch64-apple-darwin + sccache_azure_key_prefix: sccache/cxx-macos-14-arm + runs-on: ${{ matrix.os }} + env: + EXTRA_CFLAGS: ${{ matrix.extra_cflags }} + EXTRA_CXXFLAGS: ${{ matrix.extra_cflags }} + SCCACHE_AZURE_BLOB_CONTAINER: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + SCCACHE_AZURE_KEY_PREFIX: ${{ matrix.sccache_azure_key_prefix }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ env.DEFAULT_RUBY_VERSION }} + bundler-cache: true + + - uses: actions/setup-node@v4 + with: + node-version: 18 + cache: npm + + - name: Setup bundle gem path + run: ./dev/ci/setup-bundle-gem-path + + - name: Setup sccache + run: ./dev/ci/setup-sccache + timeout-minutes: 1 + env: + ARCH_AND_OS: ${{ matrix.sccache_arch_and_os }} + SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + + - name: Setup misc + run: | + sudo chmod 755 "$HOME" + sed "s|_USER_|$(whoami)|" ${{ matrix.config_file }} > test/config.json + + - name: Setup Linux dependencies + run: > + sudo apt update && + sudo apt install -y libcurl4-openssl-dev + if: matrix.name == 'Linux' + + - name: Setup hosts entries + run: | + echo 127.0.0.1 passenger.test | sudo tee -a /etc/hosts + echo 127.0.0.1 1.passenger.test 2.passenger.test 3.passenger.test | sudo tee -a /etc/hosts + echo 127.0.0.1 4.passenger.test 5.passenger.test 6.passenger.test | sudo tee -a /etc/hosts + echo 127.0.0.1 7.passenger.test 8.passenger.test 9.passenger.test | sudo tee -a /etc/hosts + + - run: npm ci + + - name: Test installation + run: ./bin/passenger-install-nginx-module --auto --force-colors --prefix=/tmp/nginx --auto-download + + - name: Integration tests + run: bundle exec rake test:integration:nginx + + - name: Archive logs + uses: actions/upload-artifact@v4 + with: + name: nginx-test-logs-${{ matrix.os }} + path: 'buildout/testlogs/*' + + - name: Teardown sccache + run: ./dev/ci/teardown-sccache + if: '!cancelled()' + + nginx_dynamic: + name: "Nginx dynamic module tests on ${{ matrix.name }}" strategy: fail-fast: false matrix: - user: - - root - - 'normal-user' - os: - - macos-latest - - ubuntu-latest - exclude: - - os: macos-latest - user: 'root' + include: + - name: Linux + os: ubuntu-24.04 + extra_cflags: '-fdiagnostics-color' + sccache_arch_and_os: x86_64-unknown-linux-musl + sccache_azure_key_prefix: sccache/cxx-ubuntu-24.04-x86_64 + - name: macOS + os: macos-14 + extra_cflags: '-I/opt/homebrew/opt/openssl@3/include -fcolor-diagnostics' + sccache_arch_and_os: aarch64-apple-darwin + sccache_azure_key_prefix: sccache/cxx-macos-14-arm runs-on: ${{ matrix.os }} env: - SUDO: ${{ matrix.user == 'root' && 1 || 0 }} + EXTRA_CFLAGS: ${{ matrix.extra_cflags }} + EXTRA_CXXFLAGS: ${{ matrix.extra_cflags }} + SCCACHE_AZURE_BLOB_CONTAINER: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + SCCACHE_AZURE_KEY_PREFIX: ${{ matrix.sccache_azure_key_prefix }} steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ env.DEFAULT_RUBY_VERSION }} - bundler-cache: true - - run: ./dev/ci/setup-host cxx - - run: ./dev/ci/run-tests-with-docker cxx - if: matrix.os == 'ubuntu-latest' - - run: ./dev/ci/run-tests-natively cxx - if: matrix.os == 'macos-latest' - - run: ls -R buildout - - uses: actions/upload-artifact@v4 - with: - name: cxx-${{ matrix.os }}-${{ matrix.user }} - path: 'buildout/testlogs/*' - - integration: - name: '${{ matrix.integration.name }} integration tests on ${{ matrix.os }}' + - uses: actions/checkout@v4 + with: + submodules: true + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ env.DEFAULT_RUBY_VERSION }} + bundler-cache: true + + - name: Setup bundle gem path + run: ./dev/ci/setup-bundle-gem-path + + - name: Setup sccache + run: ./dev/ci/setup-sccache + timeout-minutes: 1 + env: + ARCH_AND_OS: ${{ matrix.sccache_arch_and_os }} + SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + + - name: Setup Linux dependencies + run: > + sudo apt update && + sudo apt install -y libcurl4-openssl-dev + if: matrix.name == 'Linux' + + - name: Setup parameters + run: | + TEST_DYNAMIC_WITH_NGINX_VERSION=$(ruby -r "./src/ruby_supportlib/phusion_passenger.rb" -e 'puts PhusionPassenger::PREFERRED_NGINX_VERSION') + NGINX_ADDON_DIR=$(./bin/passenger-config --nginx-addon-dir) + echo "TEST_DYNAMIC_WITH_NGINX_VERSION=$TEST_DYNAMIC_WITH_NGINX_VERSION" >> "$GITHUB_ENV" + echo "NGINX_ADDON_DIR=$NGINX_ADDON_DIR" >> "$GITHUB_ENV" + + - name: Compile module + run: bundle exec drake -j4 nginx:as_dynamic_module + + - name: Download Nginx source + run: curl -sSLO "https://www.nginx.org/download/nginx-$TEST_DYNAMIC_WITH_NGINX_VERSION.tar.gz" + + - name: Extract Nginx source + run: tar -xzf "nginx-$TEST_DYNAMIC_WITH_NGINX_VERSION.tar.gz" + + - name: Configure Nginx source + run: ./configure --with-cc-opt='-Wno-error' --add-dynamic-module="$NGINX_ADDON_DIR" + working-directory: "nginx-${{ env.TEST_DYNAMIC_WITH_NGINX_VERSION }}" + + - name: Compile Nginx with dynamic module + run: make -j4 + working-directory: "nginx-${{ env.TEST_DYNAMIC_WITH_NGINX_VERSION }}" + + - name: Teardown sccache + run: ./dev/ci/teardown-sccache + if: '!cancelled()' + + standalone: + name: "Passenger Standalone tests on ${{ matrix.name }}" + strategy: + fail-fast: false + matrix: + include: + - name: Linux + os: ubuntu-24.04 + extra_cflags: '-fdiagnostics-color' + sccache_arch_and_os: x86_64-unknown-linux-musl + sccache_azure_key_prefix: sccache/cxx-ubuntu-24.04-x86_64 + - name: macOS + os: macos-14 + extra_cflags: '-I/opt/homebrew/opt/openssl@3/include -fcolor-diagnostics' + sccache_arch_and_os: aarch64-apple-darwin + sccache_azure_key_prefix: sccache/cxx-macos-14-arm + runs-on: ${{ matrix.os }} + env: + EXTRA_CFLAGS: ${{ matrix.extra_cflags }} + EXTRA_CXXFLAGS: ${{ matrix.extra_cflags }} + SCCACHE_AZURE_BLOB_CONTAINER: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + SCCACHE_AZURE_KEY_PREFIX: ${{ matrix.sccache_azure_key_prefix }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ env.DEFAULT_RUBY_VERSION }} + bundler-cache: true + + - name: Setup bundle gem path + run: ./dev/ci/setup-bundle-gem-path + + - name: Setup sccache + run: ./dev/ci/setup-sccache + timeout-minutes: 1 + env: + ARCH_AND_OS: ${{ matrix.sccache_arch_and_os }} + SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + + - name: Setup Linux dependencies + run: > + sudo apt update && + sudo apt install -y libcurl4-openssl-dev + if: matrix.name == 'Linux' + + - name: Compile prerequisites + run: bundle exec drake -j4 nginx + + - name: Integration test + run: bundle exec drake -j4 test:integration:standalone + + - name: Teardown sccache + run: ./dev/ci/teardown-sccache + if: '!cancelled()' + + ruby: + name: "Ruby tests on ${{ matrix.name }}" strategy: fail-fast: false matrix: - os: - - macos-latest - - ubuntu-latest - integration: - - name: 'Apache 2' - label: 'apache2' - - name: 'Nginx dynamic module' - label: 'nginx-dynamic' - - name: 'Nginx' - label: 'nginx' - - name: 'Standalone' - label: 'standalone' + include: + - name: Linux + os: ubuntu-24.04 + config_file: test/config.json.github-ci-linux + extra_cflags: '-fdiagnostics-color' + sccache_arch_and_os: x86_64-unknown-linux-musl + sccache_azure_key_prefix: sccache/cxx-ubuntu-24.04-x86_64 + - name: macOS + os: macos-14 + config_file: test/config.json.github-ci-macos + extra_cflags: '-I/opt/homebrew/opt/openssl@3/include -fcolor-diagnostics' + sccache_arch_and_os: aarch64-apple-darwin + sccache_azure_key_prefix: sccache/cxx-macos-14-arm runs-on: ${{ matrix.os }} + env: + EXTRA_CFLAGS: ${{ matrix.extra_cflags }} + EXTRA_CXXFLAGS: ${{ matrix.extra_cflags }} + SCCACHE_AZURE_BLOB_CONTAINER: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + SCCACHE_AZURE_KEY_PREFIX: ${{ matrix.sccache_azure_key_prefix }} steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ env.DEFAULT_RUBY_VERSION }} - bundler-cache: true - - run: ./dev/ci/setup-host ${{ matrix.integration.label }} - - run: ./dev/ci/run-tests-with-docker ${{ matrix.integration.label }} - if: matrix.os == 'ubuntu-latest' - - run: ./dev/ci/run-tests-natively ${{ matrix.integration.label }} - if: matrix.os == 'macos-latest' - - run: ls -R buildout - - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.integration.label }}-${{ matrix.os }} - path: 'buildout/testlogs/*' - - language: - name: '${{ matrix.lang.name }} unit tests on ${{ matrix.os }}' + - uses: actions/checkout@v4 + with: + submodules: true + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ env.DEFAULT_RUBY_VERSION }} + bundler-cache: true + + - name: Setup bundle gem path + run: ./dev/ci/setup-bundle-gem-path + + - name: Setup sccache + run: ./dev/ci/setup-sccache + timeout-minutes: 1 + env: + ARCH_AND_OS: ${{ matrix.sccache_arch_and_os }} + SCCACHE_AZURE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + + - name: Setup misc + run: | + sudo chmod 755 "$HOME" + sed "s|_USER_|$(whoami)|" ${{ matrix.config_file }} > test/config.json + + - name: Setup Linux dependencies + run: > + sudo apt update && + sudo apt install -y libcurl4-openssl-dev + if: matrix.name == 'Linux' + + - name: Compile + run: bundle exec drake -j4 agent + + - name: Test + run: bundle exec rake test:ruby + + - name: Teardown sccache + run: ./dev/ci/teardown-sccache + if: '!cancelled()' + + nodejs: + name: "Node.js tests on ${{ matrix.name }}" strategy: fail-fast: false matrix: - lang: - - name: 'Node.js' - label: 'nodejs' - - name: 'Ruby' - label: 'ruby' - os: - - macos-latest - - ubuntu-latest + include: + - name: Linux + os: ubuntu-24.04 + - name: macOS + os: macos-14 runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ env.DEFAULT_RUBY_VERSION }} - bundler-cache: true - - run: ./dev/ci/setup-host ${{ matrix.lang.label }} - - run: ./dev/ci/run-tests-with-docker ${{ matrix.lang.label }} - if: matrix.os == 'ubuntu-latest' - - run: ./dev/ci/run-tests-natively ${{ matrix.lang.label }} - if: matrix.os == 'macos-latest' - - run: ls -R buildout - - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.lang.label }}-${{ matrix.os }} - path: 'buildout/testlogs/*' + - uses: actions/checkout@v4 + with: + submodules: true + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ env.DEFAULT_RUBY_VERSION }} + bundler-cache: true + + - uses: actions/setup-node@v4 + with: + node-version: 18 + cache: npm + + - name: Setup bundle gem path + run: ./dev/ci/setup-bundle-gem-path + + - run: npm ci + + - name: Test + run: bundle exec rake test:node homebrew_packaging: - name: 'Homebrew packaging unit tests' - runs-on: macos-latest - env: - GEM_HOME: ${{ github.workspace }}/.gem + name: Homebrew packaging tests + runs-on: macos-14 steps: - - uses: actions/checkout@v4 - with: - submodules: true - - run: ./dev/ci/setup-host homebrew-packaging - - run: ./dev/ci/run-tests-natively homebrew-packaging - - run: ls -R buildout - - uses: actions/upload-artifact@v4 - with: - name: homebrew-${{ matrix.os }} - path: 'buildout/testlogs/*' + - uses: actions/checkout@v4 + with: + submodules: true + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ env.DEFAULT_RUBY_VERSION }} + bundler-cache: true + + - name: Setup bundle gem path + run: ./dev/ci/setup-bundle-gem-path + + - name: Create tarball + run: bundle exec rake package:set_official package:tarball + + - name: Check whether formula is up-to-date + run: > + ./packaging/homebrew/verify-oss-formula-uptodate + -c homebrew-core.git + -r upstream + + - name: Modify formula + run: > + ./packaging/homebrew/modify-formula + --passenger-dir . + --formula packaging/homebrew/Formula/passenger.rb + --tarball pkg/*.tar.gz + --output pkg/passenger.rb + + - name: Test formula + run: > + ./packaging/homebrew/test-formula + -p . + -f pkg/passenger.rb + -t pkg/*.tar.gz source_packaging: - name: 'Source packaging unit tests' - runs-on: ubuntu-latest + name: Source packaging tests + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ env.DEFAULT_RUBY_VERSION }} - bundler-cache: true - - run: ./dev/ci/setup-host source-packaging - - run: ./dev/ci/run-tests-with-docker source-packaging - - run: ls -R buildout - - uses: actions/upload-artifact@v4 - with: - name: source-packaging-${{ matrix.os }} - path: 'buildout/testlogs/*' + - uses: actions/checkout@v4 + with: + submodules: true + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ env.DEFAULT_RUBY_VERSION }} + bundler-cache: true + + - name: Setup bundle gem path + run: ./dev/ci/setup-bundle-gem-path + + - name: Test + run: bundle exec rake test:source_packaging diff --git a/.github/workflows/rpm.yml b/.github/workflows/rpm.yml index 86410bc78e..53d854f21b 100644 --- a/.github/workflows/rpm.yml +++ b/.github/workflows/rpm.yml @@ -1,17 +1,12 @@ name: Passenger RPM packaging tests env: - ENTERPRISE: 0 + FORCE_COLOR: 1 + RUST_LOG_STYLE: always on: - push: - branches: - - 'stable-*' - - 'feature/*' - pull_request: - branches: - - 'stable-*' - - 'feature/*' + push: {} + pull_request: {} jobs: define-matrix: @@ -27,29 +22,84 @@ jobs: run: echo "distros=[$(awk -F= '/DEFAULT_DISTROS/{print $2}' packaging/rpm/internal/lib/distro_info.sh | sed -e 's/ /", "/g')]" >> "$GITHUB_OUTPUT" test: - name: "Test ${{ matrix.distro }} ${{ matrix.arch }} packages" + name: "Test ${{ matrix.distro }} ${{ matrix.arch.rpm_arch }} packages" needs: define-matrix - runs-on: ubuntu-latest strategy: fail-fast: false matrix: distro: ${{ fromJSON(needs.define-matrix.outputs.distros) }} arch: - - x86_64 - - aarch64 - env: - WORKSPACE: ${{ github.workspace }} - ARCHITECTURE: ${{ matrix.arch }} - CACHE_DIR: ${{ github.workspace }}/cache/rpm-test/${{ matrix.distro }}-${{ matrix.arch }} - DISTRIBUTION: ${{ matrix.distro }} + - rpm_arch: x86_64 + docker_arch: amd64 + runner: ubuntu-24.04 + - rpm_arch: aarch64 + docker_arch: arm64 + runner: passenger-ubuntu-24.04-arm64-4cpu + runs-on: ${{ matrix.arch.runner }} steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - run: ./dev/ci/tests/rpm/run - - uses: actions/upload-artifact@v4 - with: - name: rpm-${{ matrix.distro }}-${{ matrix.arch }} - path: 'output/${{ matrix.distro }}/*' + - uses: actions/checkout@v4 + with: + submodules: true + + - run: mkdir work cache output + working-directory: packaging/rpm + + - name: Determine test distribution name + id: determine_test_distro + run: | + source packaging/rpm/internal/lib/distro_info.sh + TEST_DISTRO_NAME=$(el_name_to_distro_name "$DISTRIBUTION") + echo "name=$TEST_DISTRO_NAME" >> "$GITHUB_OUTPUT" + env: + DISTRIBUTION: ${{ matrix.distro }} + + - name: Fetch cache + run: ./dev/ci/fetch-cache-az-blob-storage + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + BLOB_NAME: "binaries/rpm-cache-${{ matrix.distro }}-${{ matrix.arch.rpm_arch }}.tar.zstd" + CACHE_PATH: packaging/rpm/cache + SUDO: true + + - name: Build + run: ./build -w work -c cache -o output -p "$WORKSPACE" -d "$DISTRIBUTION" -a "$RPM_ARCH" -A "$DOCKER_ARCH" -R rpm:all + env: + WORKSPACE: ${{ github.workspace }} + DISTRIBUTION: ${{ matrix.distro }} + RPM_ARCH: ${{ matrix.arch.rpm_arch }} + DOCKER_ARCH: ${{ matrix.arch.docker_arch }} + working-directory: packaging/rpm + + - name: Test + run: > + ./test + -p "$WORKSPACE" + -d "output/$DISTRIBUTION" + -c cache + -x "$TEST_DISTRO_NAME" + -a "$RPM_ARCH" + -A "$DOCKER_ARCH" + -j + env: + WORKSPACE: ${{ github.workspace }} + DISTRIBUTION: ${{ matrix.distro }} + TEST_DISTRO_NAME: ${{ steps.determine_test_distro.outputs.name }} + RPM_ARCH: ${{ matrix.arch.rpm_arch }} + DOCKER_ARCH: ${{ matrix.arch.docker_arch }} + working-directory: packaging/rpm + + - name: Update cache + run: ./dev/ci/update-cache-az-blob-storage + env: + AZURE_STORAGE_CONNECTION_STRING: ${{ secrets.AZURE_CI_STORAGE_CONNECTION_STRING }} + CONTAINER_NAME: ${{ vars.SCCACHE_AZURE_BLOB_CONTAINER }} + BLOB_NAME: "binaries/rpm-cache-${{ matrix.distro }}-${{ matrix.arch.rpm_arch }}.tar.zstd" + CACHE_PATH: packaging/rpm/cache + SUDO: true + if: '!cancelled()' + + - uses: actions/upload-artifact@v4 + with: + name: rpm-${{ matrix.distro }}-${{ matrix.arch.rpm_arch }} + path: 'packaging/rpm/output/${{ matrix.distro }}/*' diff --git a/.vscode/settings.json b/.vscode/settings.json index 33a0b721cb..67c703a4fd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { - "files.exclude": { + "files.trimTrailingWhitespace": true, + "files.exclude": { ".externalToolBuilders": true, ".settings": true, ".cproject": true, diff --git a/Gemfile b/Gemfile index aa62e0c420..f5330a0b36 100644 --- a/Gemfile +++ b/Gemfile @@ -1,26 +1,22 @@ source 'https://rubygems.org/' -require 'rubygems/version.rb' -# Make Bundler handle Ruby compat: https://github.com/rubygems/bundler-features/issues/120 -ruby RUBY_VERSION +ruby '>= 2.5' -group :base do +group :development do gem 'json' gem 'mime-types', '~> 3.5.1' + gem 'drake' gem 'rack' + gem 'rackup', '>= 2.1' gem 'rake' gem 'rspec', '~> 3.12.0' gem 'rspec-collection_matchers' -end - -group :future do gem 'webrick', '~> 1.8.1' - gem 'rackup', '>= 2.1' end if ENV['USER'] == 'camdennarzt' -group :development do - gem 'solargraph' - gem 'gpgme' -end + group :development do + gem 'solargraph' + gem 'gpgme' + end end diff --git a/Gemfile.lock b/Gemfile.lock index 00e0b19a89..1017b9edfa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,37 +1,44 @@ GEM remote: https://rubygems.org/ specs: - diff-lcs (1.5.0) - json (2.5.1) - mime-types (3.5.1) + comp_tree (1.1.3) + diff-lcs (1.5.1) + drake (0.9.2.0.3.1) + comp_tree (>= 1.1.3) + json (2.7.2) + mime-types (3.5.2) mime-types-data (~> 3.2015) - mime-types-data (3.2023.1205) - rack (3.0.10) + mime-types-data (3.2024.1001) + rack (3.1.8) rackup (2.1.0) rack (>= 3) webrick (~> 1.8) - rake (12.3.3) + rake (13.2.1) rspec (3.12.0) rspec-core (~> 3.12.0) rspec-expectations (~> 3.12.0) rspec-mocks (~> 3.12.0) rspec-collection_matchers (1.2.1) rspec-expectations (>= 2.99.0.beta1) - rspec-core (3.12.2) + rspec-core (3.12.3) rspec-support (~> 3.12.0) - rspec-expectations (3.12.3) + rspec-expectations (3.12.4) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.12.0) - rspec-mocks (3.12.6) + rspec-mocks (3.12.7) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.12.0) - rspec-support (3.12.1) - webrick (1.8.1) + rspec-support (3.12.2) + webrick (1.8.2) PLATFORMS + arm64-darwin-22 + arm64-darwin-23 ruby + x86_64-linux DEPENDENCIES + drake json mime-types (~> 3.5.1) rack @@ -41,3 +48,8 @@ DEPENDENCIES rspec-collection_matchers webrick (~> 1.8.1) +RUBY VERSION + ruby 3.3.0p0 + +BUNDLED WITH + 2.5.10 diff --git a/Rakefile b/Rakefile index 81f9e35123..0942daed64 100644 --- a/Rakefile +++ b/Rakefile @@ -28,13 +28,17 @@ $LOAD_PATH.unshift("#{SOURCE_ROOT}/src/ruby_supportlib") if defined?(Bundler) clean_env = nil - if Bundler.method_defined?(:with_unbundled_env) + if Bundler.respond_to?(:with_original_env) + Bundler.with_original_env do + clean_env = ENV.to_hash.dup + end + elsif Bundler.respond_to?(:with_unbundled_env) Bundler.with_unbundled_env do - clean_env = ENV.to_hash + clean_env = ENV.to_hash.dup end else Bundler.with_clean_env do - clean_env = ENV.to_hash + clean_env = ENV.to_hash.dup end end ENV.replace(clean_env) diff --git a/bin/passenger-install-apache2-module b/bin/passenger-install-apache2-module index f0b5a5f729..532e8e3798 100755 --- a/bin/passenger-install-apache2-module +++ b/bin/passenger-install-apache2-module @@ -122,6 +122,8 @@ class Installer < PhusionPassenger::AbstractInstaller end def run_steps + PlatformInfo.verbose = true if @verbose_depcheck + if PhusionPassenger.build_system_dir.nil? # Invariant: PhusionPassenger.custom_packaged? if apache_module_available? @@ -944,6 +946,9 @@ parser = OptionParser.new do |opts| opts.on("--force-colors", "Display colors even if stdout is not a TTY") do options[:colorize] = true end + opts.on("--verbose-depcheck", "Show more dependency checking-related logs") do + options[:verbose_depcheck] = true + end opts.on("--snippet", "Show just the Apache config snippet.") do options[:snippet] = true end diff --git a/build/apache2.rb b/build/apache2.rb index 74674259a0..42843f335d 100644 --- a/build/apache2.rb +++ b/build/apache2.rb @@ -126,7 +126,7 @@ PlatformInfo.apache2_module_cxx_ldflags, PlatformInfo.portability_cxx_ldflags, OPTIMIZE ? '-O' : nil, - USE_ASAN ? "-shared-libasan" : nil + USE_ASAN ? PlatformInfo.address_sanitizer_flags : nil, ].compact ) end diff --git a/build/basics.rb b/build/basics.rb index 04bf33329e..755bfb7a24 100644 --- a/build/basics.rb +++ b/build/basics.rb @@ -27,6 +27,7 @@ rescue LoadError end require 'fileutils' +require 'shellwords' require 'phusion_passenger' PhusionPassenger.locate_directories PhusionPassenger.require_passenger_lib 'constants' diff --git a/build/cxx_tests.rb b/build/cxx_tests.rb index 818b18adea..4602b193fb 100644 --- a/build/cxx_tests.rb +++ b/build/cxx_tests.rb @@ -212,14 +212,16 @@ ) end -dependencies = [ +cxx_test_dependencies = [ TEST_CXX_TARGET, "#{TEST_OUTPUT_DIR}allocate_memory", NATIVE_SUPPORT_TARGET, AGENT_TARGET ].compact +task 'test:cxx:build' => cxx_test_dependencies + desc "Run unit tests for the C++ components" -task 'test:cxx' => dependencies do +task 'test:cxx' => cxx_test_dependencies do args = ENV['GROUPS'].to_s.split(";").map{ |name| "-g #{name}" } if level = string_option('LOG_LEVEL') diff --git a/build/test_basics.rb b/build/test_basics.rb index 90ddaba7e0..3a98b8784f 100644 --- a/build/test_basics.rb +++ b/build/test_basics.rb @@ -66,19 +66,11 @@ sh "#{gem_install} bundler" end - if install_base_deps - unless Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.0.0') || RUBY_PLATFORM =~ /darwin/ - if bundler_too_new? - sh "bundle config set --local without future" - else - bundle_args.concat(["--without", "future"]) - end - end - else + if !install_base_deps if bundler_too_new? - sh "bundle config set --local without 'base future'" + sh "bundle config set --local without 'base'" else - bundle_args.concat(["--without", "base", "future"]) + bundle_args.concat(["--without", "base"]) end end sh "bundle install #{bundle_args.join(' ')} #{ENV['BUNDLE_ARGS']}" diff --git a/dev/ci/fetch-cache-az-blob-storage b/dev/ci/fetch-cache-az-blob-storage new file mode 100755 index 0000000000..b6e5daff5c --- /dev/null +++ b/dev/ci/fetch-cache-az-blob-storage @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -eo pipefail + +if [[ -z "$AZURE_STORAGE_CONNECTION_STRING" ]]; then + echo "AZURE_STORAGE_CONNECTION_STRING is required" + exit 1 +fi + +if [[ -z "$CONTAINER_NAME" ]]; then + echo "CONTAINER_NAME is required" + exit 1 +fi + +if [[ -z "$BLOB_NAME" ]]; then + echo "BLOB_NAME is required" + exit 1 +fi + +if [[ "$SUDO" = true ]]; then + SUDO_COMMAND=(sudo) +else + SUDO_COMMAND=() +fi + +CACHE_PATH=${CACHE_PATH:-$BLOB_NAME} + + +echo "--> Checking whether blob exists" +EXISTS=$( + az storage blob exists \ + --container-name "$CONTAINER_NAME" \ + --name "$BLOB_NAME" \ + --connection-string "$AZURE_STORAGE_CONNECTION_STRING" \ + --output tsv +) +echo "$EXISTS" + +if [[ "$EXISTS" = True ]]; then + echo "--> Downloading and extracting blob" + mkdir -p "$CACHE_PATH" + az storage blob download \ + --container-name "$CONTAINER_NAME" \ + --name "$BLOB_NAME" \ + --no-progress \ + | "${SUDO_COMMAND[@]}" env ZSTD_NBTHREADS=0 tar -C "$CACHE_PATH" -x --zstd -f - + echo "Extracted" + + echo "cache-hit=true" >> "$GITHUB_OUTPUT" +else + echo "cache-hit=false" >> "$GITHUB_OUTPUT" +fi diff --git a/dev/ci/setup-bundle-gem-path b/dev/ci/setup-bundle-gem-path new file mode 100755 index 0000000000..8ca7971aae --- /dev/null +++ b/dev/ci/setup-bundle-gem-path @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby +require 'rubygems' + +File.open(ENV['GITHUB_ENV'], 'a') do |f| + f.puts "GEM_PATH=#{Dir.pwd}/vendor/bundle/#{RUBY_ENGINE}/#{Gem.ruby_api_version}:#{Gem.paths.path.join(':')}" +end diff --git a/dev/ci/setup-sccache b/dev/ci/setup-sccache new file mode 100755 index 0000000000..dac8fe4e49 --- /dev/null +++ b/dev/ci/setup-sccache @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -e + +## Check parameters & set defaults. + +if [[ -z "$ARCH_AND_OS" ]]; then + echo "ARCH_AND_OS is required" + exit 1 +fi + +if [[ -z "$SCCACHE_AZURE_CONNECTION_STRING" ]]; then + echo "SCCACHE_AZURE_CONNECTION_STRING is required" + exit 1 +fi + +SCCACHE_LOG=${SCCACHE_LOG:-info} # set to 'debug' or 'trace' for more verbose logging +SUDO=${SUDO:-false} + + +## Ensure Github Actions print stdout and stderr data in correct order. +exec 2>&1 + + +## Download and install sccache. +if [[ ! -e /usr/local/bin/sccache ]]; then + echo "::group::Download sccache" + wget --output-document sccache.tar.gz "https://github.com/mozilla/sccache/releases/download/v0.8.2/sccache-v0.8.2-$ARCH_AND_OS.tar.gz" + echo "::endgroup::" + set -x + tar xzf sccache.tar.gz + rm sccache.tar.gz + sudo mv sccache*/sccache /usr/local/bin/sccache + rm -rf sccache* +fi + + +## Set up sccache as a compiler wrapper. +set -x +echo "PATH=/usr/local/libexec/sccache:/usr/local/bin:$PATH" >> "$GITHUB_ENV" +sudo mkdir -p /usr/local/libexec/sccache +for PROG in cc c++; do + FULLPATH=$(command -v "$PROG") + echo '#!/bin/sh' > "$PROG" + echo "exec /usr/local/bin/sccache $FULLPATH \"\$@\"" >> "$PROG" + chmod +x "$PROG" + sudo mv "$PROG" /usr/local/libexec/sccache/ +done + + +## Update environment variables. + +HAS_SCCACHE_IN_PATH=$(ruby -e 'puts ENV["PATH"].split(":").include?("/usr/local/bin")') +if ! $HAS_SCCACHE_IN_PATH; then + echo "PATH=/usr/local/bin:$PATH" >> "$GITHUB_ENV" +fi + +HAS_COMPILER_WRAPPERS_IN_PATH=$(ruby -e 'puts ENV["PATH"].split(":").include?("/usr/local/libexec/sccache")') +if $HAS_COMPILER_WRAPPERS_IN_PATH; then + # When starting the sccache server, the compiler wrappers must *not* be in PATH + # so that the sccache server uses the non-wrapped compilers. + PATH=$(ruby -e 'paths = ENV["PATH"].split(":"); paths.delete("/usr/local/libexec/sccache"); puts paths.join(":")') + export PATH +else + echo "PATH=/usr/local/libexec/sccache:$PATH" >> "$GITHUB_ENV" +fi + + +## Start sccache server. + +rm -f sccache.log +export RUST_LOG_STYLE=always +export SCCACHE_LOG +export SCCACHE_IDLE_TIMEOUT=0 +export SCCACHE_ERROR_LOG="$(pwd)/sccache.log" +echo "SCCACHE_ERROR_LOG=$SCCACHE_ERROR_LOG" >> "$GITHUB_ENV" + +if $SUDO; then + sudo -E env SCCACHE_START_SERVER=1 sccache +else + env SCCACHE_START_SERVER=1 sccache +fi + +set +x +echo "Waiting until sccache server is started..." +while [[ ! -e sccache.log ]]; do + sleep 0.1 +done +sleep 0.5 +echo "sccache server is started." diff --git a/dev/ci/teardown-sccache b/dev/ci/teardown-sccache new file mode 100755 index 0000000000..b9ca20c722 --- /dev/null +++ b/dev/ci/teardown-sccache @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -e + +SUDO=${SUDO:-false} + +# Ensure Github Actions print stdout and stderr data in correct order. +exec 2>&1 +set -x + +# teardown-sccache can still be called even when setup-sccache fails, so don't +# assume that sccache is in PATH and also don't assume that the log file exists. + +if [[ -e /usr/local/bin/sccache ]]; then + if $SUDO; then + sudo /usr/local/bin/sccache --stop-server + else + /usr/local/bin/sccache --stop-server + fi +fi + +if test -e sccache.log; then + cat sccache.log; +fi + +# Remove compiler wrappers from PATH. +PATH=$(ruby -e 'paths = ENV["PATH"].split(":"); paths.delete("/usr/local/libexec/sccache"); puts paths.join(":")') +echo "PATH=$PATH" >> "$GITHUB_ENV" diff --git a/dev/ci/tests/binaries/Jenkinsfile b/dev/ci/tests/binaries/Jenkinsfile index a735b34d66..1fa437ce4f 100644 --- a/dev/ci/tests/binaries/Jenkinsfile +++ b/dev/ci/tests/binaries/Jenkinsfile @@ -1,12 +1,13 @@ JOB_NAME_AS_ID = null -def setupLinuxTest(enablerFlag, architecture, block) { +def setupLinuxTest(enablerFlag, architecture, dockerArch, block) { if (enablerFlag) { node("linux && ${architecture}") { withEnv([ "OUTPUT_DIR=${env.WORKSPACE}/output-linux-${architecture}", "CACHE_DIR=${env.JENKINS_HOME}/cache/${env.JOB_NAME_AS_ID}/linux-${architecture}/executor-${env.EXECUTOR_NUMBER}", - "ARCHITECTURE=${architecture}" + "ARCHITECTURE=${architecture}", + "DOCKER_ARCH=${dockerArch}" ], block) } } else { @@ -41,7 +42,7 @@ pipeline { parameters { booleanParam(name: 'LINUX_X86_64', defaultValue: true, description: 'Linux x86_64 binaries') - booleanParam(name: 'LINUX_ARM64', defaultValue: true, description: 'Linux arm64 binaries') + booleanParam(name: 'LINUX_AARCH64', defaultValue: true, description: 'Linux aarch64 binaries') booleanParam(name: 'MACOS', defaultValue: true, description: 'macOS binaries') } @@ -73,15 +74,15 @@ pipeline { script { parallel( 'Linux x86_64': { - setupLinuxTest(params.LINUX_X86_64, 'x86_64') { + setupLinuxTest(params.LINUX_X86_64, 'x86_64', 'amd64') { checkout scm sh './dev/ci/tests/binaries/build-linux' archiveArtifacts artifacts: 'output-linux-x86_64/**/*' sh './dev/ci/tests/binaries/test-linux' } }, - 'Linux arm64': { - setupLinuxTest(params.LINUX_ARM64, 'aarch64') { + 'Linux aarch64': { + setupLinuxTest(params.LINUX_AARCH64, 'aarch64', 'arm64') { checkout scm sh './dev/ci/tests/binaries/build-linux' archiveArtifacts artifacts: 'output-linux-aarch64/**/*' diff --git a/dev/ci/tests/binaries/build-linux b/dev/ci/tests/binaries/build-linux index e9c787ed47..1d1f9d373f 100755 --- a/dev/ci/tests/binaries/build-linux +++ b/dev/ci/tests/binaries/build-linux @@ -6,6 +6,7 @@ # # WORKSPACE # ARCHITECTURE +# DOCKER_ARCH # # Optional environment variables: # @@ -21,6 +22,7 @@ source "./shared/lib/library.sh" require_envvar WORKSPACE "$WORKSPACE" require_envvar ARCHITECTURE "$ARCHITECTURE" +require_envvar DOCKER_ARCH "$DOCKER_ARCH" PASSENGER_ROOT="${PASSENGER_ROOT:-$WORKSPACE}" OUTPUT_DIR="${OUTPUT_DIR:-$WORKSPACE/output}" @@ -33,6 +35,6 @@ run ./linux/build \ -p "$PASSENGER_ROOT" \ -c "$CACHE_DIR" \ -o "$OUTPUT_DIR" \ - -a "$ARCHITECTURE" \ + -A "$DOCKER_ARCH" \ -j 1 \ passenger nginx diff --git a/dev/ci/tests/binaries/test-linux b/dev/ci/tests/binaries/test-linux index 9bea870a66..6a63f68c48 100755 --- a/dev/ci/tests/binaries/test-linux +++ b/dev/ci/tests/binaries/test-linux @@ -6,6 +6,7 @@ # # WORKSPACE # ARCHITECTURE +# DOCKER_ARCH # # Optional environment variables: # @@ -21,6 +22,7 @@ source "./shared/lib/library.sh" require_envvar WORKSPACE "$WORKSPACE" require_envvar ARCHITECTURE "$ARCHITECTURE" +require_envvar DOCKER_ARCH "$DOCKER_ARCH" PASSENGER_ROOT="${PASSENGER_ROOT:-$WORKSPACE}" OUTPUT_DIR="${OUTPUT_DIR:-$WORKSPACE/output}" @@ -41,5 +43,5 @@ run ./linux/test \ -p "$PASSENGER_ROOT" \ -i "$OUTPUT_DIR" \ -I "$OUTPUT_DIR" \ - -a "$ARCHITECTURE" \ + -A "$DOCKER_ARCH" \ "${EXTRA_TEST_PARAMS[@]}" diff --git a/dev/ci/tests/rpm/Jenkinsfile b/dev/ci/tests/rpm/Jenkinsfile index c228f699fc..fc243d7bcf 100644 --- a/dev/ci/tests/rpm/Jenkinsfile +++ b/dev/ci/tests/rpm/Jenkinsfile @@ -1,10 +1,11 @@ -def setupTest(enablerFlag, distribution, architecture, block) { +def setupTest(enablerFlag, distribution, rpmArch, dockerArch, block) { if (enablerFlag) { - node("linux && ${architecture}") { + node("linux && ${rpmArch}") { withEnv([ - "CACHE_DIR=${env.JENKINS_HOME}/cache/${env.JOB_NAME}/${distribution}-${architecture}", + "CACHE_DIR=${env.JENKINS_HOME}/cache/${env.JOB_NAME}/${distribution}-${rpmArch}", "DISTRIBUTION=${distribution}", - "ARCHITECTURE=${architecture}" + "RPM_ARCH=${rpmArch}", + "DOCKER_ARCH=${dockerArch}" ], block) } } else { @@ -48,25 +49,25 @@ pipeline { script { parallel( 'el8 x86_64': { - setupTest(params.EL8, 'el8', 'x86_64') { + setupTest(params.EL8, 'el8', 'x86_64', 'amd64') { checkout scm sh './dev/ci/tests/rpm/run' } }, 'el9 x86_64': { - setupTest(params.EL9, 'el9', 'x86_64') { + setupTest(params.EL9, 'el9', 'x86_64', 'amd64') { checkout scm sh './dev/ci/tests/rpm/run' } }, 'el8 aarch64': { - setupTest(params.EL8, 'el8', 'aarch64') { + setupTest(params.EL8, 'el8', 'aarch64', 'arm64') { checkout scm sh './dev/ci/tests/rpm/run' } }, 'el9 aarch64': { - setupTest(params.EL9, 'el9', 'aarch64') { + setupTest(params.EL9, 'el9', 'aarch64', 'arm64') { checkout scm sh './dev/ci/tests/rpm/run' } diff --git a/dev/ci/tests/rpm/run b/dev/ci/tests/rpm/run index b881cdc96a..724d241ab4 100755 --- a/dev/ci/tests/rpm/run +++ b/dev/ci/tests/rpm/run @@ -29,7 +29,8 @@ source "./internal/lib/distro_info.sh" require_envvar WORKSPACE "$WORKSPACE" require_envvar DISTRIBUTION "$DISTRIBUTION" -require_envvar ARCHITECTURE "$ARCHITECTURE" +require_envvar RPM_ARCH "$RPM_ARCH" +require_envvar DOCKER_ARCH "$DOCKER_ARCH" PASSENGER_ROOT="${PASSENGER_ROOT:-$WORKSPACE}" CACHE_DIR="${CACHE_DIR:-$WORKSPACE/cache}" @@ -51,7 +52,8 @@ run ./build \ -o "$WORKSPACE/output" \ -p "$PASSENGER_ROOT" \ -d "$DISTRIBUTION" \ - -a "$ARCHITECTURE" \ + -a "$RPM_ARCH" \ + -A "$DOCKER_ARCH" \ -R \ rpm:all run ./test \ @@ -59,6 +61,7 @@ run ./test \ -d "$WORKSPACE/output/$DISTRIBUTION" \ -c "$CACHE_DIR" \ -x "$TEST_DISTRO_NAME" \ - -a "$ARCHITECTURE" \ + -a "$RPM_ARCH" \ + -A "$DOCKER_ARCH" \ -j \ $EXTRA_TEST_PARAMS diff --git a/dev/ci/update-cache-az-blob-storage b/dev/ci/update-cache-az-blob-storage new file mode 100755 index 0000000000..93543df1ad --- /dev/null +++ b/dev/ci/update-cache-az-blob-storage @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -eo pipefail + +if [[ -z "$AZURE_STORAGE_CONNECTION_STRING" ]]; then + echo "AZURE_STORAGE_CONNECTION_STRING is required" + exit 1 +fi + +if [[ -z "$CONTAINER_NAME" ]]; then + echo "CONTAINER_NAME is required" + exit 1 +fi + +if [[ -z "$BLOB_NAME" ]]; then + echo "BLOB_NAME is required" + exit 1 +fi + +if [[ "$SUDO" = true ]]; then + SUDO_COMMAND=(sudo) +else + SUDO_COMMAND=() +fi + +CACHE_PATH=${CACHE_PATH:-$BLOB_NAME} + + +echo "--> Packaging cache directory" +"${SUDO_COMMAND[@]}" env ZSTD_NBTHREADS=0 tar -C "$CACHE_PATH" -c --zstd -f "$CACHE_PATH.tar.zstd" . + +echo "--> Uploading blob" +az storage blob upload \ + -c "$CONTAINER_NAME" \ + -n "$BLOB_NAME" \ + -f "$CACHE_PATH.tar.zstd" \ + --overwrite \ + --no-progress +rm "$CACHE_PATH.tar.zstd" diff --git a/dev/configkit-schemas/index.json b/dev/configkit-schemas/index.json index 63396fad9e..282da0c4b5 100644 --- a/dev/configkit-schemas/index.json +++ b/dev/configkit-schemas/index.json @@ -889,8 +889,8 @@ "type" : "string" }, "spawn_dir" : { + "has_default_value" : "dynamic", "read_only" : true, - "required" : true, "type" : "string" }, "standalone_engine" : { diff --git a/packaging/binaries b/packaging/binaries index 258988c3eb..9b4cc8f6cf 160000 --- a/packaging/binaries +++ b/packaging/binaries @@ -1 +1 @@ -Subproject commit 258988c3eb02a80861a5ed3c606c7959857163db +Subproject commit 9b4cc8f6cf7ffcbfe01c72fdc3ffe28d3007860a diff --git a/packaging/rpm b/packaging/rpm index 0f3f49bd32..f4477869a7 160000 --- a/packaging/rpm +++ b/packaging/rpm @@ -1 +1 @@ -Subproject commit 0f3f49bd32134f52295e615c2294a45bf4bb817f +Subproject commit f4477869a7e2c2df9059faeb4eedf33cd3866025 diff --git a/src/agent/Core/ApplicationPool/Process.h b/src/agent/Core/ApplicationPool/Process.h index 7275402d6d..0e5aafd524 100644 --- a/src/agent/Core/ApplicationPool/Process.h +++ b/src/agent/Core/ApplicationPool/Process.h @@ -338,8 +338,9 @@ class Process { } void destroySelf() const { + Context *context = getContext(); this->~Process(); - LockGuard l(getContext()->memoryManagementSyncher); + LockGuard l(context->memoryManagementSyncher); getContext()->processObjectPool.free(const_cast(this)); } diff --git a/src/agent/Core/Config.h b/src/agent/Core/Config.h index 7b23c35c65..af476b55ca 100644 --- a/src/agent/Core/Config.h +++ b/src/agent/Core/Config.h @@ -173,7 +173,7 @@ using namespace std; * single_app_mode_app_start_command string - read_only * single_app_mode_app_type string - read_only * single_app_mode_startup_file string - read_only - * spawn_dir string required read_only + * spawn_dir string - default,read_only * standalone_engine string - default * stat_throttle_rate unsigned integer - default(10) * telemetry_collector_ca_certificate_path string - - @@ -240,6 +240,10 @@ class Schema: public ConfigKit::Schema { return 80; } + static Json::Value getDefaultSpawnDir(const ConfigKit::Store &store) { + return getSystemTempDir(); + } + static Json::Value getDefaultThreads(const ConfigKit::Store &store) { return Json::UInt(boost::thread::hardware_concurrency()); } @@ -488,7 +492,7 @@ class Schema: public ConfigKit::Schema { overrideWithDynamicDefault("default_server_port", UINT_TYPE, OPTIONAL, getDefaultServerPort); add("passenger_root", STRING_TYPE, REQUIRED | READ_ONLY); - add("spawn_dir", STRING_TYPE, REQUIRED | READ_ONLY); + addWithDynamicDefault("spawn_dir", STRING_TYPE, OPTIONAL | READ_ONLY | CACHE_DEFAULT_VALUE, getDefaultSpawnDir); add("config_manifest", OBJECT_TYPE, OPTIONAL | READ_ONLY); add("pid_file", STRING_TYPE, OPTIONAL | READ_ONLY); add("web_server_version", STRING_TYPE, OPTIONAL | READ_ONLY); diff --git a/src/agent/Core/CoreMain.cpp b/src/agent/Core/CoreMain.cpp index 561f5d91eb..426f0af3d8 100644 --- a/src/agent/Core/CoreMain.cpp +++ b/src/agent/Core/CoreMain.cpp @@ -85,7 +85,6 @@ #include #include #include -#include #include #include #include @@ -145,7 +144,6 @@ namespace Core { boost::mutex configSyncher; - ResourceLocator resourceLocator; RandomGeneratorPtr randomGenerator; SpawningKit::Context::Schema spawningKitContextSchema; SpawningKit::ContextPtr spawningKitContext; @@ -697,8 +695,6 @@ initializeNonPrivilegedWorkingObjects() { setenv("SERVER_SOFTWARE", coreConfig->get("server_software").asCString(), 1); - wo->resourceLocator = ResourceLocator(coreConfig->get("passenger_root").asString()); - wo->randomGenerator = boost::make_shared(); // Check whether /dev/urandom is actually random. // https://code.google.com/p/phusion-passenger/issues/detail?id=516 @@ -710,7 +706,7 @@ initializeNonPrivilegedWorkingObjects() { UPDATE_TRACE_POINT(); wo->spawningKitContext = boost::make_shared( wo->spawningKitContextSchema); - wo->spawningKitContext->resourceLocator = &wo->resourceLocator; + wo->spawningKitContext->resourceLocator = Agent::Fundamentals::context->resourceLocator; wo->spawningKitContext->wrapperRegistry = coreWrapperRegistry; wo->spawningKitContext->randomGenerator = wo->randomGenerator; wo->spawningKitContext->integrationMode = coreConfig->get("integration_mode").asString(); @@ -772,7 +768,7 @@ initializeNonPrivilegedWorkingObjects() { &coreSchema->controllerSingleAppMode.schema, &wo->singleAppModeConfig, coreSchema->controllerSingleAppMode.translator); - two.controller->resourceLocator = &wo->resourceLocator; + two.controller->resourceLocator = Agent::Fundamentals::context->resourceLocator; two.controller->wrapperRegistry = coreWrapperRegistry; two.controller->appPool = wo->appPool; two.controller->shutdownFinishCallback = controllerShutdownFinished; @@ -877,7 +873,7 @@ initializeSecurityUpdateChecker() { config, coreSchema->securityUpdateChecker.translator); workingObjects->securityUpdateChecker = checker; - checker->resourceLocator = &workingObjects->resourceLocator; + checker->resourceLocator = Agent::Fundamentals::context->resourceLocator; checker->initialize(); checker->start(); } @@ -927,7 +923,7 @@ initializeAdminPanelConnector() { AdminPanelConnector *connector = new Core::AdminPanelConnector( coreSchema->adminPanelConnector.schema, config, coreSchema->adminPanelConnector.translator); - connector->resourceLocator = &wo.resourceLocator; + connector->resourceLocator = Agent::Fundamentals::context->resourceLocator; connector->appPool = wo.appPool; connector->configGetter = inspectConfig; for (unsigned int i = 0; i < wo.threadWorkingObjects.size(); i++) { @@ -956,7 +952,7 @@ prestartWebApps() { } boost::function func = boost::bind(prestartWebApps, - wo->resourceLocator, + *Agent::Fundamentals::context->resourceLocator, coreConfig->get("default_ruby").asString(), prestartURLs ); @@ -983,7 +979,7 @@ warnIfPassengerRootVulnerable() { return; // Passenger is not root, so no escalation. } - string root = workingObjects->resourceLocator.getInstallSpec(); + string root = Agent::Fundamentals::context->resourceLocator->getInstallSpec(); vector errors, checkErrors; if (isPathProbablySecureForRootUse(root, errors, checkErrors)) { if (!checkErrors.empty()) { @@ -1400,13 +1396,10 @@ parseOptions(int argc, const char *argv[], ConfigKit::Store &config) { } } - if (!updates.empty()) { - vector errors; - if (!config.update(updates, errors)) { - P_BUG("Unable to set initial configuration: " << - ConfigKit::toString(errors) << "\n" - "Raw initial configuration: " << updates.toStyledString()); - } + vector errors; + if (!config.update(updates, errors)) { + P_ERROR("*** Error in options: " << ConfigKit::toString(errors)); + std::exit(1); } } diff --git a/src/agent/Shared/Fundamentals/AbortHandler.cpp b/src/agent/Shared/Fundamentals/AbortHandler.cpp index 38f4f79620..ef7473ead4 100644 --- a/src/agent/Shared/Fundamentals/AbortHandler.cpp +++ b/src/agent/Shared/Fundamentals/AbortHandler.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -685,11 +686,25 @@ dumpWithCrashWatch(AbortHandlerWorkingState &state) { close(p[0]); backtrace_symbols_fd(backtraceStore, frames, p[1]); close(p[1]); - if (waitpid(pid, &status, 0) == -1 || status != 0) { + + int ret = waitpid(pid, &status, 0); + if (ret == -1 || status != 0) { + int e = errno; pos = state.messageBuf; pos = ASSU::appendData(pos, end, "ERROR: cannot execute '"); pos = ASSU::appendData(pos, end, ctx->backtraceSanitizerCommand); - pos = ASSU::appendData(pos, end, "' for sanitizing the backtrace, writing to stderr directly...\n"); + pos = ASSU::appendData(pos, end, "' for sanitizing the backtrace ("); + if (ret == -1) { + pos = ASSU::appendData(pos, end, "waitpid() failed, errno="); + pos = ASSU::appendInteger(pos, end, e); + } else if (WIFSIGNALED(status)) { + pos = ASSU::appendData(pos, end, "exited with signal "); + pos = ASSU::appendInteger(pos, end, WTERMSIG(status)); + } else { + pos = ASSU::appendData(pos, end, "exit status "); + pos = ASSU::appendInteger(pos, end, WEXITSTATUS(status)); + } + pos = ASSU::appendData(pos, end, "), writing to stderr directly...\n"); write_nowarn(STDERR_FILENO, state.messageBuf, pos - state.messageBuf); backtrace_symbols_fd(backtraceStore, frames, STDERR_FILENO); } @@ -1262,6 +1277,12 @@ abortHandlerConfigChanged() { char *oldCrashWatchCommand = ctx->crashWatchCommand; char *oldBacktraceSanitizerCommand = ctx->backtraceSanitizerCommand; + if (config->origArgv == NULL) { + fprintf(stderr, "AbortHandler requires config->origArgv!\n"); + fflush(stderr); + abort(); + } + if (config->resourceLocator != NULL) { string path; const ResourceLocator *locator = config->resourceLocator; diff --git a/src/agent/Shared/Fundamentals/Initialization.cpp b/src/agent/Shared/Fundamentals/Initialization.cpp index fe379098ae..74dd2eec98 100644 --- a/src/agent/Shared/Fundamentals/Initialization.cpp +++ b/src/agent/Shared/Fundamentals/Initialization.cpp @@ -525,6 +525,12 @@ storeArgvCopy(int argc, char *argv[]) { for (int i = 0; i < argc; i++) { context->origArgv[i] = strdup(argv[i]); } + + if (abortHandlerInstalled()) { + // Let AbortHandler use the copy. + context->abortHandlerConfig.origArgv = context->origArgv; + abortHandlerConfigChanged(); + } } static void @@ -558,7 +564,7 @@ initializeAgent(int argc, char **argv[], const char *processName, { const char *seedStr; - context = new Context(); + context = new Context(argc, *argv); seedStr = getEnvString("PASSENGER_RANDOM_SEED"); if (seedStr == NULL) { diff --git a/src/agent/Shared/Fundamentals/Initialization.h b/src/agent/Shared/Fundamentals/Initialization.h index fa8f9271d7..b20a11e21b 100644 --- a/src/agent/Shared/Fundamentals/Initialization.h +++ b/src/agent/Shared/Fundamentals/Initialization.h @@ -48,11 +48,11 @@ struct Context { bool feedbackFdAvailable; AbortHandlerConfig abortHandlerConfig; - Context() + Context(int argc, char *argv[]) : resourceLocator(NULL), randomSeed(0), - origArgc(0), - origArgv(NULL), + origArgc(argc), + origArgv(argv), feedbackFdAvailable(false) { } }; diff --git a/src/agent/Watchdog/WatchdogMain.cpp b/src/agent/Watchdog/WatchdogMain.cpp index 4d08b35865..c75297eeca 100644 --- a/src/agent/Watchdog/WatchdogMain.cpp +++ b/src/agent/Watchdog/WatchdogMain.cpp @@ -836,13 +836,10 @@ parseOptions(int argc, const char *argv[], ConfigKit::Store &config) { } } - if (!updates.empty()) { - vector errors; - if (!config.update(updates, errors)) { - P_BUG("Unable to set initial configuration: " << - ConfigKit::toString(errors) << "\n" - "Raw initial configuration: " << updates.toStyledString()); - } + vector errors; + if (!config.update(updates, errors)) { + P_ERROR("*** Error in options: " << ConfigKit::toString(errors)); + std::exit(1); } } diff --git a/src/cxx_supportlib/oxt/system_calls.cpp b/src/cxx_supportlib/oxt/system_calls.cpp index 89a39a47af..81078b038e 100644 --- a/src/cxx_supportlib/oxt/system_calls.cpp +++ b/src/cxx_supportlib/oxt/system_calls.cpp @@ -65,9 +65,16 @@ oxt::setup_syscall_interruption_support() { do { ret = sigaction(INTERRUPTION_SIGNAL, &action, NULL); } while (ret == -1 && errno == EINTR); - do { - ret = siginterrupt(INTERRUPTION_SIGNAL, 1); - } while (ret == -1 && errno == EINTR); + #if !defined(__linux__) + // On Linux, siginterrupt() is deprecated. System call interruption works + // by merely calling sigaction() with sa_flags without SA_RESTART. + // Unfortunately, this does not seem to be the case on other operationg systems. + // For example, on macOS, we need to call *both* sigaction() without SA_RESTART, + // *and* siginterrupt(). + do { + ret = siginterrupt(INTERRUPTION_SIGNAL, 1); + } while (ret == -1 && errno == EINTR); + #endif } void diff --git a/src/ruby_supportlib/phusion_passenger/config/compile_nginx_engine_command.rb b/src/ruby_supportlib/phusion_passenger/config/compile_nginx_engine_command.rb index 1a889188f7..4d9ae71933 100644 --- a/src/ruby_supportlib/phusion_passenger/config/compile_nginx_engine_command.rb +++ b/src/ruby_supportlib/phusion_passenger/config/compile_nginx_engine_command.rb @@ -39,7 +39,8 @@ def run :colorize => :auto, :force_tip => true, :connect_timeout => 30, - :idle_timeout => 30 + :idle_timeout => 30, + :address_sanitizer => false, } parse_options initialize_objects @@ -70,7 +71,7 @@ def self.create_option_parser(options) opts.on("--nginx-tarball PATH", String, "Use the given Nginx tarball instead of#{nl}" + "downloading it. You MUST also specify the#{nl}" + "Nginx version with --nginx-version") do |val| - options[:nginx_tarball] = val + options[:nginx_tarball] = File.absolute_path(val) end opts.on("-f", "--force", "Skip sanity checks") do options[:force] = true @@ -88,6 +89,9 @@ def self.create_option_parser(options) opts.on("--idle-timeout SECONDS", Integer, "The maximum idle read time. Default: 30") do |val| options[:idle_timeout] = val end + opts.on("--address-sanitizer", "Compile with AddressSanitizer support") do + options[:address_sanitizer] = true + end opts.on("-h", "--help", "Show this help") do options[:help] = true end diff --git a/src/ruby_supportlib/phusion_passenger/config/install_standalone_runtime_command.rb b/src/ruby_supportlib/phusion_passenger/config/install_standalone_runtime_command.rb index cf2b7ad118..bc591536da 100644 --- a/src/ruby_supportlib/phusion_passenger/config/install_standalone_runtime_command.rb +++ b/src/ruby_supportlib/phusion_passenger/config/install_standalone_runtime_command.rb @@ -272,11 +272,21 @@ def compile_nginx_engine(tmpdir) args << "--nginx-tarball" args << @options[:nginx_tarball] end + # The agent and the support libraries are compiled through Rake, which + # already supports $USE_ASAN, so it makes sense to use that environment + # variable here as well rather than introducing a CLI flag. + if boolean_option('USE_ASAN') + args << "--address-sanitizer" + end CompileNginxEngineCommand.new(args).run else abort "No precompiled Nginx engine could be downloaded. Refusing to compile because --no-compile is given." end end + + def boolean_option(env_name) + ["true", "on", "yes", "1"].include?(ENV[env_name]) + end end end # module Config diff --git a/src/ruby_supportlib/phusion_passenger/config/nginx_engine_compiler.rb b/src/ruby_supportlib/phusion_passenger/config/nginx_engine_compiler.rb index dcc8478e70..6a0a3d845f 100644 --- a/src/ruby_supportlib/phusion_passenger/config/nginx_engine_compiler.rb +++ b/src/ruby_supportlib/phusion_passenger/config/nginx_engine_compiler.rb @@ -44,11 +44,19 @@ module Config class NginxEngineCompiler < AbstractInstaller include InstallationUtils - def self.configure_script_options - extra_cflags = "-Wno-error #{PlatformInfo.openssl_extra_cflags} #{PlatformInfo.pcre_extra_cflags}".strip + def self.configure_script_options(address_sanitizer: false) + extra_cflags = [ + "-Wno-error", + PlatformInfo.openssl_extra_cflags, + PlatformInfo.pcre_extra_cflags, + ].compact.join(" ").strip result = "--with-cc-opt=#{Shellwords.escape extra_cflags} " - extra_ldflags = "#{PlatformInfo.openssl_extra_ldflags} #{PlatformInfo.pcre_extra_ldflags}".strip + extra_ldflags = [ + PlatformInfo.openssl_extra_ldflags, + PlatformInfo.pcre_extra_ldflags, + address_sanitizer ? PlatformInfo.address_sanitizer_flags : nil, + ].compact.join(" ").strip if !extra_ldflags.empty? result << "--with-ld-opt=#{Shellwords.escape extra_ldflags} " end @@ -339,7 +347,7 @@ def configure_nginx # work around the problem by configure Nginx with prefix # /tmp. command << "#{shell} ./configure --prefix=/tmp " + - "#{self.class.configure_script_options} " + + "#{self.class.configure_script_options(address_sanitizer: @address_sanitizer)} " + "--add-module=#{Shellwords.escape PhusionPassenger.nginx_module_source_dir}" run_command_yield_activity(command) do yield diff --git a/test/config.json.github-ci-linux b/test/config.json.github-ci-linux new file mode 100644 index 0000000000..337c0a3dbf --- /dev/null +++ b/test/config.json.github-ci-linux @@ -0,0 +1,15 @@ +{ + "normal_user_1": "games", + "normal_user_2": "daemon", + "default_user": "man", + "normal_group_1": "daemon", + "normal_group_2": "man", + "default_group": "games", + + "nonexistant_user": "xxxxxxxxxxxxxxxxxxx", + "nonexistant_group": "xxxxxxxxxxxxxxxxxxx", + "nonexistant_uid": 9999, + "nonexistant_gid": 9999, + + "nginx": "/tmp/nginx/sbin/nginx" +} diff --git a/test/config.json.github-ci-macos b/test/config.json.github-ci-macos new file mode 100644 index 0000000000..74a3fee5db --- /dev/null +++ b/test/config.json.github-ci-macos @@ -0,0 +1,15 @@ +{ + "normal_user_1": "_USER_", + "normal_user_2": "daemon", + "default_user": "_sandbox", + "normal_group_1": "daemon", + "normal_group_2": "_sandbox", + "default_group": "daemon", + "nonexistant_user": "xxxxxxxxxxxxxxxxxxx", + "nonexistant_group": "xxxxxxxxxxxxxxxxxxx", + "nonexistant_uid": 9999, + "nonexistant_gid": 9999, + "codesigning_identity": "Github CI", + "codesigning_keychain": "_SOURCE_ROOT_/mykeychain.keychain", + "nginx": "/tmp/nginx/sbin/nginx" +} diff --git a/test/cxx/CxxTestMain.cpp b/test/cxx/CxxTestMain.cpp index 60f50c199d..8e40ae9826 100644 --- a/test/cxx/CxxTestMain.cpp +++ b/test/cxx/CxxTestMain.cpp @@ -157,8 +157,8 @@ parseOptions(int argc, const char *argv[], ConfigKit::Store &config) { vector errors; if (!config.update(updates, errors)) { - P_BUG("Unable to set initial configuration: " << - ConfigKit::toString(errors)); + P_ERROR("*** Error in options: " << ConfigKit::toString(errors)); + std::exit(1); } } diff --git a/test/cxx/IOTools/BufferedIOTest.cpp b/test/cxx/IOTools/BufferedIOTest.cpp index 98cd6b4116..72cfc7ddc8 100644 --- a/test/cxx/IOTools/BufferedIOTest.cpp +++ b/test/cxx/IOTools/BufferedIOTest.cpp @@ -336,7 +336,7 @@ namespace tut { ensure_equals(io.readLine(), "hello\n"); ensure_equals(io.getBuffer(), "world\n."); ensure("At least 33 msec elapsed", timer1.elapsed() >= 33); - ensure("At most 250 msec elapsed", timer1.elapsed() <= 250); + ensure("At most 350 msec elapsed", timer1.elapsed() <= 350); TempThread thr3(boost::bind(closeAfterSomeTime, writer, 20000)); Timer<> timer2; diff --git a/test/integration_tests/nginx_tests.rb b/test/integration_tests/nginx_tests.rb index 947b5ba5b2..797220c3d3 100644 --- a/test/integration_tests/nginx_tests.rb +++ b/test/integration_tests/nginx_tests.rb @@ -488,7 +488,7 @@ def rename_entrypoint_file PhusionPassenger.on_event(:oob_work) do f = File.open("#{@stub.full_app_root}/oob_work.\#{$$}", 'w') f.close - sleep 1 + sleep 3 end app = lambda do |env| if env['PATH_INFO'] == '/oobw' @@ -515,7 +515,7 @@ def rename_entrypoint_file t0 = Time.now get("/oobw") secs = Time.now - t0 - secs.should <= 0.1 + secs.should <= 0.5 end end