diff --git a/.buildkite/Makefile b/.buildkite/Makefile index b2491bbd4786..43eba7a3db66 100644 --- a/.buildkite/Makefile +++ b/.buildkite/Makefile @@ -1,3 +1,4 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. export TOP := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) export ARCH ?= $(shell if [[ $$(arch) == x86_64 ]]; then echo amd64; else echo arm64; fi) diff --git a/.buildkite/basic-search-test.sh b/.buildkite/basic-search-test.sh index 092484ece520..5daaa55e2d6a 100755 --- a/.buildkite/basic-search-test.sh +++ b/.buildkite/basic-search-test.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail diff --git a/.buildkite/bootstrap-cmake.sh b/.buildkite/bootstrap-cmake.sh index 75d481423787..2512c7061e89 100755 --- a/.buildkite/bootstrap-cmake.sh +++ b/.buildkite/bootstrap-cmake.sh @@ -1,7 +1,9 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail +# shellcheck disable=1091 source /etc/profile.d/enable-gcc-toolset.sh VESPA_CMAKE_SANITIZERS_OPTION="" diff --git a/.buildkite/bootstrap.sh b/.buildkite/bootstrap.sh index d00391709eca..1a29c117b4bd 100755 --- a/.buildkite/bootstrap.sh +++ b/.buildkite/bootstrap.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail diff --git a/.buildkite/build-container.sh b/.buildkite/build-container.sh index 4d2542917afa..1b91b2591345 100755 --- a/.buildkite/build-container.sh +++ b/.buildkite/build-container.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail diff --git a/.buildkite/build-rpms.sh b/.buildkite/build-rpms.sh index 3dc05e6e6ceb..642a4dc7d7a3 100755 --- a/.buildkite/build-rpms.sh +++ b/.buildkite/build-rpms.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail diff --git a/.buildkite/cpp-test.sh b/.buildkite/cpp-test.sh index dafc754742d8..96a888382f92 100755 --- a/.buildkite/cpp-test.sh +++ b/.buildkite/cpp-test.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail diff --git a/.buildkite/cpp.sh b/.buildkite/cpp.sh index 1403bb916b11..6ee7effdfd06 100755 --- a/.buildkite/cpp.sh +++ b/.buildkite/cpp.sh @@ -1,7 +1,9 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail +# shellcheck disable=1091 source /etc/profile.d/enable-gcc-toolset.sh cd "$SOURCE_DIR" diff --git a/screwdriver/pubring.gpg.enc b/.buildkite/deploy/pubring.gpg.enc similarity index 100% rename from screwdriver/pubring.gpg.enc rename to .buildkite/deploy/pubring.gpg.enc diff --git a/screwdriver/secring.gpg.enc b/.buildkite/deploy/secring.gpg.enc similarity index 100% rename from screwdriver/secring.gpg.enc rename to .buildkite/deploy/secring.gpg.enc diff --git a/.buildkite/execute.sh b/.buildkite/execute.sh index dcf42394e525..111167b28ea8 100755 --- a/.buildkite/execute.sh +++ b/.buildkite/execute.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail @@ -7,7 +8,7 @@ if [[ $# != 1 ]]; then exit 1 fi -readonly MYDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )" +MYDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )" readonly STEP=$1 readonly VERBOSE=${VERBOSE:-} diff --git a/.buildkite/go.sh b/.buildkite/go.sh index 273a8282e9ed..c7d5f4aabddd 100755 --- a/.buildkite/go.sh +++ b/.buildkite/go.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail diff --git a/.buildkite/install.sh b/.buildkite/install.sh index 42faf5be0766..9a568c614435 100755 --- a/.buildkite/install.sh +++ b/.buildkite/install.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail diff --git a/.buildkite/java.sh b/.buildkite/java.sh index bc23533c56d3..fa40ff07c8a5 100755 --- a/.buildkite/java.sh +++ b/.buildkite/java.sh @@ -1,7 +1,9 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail +# shellcheck disable=1091 source /etc/profile.d/enable-gcc-toolset.sh cd "$SOURCE_DIR" diff --git a/.buildkite/plugins/factory-reporter/README.md b/.buildkite/plugins/factory-reporter/README.md index 33297ad175b9..e8bbbc99a92c 100644 --- a/.buildkite/plugins/factory-reporter/README.md +++ b/.buildkite/plugins/factory-reporter/README.md @@ -1,4 +1,5 @@ # Factory Reporter Buildkite Plugin +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. Reports a build to Vespa Factory diff --git a/.buildkite/plugins/factory-reporter/hooks/pre-command b/.buildkite/plugins/factory-reporter/hooks/pre-command index 3ae6fddf9d7b..50a8d0e45501 100755 --- a/.buildkite/plugins/factory-reporter/hooks/pre-command +++ b/.buildkite/plugins/factory-reporter/hooks/pre-command @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail diff --git a/.buildkite/plugins/factory-reporter/hooks/pre-exit b/.buildkite/plugins/factory-reporter/hooks/pre-exit index 1b0b3ef7513d..1f8e71389222 100755 --- a/.buildkite/plugins/factory-reporter/hooks/pre-exit +++ b/.buildkite/plugins/factory-reporter/hooks/pre-exit @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -eo pipefail diff --git a/.buildkite/plugins/factory-reporter/plugin.yml b/.buildkite/plugins/factory-reporter/plugin.yml index 1164de7f96ec..81f488a4fc22 100644 --- a/.buildkite/plugins/factory-reporter/plugin.yml +++ b/.buildkite/plugins/factory-reporter/plugin.yml @@ -1,3 +1,4 @@ +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. name: Factory Reporter description: Reports build pipeline status to Vespa Factory author: https://github.com/vespa-engine diff --git a/.buildkite/prepare.sh b/.buildkite/prepare.sh index cb48f3de7598..cec886ec7b53 100755 --- a/.buildkite/prepare.sh +++ b/.buildkite/prepare.sh @@ -1,8 +1,9 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail -"$SOURCE_DIR/screwdriver/replace-vespa-version-in-poms.sh" "$VESPA_VERSION" "$SOURCE_DIR" +"$SOURCE_DIR/.buildkite/replace-vespa-version-in-poms.sh" "$VESPA_VERSION" "$SOURCE_DIR" # We disable javadoc for all modules not marked as public API for MODULE in $(comm -2 -3 \ diff --git a/.buildkite/publish-artifacts.sh b/.buildkite/publish-artifacts.sh index 507232c5e83e..23a17d87bd02 100755 --- a/.buildkite/publish-artifacts.sh +++ b/.buildkite/publish-artifacts.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail diff --git a/.buildkite/publish-container.sh b/.buildkite/publish-container.sh index 824ebb8f6240..09a2ea833f2d 100755 --- a/.buildkite/publish-container.sh +++ b/.buildkite/publish-container.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail diff --git a/.buildkite/quick-start-guide.sh b/.buildkite/quick-start-guide.sh index 05cb08429479..dee77bcead44 100755 --- a/.buildkite/quick-start-guide.sh +++ b/.buildkite/quick-start-guide.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail @@ -7,4 +8,4 @@ if [[ $VESPA_USE_SANITIZER != null ]]; then exit 0 fi -"$SOURCE_DIR/screwdriver/test-quick-start-guide.sh" +"$SOURCE_DIR/.buildkite/test-quick-start-guide.sh" diff --git a/screwdriver/release-ann-benchmark.sh b/.buildkite/release-ann-benchmark.sh similarity index 78% rename from screwdriver/release-ann-benchmark.sh rename to .buildkite/release-ann-benchmark.sh index 581ee289f191..438c5a5355b6 100755 --- a/screwdriver/release-ann-benchmark.sh +++ b/.buildkite/release-ann-benchmark.sh @@ -1,4 +1,5 @@ #!/usr/bin/ssh-agent /bin/bash +# shellcheck shell=bash disable=SC1008 # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail @@ -16,16 +17,17 @@ if [[ -z "$ANN_BENCHMARK_DEPLOY_TOKEN" ]]; then fi BUILD_DIR=$(mktemp -d) +# shellcheck disable=2064 trap "rm -rf $BUILD_DIR" EXIT -cd $BUILD_DIR +cd "$BUILD_DIR" ssh-add -D -ssh-add <(echo $ANN_BENCHMARK_DEPLOY_TOKEN | base64 -d) +ssh-add <(echo "$ANN_BENCHMARK_DEPLOY_TOKEN" | base64 -d) git clone git@github.com:vespa-engine/vespa-ann-benchmark cd vespa-ann-benchmark RELEASE_TAG="v$VESPA_VERSION" -if ! git rev-parse $RELEASE_TAG &> /dev/null; then +if ! git rev-parse "$RELEASE_TAG" &> /dev/null; then git tag -a "$RELEASE_TAG" -m "Release version $VESPA_VERSION" git push origin "$RELEASE_TAG" fi diff --git a/screwdriver/release-container-image-crane.sh b/.buildkite/release-container-image-crane.sh similarity index 50% rename from screwdriver/release-container-image-crane.sh rename to .buildkite/release-container-image-crane.sh index e08ee2f7ed08..82f5ff16d21e 100755 --- a/screwdriver/release-container-image-crane.sh +++ b/.buildkite/release-container-image-crane.sh @@ -9,7 +9,7 @@ if [[ $# -ne 1 ]]; then fi readonly VESPA_VERSION=$1 -readonly VESPA_MAJOR=$(echo $VESPA_VERSION | cut -d. -f1) +VESPA_MAJOR=$(echo "$VESPA_VERSION" | cut -d. -f1) if [[ -z "$DOCKER_HUB_DEPLOY_TOKEN" ]]; then echo "Environment variable DOCKER_HUB_DEPLOY_TOKEN must be set, but is empty." @@ -20,33 +20,33 @@ if [[ -z "$GHCR_DEPLOY_TOKEN" ]]; then exit 1 fi -crane auth login -u aressem -p $GHCR_DEPLOY_TOKEN ghcr.io -SRC_IMAGE=ghcr.io/vespa-engine/vespa-preview:$VESPA_VERSION -SRC_IMAGE_DIGEST=$(crane digest $SRC_IMAGE) +crane auth login -u aressem -p "$GHCR_DEPLOY_TOKEN" ghcr.io +SRC_IMAGE="ghcr.io/vespa-engine/vespa-preview:$VESPA_VERSION" +SRC_IMAGE_DIGEST=$(crane digest "$SRC_IMAGE") cosign verify \ --certificate-identity https://buildkite.com/vespaai/vespa-engine-vespa \ --certificate-oidc-issuer https://agent.buildkite.com \ - $SRC_IMAGE@$SRC_IMAGE_DIGEST + "$SRC_IMAGE@$SRC_IMAGE_DIGEST" # Copy to Docker Hub -if curl -fsSL https://hub.docker.com/v2/repositories/vespaengine/vespa/tags/$VESPA_VERSION &> /dev/null; then +if curl -fsSL "https://hub.docker.com/v2/repositories/vespaengine/vespa/tags/$VESPA_VERSION" &> /dev/null; then echo "Container image docker.io/vespaengine/vespa:$VESPA_VERSION already exists." else - DST_IMAGE=docker.io/vespaengine/vespa:$VESPA_VERSION - crane auth login -u aressem -p $DOCKER_HUB_DEPLOY_TOKEN docker.io - crane cp $SRC_IMAGE@$SRC_IMAGE_DIGEST $DST_IMAGE - crane tag $DST_IMAGE $VESPA_MAJOR - crane tag $DST_IMAGE latest + DST_IMAGE="docker.io/vespaengine/vespa:$VESPA_VERSION" + crane auth login -u aressem -p "$DOCKER_HUB_DEPLOY_TOKEN" docker.io + crane cp "$SRC_IMAGE@$SRC_IMAGE_DIGEST" "$DST_IMAGE" + crane tag "$DST_IMAGE" "$VESPA_MAJOR" + crane tag "$DST_IMAGE" latest fi # Copy to GitHub Container Registry -JWT=$(curl -sSL -u aressem:$GHCR_DEPLOY_TOKEN "https://ghcr.io/token?service=ghcr.io&scope=repository:vespa-engine/vespa:pull" | jq -re '.token') +JWT=$(curl -sSL -u "aressem:$GHCR_DEPLOY_TOKEN" "https://ghcr.io/token?service=ghcr.io&scope=repository:vespa-engine/vespa:pull" | jq -re '.token') IMAGE_TAGS=$(curl -sSL -H "Authorization: Bearer $JWT" https://ghcr.io/v2/vespa-engine/vespa/tags/list | jq -re '.tags[]') -if grep $VESPA_VERSION <<< "$IMAGE_TAGS" &> /dev/null; then +if grep "$VESPA_VERSION" <<< "$IMAGE_TAGS" &> /dev/null; then echo "Container image ghcr.io/vespa-engine/vespa:$VESPA_VERSION already exists." else - DST_IMAGE=ghcr.io/vespa-engine/vespa:$VESPA_VERSION - crane cp $SRC_IMAGE@$SRC_IMAGE_DIGEST $DST_IMAGE - crane tag $DST_IMAGE $VESPA_MAJOR - crane tag $DST_IMAGE latest + DST_IMAGE="ghcr.io/vespa-engine/vespa:$VESPA_VERSION" + crane cp "$SRC_IMAGE@$SRC_IMAGE_DIGEST" "$DST_IMAGE" + crane tag "$DST_IMAGE" "$VESPA_MAJOR" + crane tag "$DST_IMAGE" latest fi diff --git a/screwdriver/release-generic-container-image.sh b/.buildkite/release-generic-container-image.sh similarity index 63% rename from screwdriver/release-generic-container-image.sh rename to .buildkite/release-generic-container-image.sh index d996dc279c4c..79020f5ba63a 100755 --- a/screwdriver/release-generic-container-image.sh +++ b/.buildkite/release-generic-container-image.sh @@ -11,30 +11,31 @@ fi readonly VESPA_VERSION=$1 readonly IMAGE_NAME="vespaengine/vespa-generic-intel-x86_64" -if curl -fsSL https://hub.docker.com/v2/repositories/$IMAGE_NAME/tags/$VESPA_VERSION/ &> /dev/null; then +if curl -fsSL "https://hub.docker.com/v2/repositories/$IMAGE_NAME/tags/$VESPA_VERSION/" &> /dev/null; then echo "Container image docker.io/$IMAGE_NAME:$VESPA_VERSION aldready exists." exit 0 fi TMPDIR=$(mktemp -d) +# shellcheck disable=2064 trap "rm -rf $TMPDIR" EXIT -pushd $TMPDIR +pushd "$TMPDIR" git clone -q --filter tree:0 https://github.com/vespa-engine/vespa -(cd vespa && git checkout v$VESPA_VERSION) -(cd vespa && screwdriver/replace-vespa-version-in-poms.sh $VESPA_VERSION $(pwd) ) +(cd vespa && git checkout "v$VESPA_VERSION") +(cd vespa && .buildkite/replace-vespa-version-in-poms.sh "$VESPA_VERSION" "$(pwd)" ) -make -C vespa -f .copr/Makefile srpm outdir=$(pwd) +make -C vespa -f .copr/Makefile srpm outdir="$(pwd)" rpmbuild --rebuild \ --define="_topdir $TMPDIR/vespa-rpmbuild" \ --define "debug_package %{nil}" \ --define "_debugsource_template %{nil}" \ --define '_cmake_extra_opts "-DDEFAULT_VESPA_CPU_ARCH_FLAGS=-msse3 -mcx16 -mtune=intel"' \ - *.src.rpm + ./*.src.rpm -rm -f *.src.rpm -mv $TMPDIR/vespa-rpmbuild/RPMS/*/*.rpm . +rm -f ./*.src.rpm +mv "$TMPDIR"/vespa-rpmbuild/RPMS/*/*.rpm . cat < Dockerfile ARG VESPA_VERSION @@ -46,18 +47,18 @@ EOF docker build --progress plain \ - --build-arg VESPA_VERSION=$VESPA_VERSION \ - --tag docker.io/$IMAGE_NAME:$VESPA_VERSION \ - --tag docker.io/$IMAGE_NAME:latest \ + --build-arg VESPA_VERSION="$VESPA_VERSION" \ + --tag "docker.io/$IMAGE_NAME:$VESPA_VERSION" \ + --tag "docker.io/$IMAGE_NAME:latest" \ --tag vespaengine/vespa:latest \ --file Dockerfile . -vespa/screwdriver/test-quick-start-guide.sh +vespa/.buildkite/test-quick-start-guide.sh OPT_STATE="$(set +o)" set +x docker login --username aressem --password "$DOCKER_HUB_DEPLOY_TOKEN" eval "$OPT_STATE" -docker push docker.io/$IMAGE_NAME:$VESPA_VERSION -docker push docker.io/$IMAGE_NAME:latest +docker push "docker.io/$IMAGE_NAME:$VESPA_VERSION" +docker push "docker.io/$IMAGE_NAME:latest" diff --git a/.buildkite/release-java-artifacts.sh b/.buildkite/release-java-artifacts.sh new file mode 100755 index 000000000000..7ffb938b9788 --- /dev/null +++ b/.buildkite/release-java-artifacts.sh @@ -0,0 +1,145 @@ +#!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " + exit 1 +fi + +if [[ -z $OSSRH_USER ]] || [[ -z $OSSRH_TOKEN ]] || [[ -z $GPG_KEYNAME ]] || [[ -z $GPG_PASSPHRASE_TOKEN ]] || [[ -z $GPG_ENCPHRASE_TOKEN ]]; then + echo -e "The follwing env variables must be set:\n OSSRH_USER\n OSSRH_TOKEN\n GPG_KEYNAME\n GPG_PASSPHRASE_TOKEN\n GPG_ENCPHRASE_TOKEN" + exit 1 +fi + +readonly VESPA_RELEASE="$1" + +# shellcheck shell=bash disable=SC1083 +QUERY_VERSION_HTTP_CODE=$(curl --write-out %{http_code} --silent --location --output /dev/null "https://oss.sonatype.org/content/repositories/releases/com/yahoo/vespa/parent/${VESPA_RELEASE}/parent-${VESPA_RELEASE}.pom") +if [[ "200" == "$QUERY_VERSION_HTTP_CODE" ]]; then + echo "Vespa version $VESPA_RELEASE is already promoted, exiting" + exit 0 +fi + +SOURCE_DIR=$(pwd) +export SOURCE_DIR + +mkdir -p "$SOURCE_DIR/.buildkite/deploy" +# gpg-agent in RHEL 8 runs out of memory if we use Maven and sign in parallel. Add option to overcome this. +echo "auto-expand-secmem" >> "$SOURCE_DIR/.buildkite/deploy/gpg-agent.conf" +openssl aes-256-cbc -md md5 -pass "pass:$GPG_ENCPHRASE_TOKEN" -in "$SOURCE_DIR/.buildkite/deploy/pubring.gpg.enc" -out "$SOURCE_DIR/.buildkite/deploy/pubring.gpg" -d +openssl aes-256-cbc -md md5 -pass "pass:$GPG_ENCPHRASE_TOKEN" -in "$SOURCE_DIR/.buildkite/deploy/secring.gpg.enc" -out "$SOURCE_DIR/.buildkite/deploy/secring.gpg" -d +chmod 700 "$SOURCE_DIR/.buildkite/deploy" +chmod 600 "$SOURCE_DIR"/.buildkite/deploy/* +# shellcheck shell=bash disable=SC2064 +trap "rm -rf $SOURCE_DIR/.buildkite/deploy" EXIT + +# Number of parallel uploads +NUM_PROC=10 + +MVN=${MVN:-mvn} +MVN_OPTS=${MVN_OPTS:-} +MAVEN_GPG_PASSPHRASE=$GPG_PASSPHRASE_TOKEN +export MVN +export MVN_OPTS +export MAVEN_GPG_PASSPHRASE + +TMP_STAGING=$(mktemp -d) +export TMP_STAGING +mkdir -p "$TMP_STAGING" +# shellcheck disable=2064 +trap "rm -rf $TMP_STAGING" EXIT + +sign_module() { + + #Debug + set -x + + ECHO="" + + P=$1 + V=$(basename "$P") + A=$(basename "$(dirname "$P")") + G=$(dirname "$(dirname "$P")" | sed 's,/,.,g') + POM="$P/$A-$V.pom" + JAR="$P/$A-$V.jar" + if [[ -f $JAR ]]; then + FILE_OPTS="-Dfile=$JAR -Dpackaging=jar" + else + FILE_OPTS="-Dfile=$POM -Dpackaging=pom" + fi + + AFILES=() + ATYPES=() + ACLASSIFIERS=() + # shellcheck disable=2044 + for EX in $(find "$P" \( -name "$A-$V-*.jar" -or -name "$A-$V-*.zip" \) ); do + AFILES+=("$EX") + EXB=$(basename "$EX") + EXT=${EXB##*.} + ATYPES+=("$EXT") + EXF=${EXB%.*} + # shellcheck disable=2001 + EXC=$(echo "$EXF"|sed "s,$A-$V-,,") + ACLASSIFIERS+=("$EXC") + done + + if (( ${#AFILES[@]} > 0 )); then + # shellcheck disable=2001 + EXTRA_FILES_OPTS="-Dfiles=$(echo "${AFILES[@]}" | sed 's/\ /,/g') -Dtypes=$(echo "${ATYPES[@]}" | sed 's/\ /,/g') -Dclassifiers=$(echo "${ACLASSIFIERS[@]}" | sed 's/\ /,/g')" + fi + + # shellcheck disable=2086 + $ECHO $MVN --settings="$SOURCE_DIR/.buildkite/settings-publish.xml" \ + $MVN_OPTS gpg:sign-and-deploy-file \ + -Durl="file://$TMP_STAGING" \ + -DrepositoryId=maven-central \ + -DgroupId="$G" \ + -DartifactId="$A" \ + -Dversion="$V" \ + -DpomFile="$POM" \ + -DgeneratePom=false \ + $FILE_OPTS \ + $EXTRA_FILES_OPTS + +} +export -f sign_module + +#Debug +set -x + +aws s3 cp "s3://381492154096-build-artifacts/vespa-engine--vespa/$VESPA_RELEASE/artifacts/amd64/maven-repo.tar" . +aws s3 cp "s3://381492154096-build-artifacts/vespa-engine--vespa/$VESPA_RELEASE/artifacts/amd64/maven-repo.tar.pem" . +aws s3 cp "s3://381492154096-build-artifacts/vespa-engine--vespa/$VESPA_RELEASE/artifacts/amd64/maven-repo.tar.sig" . +cosign verify-blob --certificate-identity https://buildkite.com/vespaai/vespa-engine-vespa --certificate-oidc-issuer https://agent.buildkite.com --signature maven-repo.tar.sig --certificate maven-repo.tar.pem maven-repo.tar +rm -rf maven-repo +tar xvf maven-repo.tar +REPO_ROOT=$(pwd)/maven-repo + +cd "$REPO_ROOT" +find . -name "$VESPA_RELEASE" -type d | sed 's,^./,,' | xargs -n 1 -P $NUM_PROC -I '{}' bash -c "sign_module {}" + +# Required for the nexus plugin to work with JDK 17 +export MAVEN_OPTS="--add-opens=java.base/java.util=ALL-UNNAMED" + +LOGFILE=$(mktemp) +# shellcheck disable=2086 +$MVN $MVN_OPTS --settings="$SOURCE_DIR/.buildkite/settings-publish.xml" \ + org.sonatype.plugins:nexus-staging-maven-plugin:1.6.14:deploy-staged-repository \ + -DrepositoryDirectory="$TMP_STAGING" \ + -DnexusUrl=https://oss.sonatype.org \ + -DserverId=ossrh \ + -DautoReleaseAfterClose=false \ + -DstagingProgressTimeoutMinutes=10 \ + -DstagingProfileId=407c0c3e1a197 | tee "$LOGFILE" + +STG_REPO=$(grep 'Staging repository at http' "$LOGFILE" | head -1 | awk -F/ '{print $NF}') +# shellcheck disable=2086 +$MVN $MVN_OPTS --settings="$SOURCE_DIR/.buildkite/settings-publish.xml" -N \ + org.sonatype.plugins:nexus-staging-maven-plugin:1.6.14:rc-release \ + -DnexusUrl=https://oss.sonatype.org/ \ + -DserverId=ossrh \ + -DstagingProgressTimeoutMinutes=10 \ + -DstagingRepositoryId="$STG_REPO" + diff --git a/screwdriver/release-rpms.sh b/.buildkite/release-rpms.sh similarity index 77% rename from screwdriver/release-rpms.sh rename to .buildkite/release-rpms.sh index c539ed4be185..6626d4ec615a 100755 --- a/screwdriver/release-rpms.sh +++ b/.buildkite/release-rpms.sh @@ -1,4 +1,5 @@ #!/usr/bin/ssh-agent /bin/bash +# shellcheck shell=bash disable=SC1008 # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail @@ -32,18 +33,18 @@ fi echo "Using vespa repository git reference: $VESPA_REF" ssh-add -D set +x -ssh-add <(echo $VESPA_DEPLOY_TOKEN | base64 -d) +ssh-add <(echo "$VESPA_DEPLOY_TOKEN" | base64 -d) set -x git clone git@github.com:vespa-engine/vespa cd vespa -dist/release-vespa-rpm.sh $VESPA_RELEASE $VESPA_REF +dist/release-vespa-rpm.sh "$VESPA_RELEASE" "$VESPA_REF" upload_rpms() { local ARCH=$1 - aws s3 cp s3://381492154096-build-artifacts/vespa-engine--vespa/$VESPA_RELEASE/artifacts/$ARCH/rpm-repo.tar . - aws s3 cp s3://381492154096-build-artifacts/vespa-engine--vespa/$VESPA_RELEASE/artifacts/$ARCH/rpm-repo.tar.pem . - aws s3 cp s3://381492154096-build-artifacts/vespa-engine--vespa/$VESPA_RELEASE/artifacts/$ARCH/rpm-repo.tar.sig . + aws s3 cp "s3://381492154096-build-artifacts/vespa-engine--vespa/$VESPA_RELEASE/artifacts/$ARCH/rpm-repo.tar" . + aws s3 cp "s3://381492154096-build-artifacts/vespa-engine--vespa/$VESPA_RELEASE/artifacts/$ARCH/rpm-repo.tar.pem" . + aws s3 cp "s3://381492154096-build-artifacts/vespa-engine--vespa/$VESPA_RELEASE/artifacts/$ARCH/rpm-repo.tar.sig" . cosign verify-blob \ --certificate-identity https://buildkite.com/vespaai/vespa-engine-vespa \ --certificate-oidc-issuer https://agent.buildkite.com \ @@ -52,7 +53,7 @@ upload_rpms() { rpm-repo.tar tar xvf rpm-repo.tar for rpm in rpms/*.rpm; do - screwdriver/upload-rpm-to-cloudsmith.sh $rpm + .github/scripts/upload-rpm-to-cloudsmith.sh "$rpm" done rm -rf rpms rpm-repo.* } diff --git a/screwdriver/replace-vespa-version-in-poms.sh b/.buildkite/replace-vespa-version-in-poms.sh similarity index 93% rename from screwdriver/replace-vespa-version-in-poms.sh rename to .buildkite/replace-vespa-version-in-poms.sh index d57f126aac96..dbfc84ccf615 100755 --- a/screwdriver/replace-vespa-version-in-poms.sh +++ b/.buildkite/replace-vespa-version-in-poms.sh @@ -11,7 +11,7 @@ fi readonly VESPA_VERSION=$1 readonly DIR=$2 -find $DIR -name "pom.xml" -exec sed -i \ +find "$DIR" -name "pom.xml" -exec sed -i \ -e "s,.*SNAPSHOT.*,$VESPA_VERSION," \ -e "s,.*project.version.*,$VESPA_VERSION," \ -e "s,.*project.version.*,$VESPA_VERSION," \ diff --git a/screwdriver/settings-publish.xml b/.buildkite/settings-publish.xml similarity index 93% rename from screwdriver/settings-publish.xml rename to .buildkite/settings-publish.xml index 116b1de83a96..8afb8c930941 100644 --- a/screwdriver/settings-publish.xml +++ b/.buildkite/settings-publish.xml @@ -20,7 +20,7 @@ gpg ${env.GPG_KEYNAME} false - ${env.SOURCE_DIR}/screwdriver/deploy + ${env.SOURCE_DIR}/.buildkite/deploy pubring.gpg secring.gpg diff --git a/screwdriver/test-quick-start-guide.sh b/.buildkite/test-quick-start-guide.sh similarity index 75% rename from screwdriver/test-quick-start-guide.sh rename to .buildkite/test-quick-start-guide.sh index 577d6561b853..87497fa8d69c 100755 --- a/screwdriver/test-quick-start-guide.sh +++ b/.buildkite/test-quick-start-guide.sh @@ -6,9 +6,10 @@ set -xeuo pipefail TESTDIR=$(mktemp -d) +# shellcheck disable=SC2064 trap "rm -rf $TESTDIR" EXIT -cd $TESTDIR +cd "$TESTDIR" # Clone and setup doc tests git clone -q --depth 1 https://github.com/vespa-engine/documentation @@ -23,8 +24,8 @@ if [[ $(arch) == x86_64 ]]; then else GO_ARCH=arm64 fi -curl -fsSL https://github.com/vespa-engine/vespa/releases/download/v${VESPA_CLI_VERSION}/vespa-cli_${VESPA_CLI_VERSION}_linux_${GO_ARCH}.tar.gz | tar -zxf - -C /opt -ln -sf /opt/vespa-cli_${VESPA_CLI_VERSION}_linux_${GO_ARCH}/bin/vespa /usr/local/bin/ +curl -fsSL "https://github.com/vespa-engine/vespa/releases/download/v${VESPA_CLI_VERSION}/vespa-cli_${VESPA_CLI_VERSION}_linux_${GO_ARCH}.tar.gz" | tar -zxf - -C /opt +ln -sf "/opt/vespa-cli_${VESPA_CLI_VERSION}_linux_${GO_ARCH}/bin/vespa" /usr/local/bin/ # Run test python3 test/test.py -v -c test/_quick-start.yaml diff --git a/screwdriver/update-vespa-version-in-sample-apps.sh b/.buildkite/update-vespa-version-in-sample-apps.sh similarity index 74% rename from screwdriver/update-vespa-version-in-sample-apps.sh rename to .buildkite/update-vespa-version-in-sample-apps.sh index b1d15a2f0776..8dfab9a16cfe 100755 --- a/screwdriver/update-vespa-version-in-sample-apps.sh +++ b/.buildkite/update-vespa-version-in-sample-apps.sh @@ -1,4 +1,5 @@ #!/usr/bin/ssh-agent /bin/bash +# shellcheck shell=bash disable=SC1008 # Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail @@ -9,26 +10,29 @@ if [[ $# -ne 1 ]]; then exit 1 fi -export VESPA_RELEASE="$1" -export JAVA_HOME=$(dirname $(dirname $(readlink -f /usr/bin/java))) +VESPA_RELEASE="$1" +JAVA_HOME="$(dirname "$(dirname "$(readlink -f /usr/bin/java)")")" +export VESPA_RELEASE +export JAVA_HOME BUILD_DIR=$(mktemp -d) +# shellcheck disable=2064 trap "rm -rf $BUILD_DIR" EXIT -cd $BUILD_DIR +cd "$BUILD_DIR" function is_published { local TMP_MVN_REPO=$BUILD_DIR/maven-repo - echo $TMP_MVN_REPO - mkdir -p $TMP_MVN_REPO - rm -rf $TMP_MVN_REPO/com/yahoo/vespa + echo "$TMP_MVN_REPO" + mkdir -p "$TMP_MVN_REPO" + rm -rf "$TMP_MVN_REPO/com/yahoo/vespa" # Because the transfer of artifacts to Maven Central is not atomic we can't just check a simple pom or jar to be available. Because of this we # check that the publication is complete enough to compile a Java sample app # Update Vespa version property in pom.xml - if ! mvn -V -B versions:set-property -Dproperty=vespa_version -DnewVersion=${VESPA_RELEASE}; then + if ! mvn -V -B versions:set-property -Dproperty=vespa_version -DnewVersion="$VESPA_RELEASE"; then return 1 fi - if mvn -V -B -pl ai.vespa.examples:album-recommendation-java -Dmaven.repo.local=$TMP_MVN_REPO -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -DskipTests clean package; then + if mvn -V -B -pl ai.vespa.examples:album-recommendation-java -Dmaven.repo.local="$TMP_MVN_REPO" -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -DskipTests clean package; then return 0 else return 1 @@ -40,7 +44,7 @@ function wait_until_published { until is_published; do ((cnt+=1)) # Wait max 60 minutes - if (( $cnt > 60 )); then + if (( cnt > 60 )); then echo "ERROR: Artifacts with version ${VESPA_RELEASE} not found on central maven repo." exit 1 fi @@ -51,7 +55,7 @@ function wait_until_published { ssh-add -D set +x -ssh-add <(echo $SAMPLE_APPS_DEPLOY_TOKEN | base64 -d) +ssh-add <(echo "$SAMPLE_APPS_DEPLOY_TOKEN" | base64 -d) set -x git clone git@github.com:vespa-engine/sample-apps.git diff --git a/.buildkite/upload-test-results.sh b/.buildkite/upload-test-results.sh index e95a9448adf5..2e3c6f6ea5b7 100755 --- a/.buildkite/upload-test-results.sh +++ b/.buildkite/upload-test-results.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -euo pipefail diff --git a/screwdriver/wait-for-buildkite-build.sh b/.buildkite/wait-for-buildkite-build.sh similarity index 94% rename from screwdriver/wait-for-buildkite-build.sh rename to .buildkite/wait-for-buildkite-build.sh index 582e5f376927..c13ff9ae2d6b 100755 --- a/screwdriver/wait-for-buildkite-build.sh +++ b/.buildkite/wait-for-buildkite-build.sh @@ -11,7 +11,7 @@ URL=$1 WEB_URL=$2 TIMEOUT=$3 -WAIT_UNTIL=$(( $(date +%s) + $TIMEOUT )) +WAIT_UNTIL=$(( $(date +%s) + "$TIMEOUT" )) while [[ $(date +%s) -le $WAIT_UNTIL ]]; do STATUS=$(curl -sSL -H "Content-Type: application/json" -H "Authorization: Bearer $BUILDKITE_TRIGGER_TOKEN" "$URL" | jq -re '.state') diff --git a/build_settings.cmake b/build_settings.cmake index cb5b19e77abe..995e0eedbc87 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -167,8 +167,8 @@ SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic" ) message("-- CMAKE_SHARED_LINKER_FLAGS is ${CMAKE_SHARED_LINKER_FLAGS}") -# Use C++ 20 -set(CMAKE_CXX_STANDARD 20) +# Use C++ 23 +set(CMAKE_CXX_STANDARD 23) # Always build shared libs if not explicitly specified set(BUILD_SHARED_LIBS ON) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 8675b6792390..ce9869001acf 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -122,7 +122,7 @@ interface FeatureFlags { @ModelFeatureFlag(owners = {"vekterli"}) default boolean enforceStrictlyIncreasingClusterStateVersions() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default boolean distributionConfigFromClusterController() { return false; } @ModelFeatureFlag(owners = {"arnej"}) default boolean useLegacyWandQueryParsing() { return true; } - @ModelFeatureFlag(owners = {"hmusum"}) default boolean forwardAllLogLevels() { return false; } + @ModelFeatureFlag(owners = {"hmusum"}) default boolean forwardAllLogLevels() { return true; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java index 3f336544a99b..55abf02e39ea 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java @@ -76,7 +76,7 @@ protected boolean shouldConvert(Expression exp) { String fieldName = ((OutputExpression)exp).getFieldName(); if (outputs.contains(fieldName) && !prevNames.contains(fieldName)) { throw new VerificationException(exp, "Attempting to assign conflicting values to field '" + - fieldName + "'."); + fieldName + "'"); } outputs.add(fieldName); prevNames.add(fieldName); @@ -111,24 +111,24 @@ public DataType getInputType(Expression exp, String fieldName) { } @Override - public void tryOutputType(Expression exp, String fieldName, DataType valueType) { + public void tryOutputType(Expression expression, String fieldName, DataType valueType) { String fieldDesc; DataType fieldType; - if (exp instanceof AttributeExpression) { + if (expression instanceof AttributeExpression) { Attribute attribute = schema.getAttribute(fieldName); if (attribute == null) { - throw new VerificationException(exp, "Attribute '" + fieldName + "' not found."); + throw new VerificationException(expression, "Attribute '" + fieldName + "' not found."); } fieldDesc = "attribute"; fieldType = attribute.getDataType(); - } else if (exp instanceof IndexExpression) { + } else if (expression instanceof IndexExpression) { SDField field = schema.getConcreteField(fieldName); if (field == null) { - throw new VerificationException(exp, "Index field '" + fieldName + "' not found."); + throw new VerificationException(expression, "Index field '" + fieldName + "' not found."); } fieldDesc = "index field"; fieldType = field.getDataType(); - } else if (exp instanceof SummaryExpression) { + } else if (expression instanceof SummaryExpression) { SummaryField field = schema.getSummaryField(fieldName); if (field == null) { // Use document field if summary field is not found @@ -137,7 +137,7 @@ public void tryOutputType(Expression exp, String fieldName, DataType valueType) fieldDesc = "document field"; fieldType = sdField.getDataType(); } else { - throw new VerificationException(exp, "Summary field '" + fieldName + "' not found."); + throw new VerificationException(expression, "Summary field '" + fieldName + "' not found."); } } else { fieldDesc = "summary field"; @@ -148,8 +148,8 @@ public void tryOutputType(Expression exp, String fieldName, DataType valueType) } if ( ! fieldType.isAssignableFrom(valueType) && ! fieldType.isAssignableFrom(createCompatType(valueType))) { - throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to " + fieldDesc + - " '" + fieldName + "' which is " + fieldType.getName() + "."); + throw new VerificationException(expression, "Can not assign " + valueType.getName() + " to " + fieldDesc + + " '" + fieldName + "' which is " + fieldType.getName() + "."); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 85ddd42ab226..aa4143a89f18 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -197,6 +197,7 @@ public ZooKeepersConfigProvider getZooKeepersConfigProvider() { } public void getConfig(LogdConfig.Builder builder) { + var forwardAllLogLevels = isHostedVespa && featureFlags.forwardAllLogLevels(); if (logserver == null) { builder.logserver(new LogdConfig.Logserver.Builder().use(false)); } @@ -206,8 +207,8 @@ public void getConfig(LogdConfig.Builder builder) { host(logserver.getHostName()). rpcport(logserver.getRelativePort(0))) .loglevel(new LogdConfig.Loglevel.Builder(). - debug(new LogdConfig.Loglevel.Debug.Builder().forward(featureFlags.forwardAllLogLevels())). - spam(new LogdConfig.Loglevel.Spam.Builder().forward(featureFlags.forwardAllLogLevels()))); + debug(new LogdConfig.Loglevel.Debug.Builder().forward(forwardAllLogLevels)). + spam(new LogdConfig.Loglevel.Spam.Builder().forward(forwardAllLogLevels))); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index b375c835d2c4..a4b08b5c7fd3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -113,7 +113,7 @@ private static void throwIfBudgetExceeded(double spend, BigDecimal budget, Syste private static String quotaMessage(String message, SystemName system, double spend, BigDecimal budget, boolean actual) { String quotaDescription = String.format(Locale.ENGLISH, - "The %s cost $%.2f but your quota is $%.2f", + "The %s cost $%.2f but your remaining quota is $%.2f", actual ? "resources used" : "max resources specified", spend, budget); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudAsmSecrets.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudAsmSecrets.java index 69dc5cc7c6d4..6819548bea18 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudAsmSecrets.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/CloudAsmSecrets.java @@ -3,6 +3,8 @@ import ai.vespa.secret.config.aws.AsmSecretConfig; import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.component.SimpleComponent; @@ -14,22 +16,29 @@ */ public class CloudAsmSecrets extends SimpleComponent implements AsmSecretConfig.Producer { - private static final String CLASS = "ai.vespa.secret.aws.AsmSecretStore"; + private static final String CLASS = "ai.vespa.secret.aws.AsmTenantSecretReader"; private static final String BUNDLE = "jdisc-cloud-aws"; private final URI ztsUri; private final AthenzDomain athenzDomain; + private final SystemName system; + private final TenantName tenant; - public CloudAsmSecrets(URI ztsUri, AthenzDomain athenzDomain) { + public CloudAsmSecrets(URI ztsUri, AthenzDomain athenzDomain, + SystemName system, TenantName tenant) { super(new ComponentModel(BundleInstantiationSpecification.fromStrings(CLASS, CLASS, BUNDLE))); this.ztsUri = ztsUri; this.athenzDomain = athenzDomain; + this.system = system; + this.tenant = tenant; } @Override public void getConfig(AsmSecretConfig.Builder builder) { builder.ztsUri(ztsUri.toString()) - .athenzDomain(athenzDomain.value()); + .athenzDomain(athenzDomain.value()) + .system(system.value()) + .tenant(tenant.value()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 0243075d4574..72ac906b8e00 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -238,7 +238,7 @@ private void addClusterContent(ApplicationContainerCluster cluster, Element spec // Must be added after nodes: addDeploymentSpecConfig(cluster, context, deployState.getDeployLogger()); addZooKeeper(cluster, spec); - addAthenzServiceIdentityProvider(cluster, context, deployState.getDeployLogger()); + addAthenzServiceIdentityProvider(cluster, context); addParameterStoreValidationHandler(cluster, deployState); } @@ -316,7 +316,9 @@ private void addSecrets(ApplicationContainerCluster cluster, Element spec, Deplo } cluster.addComponent(secretsConfig); cluster.addComponent(new CloudAsmSecrets(deployState.getProperties().ztsUrl(), - deployState.getProperties().tenantSecretDomain())); + deployState.getProperties().tenantSecretDomain(), + deployState.zone().system(), + deployState.getProperties().applicationId().tenant())); } } @@ -366,7 +368,7 @@ private void addCloudSecretStore(ApplicationContainerCluster cluster, Element se cluster.addComponent(cloudSecretStore); } - private void addAthenzServiceIdentityProvider(ApplicationContainerCluster cluster, ConfigModelContext context, DeployLogger deployLogger) { + private void addAthenzServiceIdentityProvider(ApplicationContainerCluster cluster, ConfigModelContext context) { if ( ! context.getDeployState().isHosted()) return; if ( ! context.getDeployState().zone().system().isPublic()) return; // Non-public is handled by deployment spec config. if ( ! context.properties().launchApplicationAthenzService()) return; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java index ac4b4623e42a..b112ce23089f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java @@ -62,8 +62,15 @@ private static Map setupProvidedModels() { register(m, "e5-large-v2", "https://data.vespa-cloud.com/onnx_models/e5-large-v2/model.onnx", Set.of(ONNX_MODEL)); register(m, "e5-large-v2-vocab", "https://data.vespa-cloud.com/onnx_models/e5-large-v2/tokenizer.json", Set.of(HF_TOKENIZER)); - register(m, "mistral-7b", "https://data.vespa-cloud.com/gguf_models/mistral-7b-instruct-v0.1.Q6_K.gguf", Set.of(GGUF_MODEL)); - register(m, "mistral-7b-q8", "https://data.vespa-cloud.com/gguf_models/mistral-7b-instruct-v0.1.Q8_0.gguf", Set.of(GGUF_MODEL)); + register(m, "llama-3.2-1b-q4", "https://data.vespa-cloud.com/gguf_models/llama-3.2-1b-instruct-q4_k_m.gguf", Set.of(GGUF_MODEL)); + register(m, "llama-3.2-1b", "https://data.vespa-cloud.com/gguf_models/llama-3.2-1b-instruct-q8_0.gguf", Set.of(GGUF_MODEL)); + register(m, "llama-3.2-3b-q4", "https://data.vespa-cloud.com/gguf_models/llama-3.2-3b-instruct-q4_k_m.gguf", Set.of(GGUF_MODEL)); + register(m, "llama-3.2-3b", "https://data.vespa-cloud.com/gguf_models/llama-3.2-3b-instruct-q8_0.gguf", Set.of(GGUF_MODEL)); + register(m, "mistral-7b", "https://data.vespa-cloud.com/gguf_models/mistral-7b-instruct-v0.1.Q6_K.gguf", Set.of(GGUF_MODEL)); + register(m, "mistral-7b-q8", "https://data.vespa-cloud.com/gguf_models/mistral-7b-instruct-v0.1.Q8_0.gguf", Set.of(GGUF_MODEL)); + register(m, "phi-3.5-mini-q4", "https://data.vespa-cloud.com/gguf_models/Phi-3.5-mini-instruct-Q4_K_M.gguf", Set.of(GGUF_MODEL)); + + register(m, "significance-en-wikipedia-v1", "https://data.vespa-cloud.com/significance_models/significance-en-wikipedia-v1.json.zst", Set.of(SIGNIFICANCE_MODEL)); return Map.copyOf(m); } diff --git a/config-model/src/test/cfg/significance/services.xml b/config-model/src/test/cfg/significance/hosted/services.xml similarity index 85% rename from config-model/src/test/cfg/significance/services.xml rename to config-model/src/test/cfg/significance/hosted/services.xml index ffdb73bfc2e3..258dc685a849 100644 --- a/config-model/src/test/cfg/significance/services.xml +++ b/config-model/src/test/cfg/significance/hosted/services.xml @@ -8,7 +8,7 @@ - + diff --git a/config-model/src/test/cfg/significance/hosts.xml b/config-model/src/test/cfg/significance/selfhosted/hosts.xml similarity index 100% rename from config-model/src/test/cfg/significance/hosts.xml rename to config-model/src/test/cfg/significance/selfhosted/hosts.xml diff --git a/config-model/src/test/cfg/significance/selfhosted/services.xml b/config-model/src/test/cfg/significance/selfhosted/services.xml new file mode 100644 index 000000000000..a66524eea978 --- /dev/null +++ b/config-model/src/test/cfg/significance/selfhosted/services.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java index d5af996bd59b..ebaa0ef37b0e 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java @@ -57,7 +57,7 @@ void requireThatOutputConflictThrows() throws ParseException { } catch (IllegalArgumentException e) { assertEquals("For schema 'indexing_output_confict', field 'bar': For expression 'index bar': Attempting " + - "to assign conflicting values to field 'bar'.", + "to assign conflicting values to field 'bar'", Exceptions.toMessageString(e)); } } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java index 4053834784f8..898f3f5b23d7 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java @@ -36,7 +36,7 @@ void testAttributeChanged() throws ParseException { } catch (IllegalArgumentException e) { assertEquals("For schema 'indexing_attribute_changed', field 'foo': For expression 'attribute foo': " + - "Attempting to assign conflicting values to field 'foo'.", + "Attempting to assign conflicting values to field 'foo'", Exceptions.toMessageString(e)); } } @@ -80,7 +80,7 @@ void testIndexChanged() throws ParseException { } catch (IllegalArgumentException e) { assertEquals("For schema 'indexing_index_changed', field 'foo': For expression 'index foo': " + - "Attempting to assign conflicting values to field 'foo'.", + "Attempting to assign conflicting values to field 'foo'", Exceptions.toMessageString(e)); } } @@ -124,7 +124,7 @@ void testSummaryChanged() throws ParseException { } catch (IllegalArgumentException e) { assertEquals("For schema 'indexing_summary_fail', field 'foo': For expression 'summary foo': Attempting " + - "to assign conflicting values to field 'foo'.", + "to assign conflicting values to field 'foo'", Exceptions.toMessageString(e)); } } @@ -186,7 +186,7 @@ void requireThatMultilineOutputConflictThrows() throws ParseException { } catch (IllegalArgumentException e) { assertEquals("For schema 'indexing_multiline_output_confict', field 'cox': For expression 'index cox': " + - "Attempting to assign conflicting values to field 'cox'.", + "Attempting to assign conflicting values to field 'cox'", Exceptions.toMessageString(e)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index 590433757c35..059411b7a9d2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -49,7 +49,7 @@ void test_deploy_above_quota_budget() { tester.deploy(null, getServices(10), Environment.prod, null, CONTAINER_CLUSTER); fail(); } catch (RuntimeException e) { - assertEquals("The resources used cost $1.63 but your quota is $1.25: Contact support to upgrade your plan.", e.getMessage()); + assertEquals("The resources used cost $1.63 but your remaining quota is $1.25: Contact support to upgrade your plan.", e.getMessage()); } } @@ -66,7 +66,7 @@ void test_deploy_above_quota_budget_in_publiccd() { tester.deploy(null, getServices(10), Environment.prod, null, CONTAINER_CLUSTER); fail(); } catch (RuntimeException e) { - assertEquals("publiccd: The resources used cost $1.63 but your quota is $1.00: Contact support to upgrade your plan.", e.getMessage()); + assertEquals("publiccd: The resources used cost $1.63 but your remaining quota is $1.00: Contact support to upgrade your plan.", e.getMessage()); } } @@ -77,7 +77,7 @@ void test_deploy_max_resources_above_quota() { tester.deploy(null, getServices(10), Environment.prod, null, CONTAINER_CLUSTER); fail(); } catch (RuntimeException e) { - assertEquals("publiccd: The resources used cost $1.63 but your quota is $1.25: Contact support to upgrade your plan.", e.getMessage()); + assertEquals("publiccd: The resources used cost $1.63 but your remaining quota is $1.25: Contact support to upgrade your plan.", e.getMessage()); } } @@ -92,7 +92,7 @@ void test_deploy_above_quota_budget_in_dev() { tester.deploy(null, getServices(2, false), Environment.dev, null, CONTAINER_CLUSTER); fail(); } catch (RuntimeException e) { - assertEquals("The resources used cost $0.16 but your quota is $0.01: Contact support to upgrade your plan.", e.getMessage()); + assertEquals("The resources used cost $0.16 but your remaining quota is $0.01: Contact support to upgrade your plan.", e.getMessage()); } // Override so that we will get 2 nodes in content cluster @@ -100,7 +100,7 @@ void test_deploy_above_quota_budget_in_dev() { tester.deploy(null, getServices(2, true), Environment.dev, null, CONTAINER_CLUSTER); fail(); } catch (RuntimeException e) { - assertEquals("The resources used cost $0.33 but your quota is $0.01: Contact support to upgrade your plan.", e.getMessage()); + assertEquals("The resources used cost $0.33 but your remaining quota is $0.01: Contact support to upgrade your plan.", e.getMessage()); } } @@ -112,7 +112,7 @@ void test_deploy_with_negative_budget() { tester.deploy(null, getServices(10), Environment.prod, null, CONTAINER_CLUSTER); fail(); } catch (RuntimeException e) { - assertEquals("The resources used cost $-.-- but your quota is $--.--: Please free up some capacity.", + assertEquals("The resources used cost $-.-- but your remaining quota is $--.--: Please free up some capacity.", ValidationTester.censorNumbers(e.getMessage())); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java index 26e8c67a226b..2aa10a20f3d7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java @@ -5,47 +5,103 @@ import com.yahoo.config.InnerNode; import com.yahoo.config.ModelNode; import com.yahoo.config.ModelReference; +import com.yahoo.config.model.api.ApplicationClusterEndpoint; +import com.yahoo.config.model.api.ContainerEndpoint; + +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.path.Path; import com.yahoo.search.significance.config.SignificanceConfig; + import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.SignificanceModelRegistry; -import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; + import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Optional; +import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; /** - * @author MariusArhaug + * @author MariusArhaug, glebashnik */ public class SignificanceModelTestCase { - private VespaModel createModel(String filename) { - return new VespaModelCreatorWithFilePkg(filename).create(); - } - @Test - void testIndexGreaterThanNumNodes() { - VespaModel vespaModel = createModel("src/test/cfg/significance"); - ApplicationContainerCluster containerCluster = vespaModel.getContainerClusters().get("container"); - assertEquals(1, containerCluster.getContainers().size()); + void testSignificanceModelConfig_selfhosted() throws Exception { + var vespaModel = loadModel(Path.fromString("src/test/cfg/significance/selfhosted/"), false); + + var cluster = vespaModel.getContainerClusters().get("container"); + assertEquals(1, cluster.getContainers().size()); + + var significanceConfig = assertSignificancePresent(cluster); + assertEquals(3, significanceConfig.model().size()); + + assertModel(significanceConfig.model(0), + Optional.of("significance-en-wikipedia-v1"), + Optional.of("models/significance-en-wikipedia-v1.json.zst"), + Optional.empty() + ); + + assertModel(significanceConfig.model(1), + Optional.empty(), + Optional.of("models/idf-norwegian-wiki.json.zst"), + Optional.empty() + ); + + assertModel(significanceConfig.model(2), + Optional.empty(), + Optional.empty(), + Optional.of("https://some/uri/blob.json") + ); } @Test - void testSignificance() { - VespaModel vespaModel = createModel("src/test/cfg/significance"); - ApplicationContainerCluster containerCluster = vespaModel.getContainerClusters().get("container"); - var significanceConfig = assertSignificancePresent(containerCluster); + void testSignificanceModelConfig_hosted() throws Exception { + var vespaModel = loadModel(Path.fromString("src/test/cfg/significance/hosted/"), true); + + var cluster = vespaModel.getContainerClusters().get("container"); + assertEquals(1, cluster.getContainers().size()); + + var significanceConfig = assertSignificancePresent(cluster); assertEquals(3, significanceConfig.model().size()); - assertEquals("models/idf-norwegian-wiki.json.zst", modelReference(significanceConfig.model().get(1), "path").path().orElseThrow().value()); - assertEquals("https://some/uri/blob.json", modelReference(significanceConfig.model().get(2), "path").url().orElseThrow().value()); + assertModel(significanceConfig.model(0), + Optional.of("significance-en-wikipedia-v1"), + Optional.empty(), + Optional.of("https://data.vespa-cloud.com/significance_models/significance-en-wikipedia-v1.json.zst") + ); + + assertModel(significanceConfig.model(1), + Optional.empty(), + Optional.of("models/idf-norwegian-wiki.json.zst"), + Optional.empty() + ); + assertModel(significanceConfig.model(2), + Optional.empty(), + Optional.empty(), + Optional.of("https://some/uri/blob.json") + ); } - private SignificanceConfig assertSignificancePresent(ApplicationContainerCluster cluster) { + private VespaModel loadModel(Path path, boolean hosted) throws Exception { + FilesApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(path.toFile()); + TestProperties properties = new TestProperties().setHostedVespa(hosted); + DeployState state = new DeployState.Builder() + .properties(properties) + .endpoints(hosted ? Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com"))) : Set.of()) + .applicationPackage(applicationPackage) + .build(); + return new VespaModel(state); + } + private SignificanceConfig assertSignificancePresent(ApplicationContainerCluster cluster) { var id = new ComponentId("com.yahoo.language.significance.impl.DefaultSignificanceModelRegistry"); var significance = (SignificanceModelRegistry) cluster.getComponentsMap().get(id); assertEquals("com.yahoo.language.significance.impl.DefaultSignificanceModelRegistry", significance.getClassId().getName()); @@ -64,5 +120,27 @@ private static ModelReference modelReference(InnerNode cfg, String name) { throw new RuntimeException(e); } } + + void assertModel(SignificanceConfig.Model model, Optional id, Optional path, Optional url) { + var modelReference = modelReference(model, "path"); + var modelId = modelReference.modelId(); + var modelPath = modelReference.path(); + var modelUrl = modelReference.url(); + + id.ifPresentOrElse( + s -> assertEquals(s, modelId.orElseThrow()), + () -> assertEquals(Optional.empty(), modelId) + ); + + path.ifPresentOrElse( + s -> assertEquals(s, modelPath.orElseThrow().value()), + () -> assertEquals(Optional.empty(), modelPath) + ); + + url.ifPresentOrElse( + s -> assertEquals(s, modelUrl.orElseThrow().value()), + () -> assertEquals(Optional.empty(), modelUrl) + ); + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 94235185f450..95ccd9593843 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -113,14 +113,7 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.Set; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.UnaryOperator; import java.util.logging.Level; @@ -662,21 +655,25 @@ public HttpResponse fileDistributionStatus(ApplicationId applicationId, Duration return fileDistributionStatus.status(getApplication(applicationId), timeout); } - public List deleteUnusedFileDistributionReferences(FileDirectory fileDirectory, Duration keepFileReferencesDuration) { + public List deleteUnusedFileDistributionReferences(FileDirectory fileDirectory) { Set fileReferencesInUse = getFileReferencesInUse(); log.log(Level.FINE, () -> "File references in use : " + fileReferencesInUse); - Instant instant = clock.instant().minus(keepFileReferencesDuration); - log.log(Level.FINE, () -> "Remove unused file references last modified before " + instant); - - List fileReferencesToDelete = sortedUnusedFileReferences(fileDirectory.getRoot(), fileReferencesInUse, instant); - // Do max 20 at a time - var toDelete = fileReferencesToDelete.subList(0, Math.min(fileReferencesToDelete.size(), 20)); - if (toDelete.size() > 0) { - log.log(Level.FINE, () -> "Will delete file references not in use: " + toDelete); - toDelete.forEach(fileReference -> fileDirectory.delete(new FileReference(fileReference), this::isFileReferenceInUse)); - log.log(Level.FINE, () -> "Deleted " + toDelete.size() + " file references not in use"); - } - return toDelete; + + List toDelete = sortedUnusedFileReferences(fileDirectory.getRoot(), fileReferencesInUse); + log.log(Level.FINE, () -> "File references not in use: " + toDelete); + List deleted = new ArrayList<>(); + toDelete.forEach(fileReference -> { + if (fileDirectory.delete(new FileReference(fileReference), this::isFileReferenceInUse, this::isFileReferenceOld)) + deleted.add(fileReference); + }); + log.log(Level.FINE, () -> "Deleted " + deleted.size() + " file references not in use"); + return deleted; + } + + private boolean isFileReferenceOld(File file) { + var keepFileReferencesDuration = Duration.ofMinutes(configserverConfig.keepUnusedFileReferencesMinutes()); + var instant = clock.instant().minus(keepFileReferencesDuration); + return isLastModifiedBefore(file, instant); } private boolean isFileReferenceInUse(FileReference fileReference) { @@ -694,14 +691,15 @@ private Set getFileReferencesInUse() { return fileReferencesInUse; } - private List sortedUnusedFileReferences(File fileReferencesPath, Set fileReferencesInUse, Instant instant) { + private List sortedUnusedFileReferences(File fileReferencesPath, Set fileReferencesInUse) { Set fileReferencesOnDisk = getFileReferencesOnDisk(fileReferencesPath); log.log(Level.FINEST, () -> "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); return fileReferencesOnDisk .stream() .filter(fileReference -> ! fileReferencesInUse.contains(fileReference)) - .filter(fileReference -> isLastModifiedBefore(new File(fileReferencesPath, fileReference), instant)) .sorted(Comparator.comparing(a -> lastModified(new File(fileReferencesPath, a)))) + // Do max 20 at a time + .limit(20) .toList(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 5b8c6a9c1a08..18dd95042eee 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -133,12 +133,18 @@ public FileReference addFile(File source) throws IOException { } } - public void delete(FileReference fileReference, Function isInUse) { + public boolean delete(FileReference fileReference, Function isInUse, Function isOld) { try (Lock lock = locks.lock(fileReference)) { if (isInUse.apply(fileReference)) log.log(FINE, "Unable to delete file reference '" + fileReference.value() + "' since it is still in use"); - else + else if ( ! isOld.apply(new File(getRoot(), fileReference.value()))) + log.log(FINE, "Unable to delete file reference '" + fileReference.value() + "' since it is recently used"); + else { deleteDirRecursively(destinationDir(fileReference)); + log.log(FINE, "Deleted file reference '" + fileReference.value() + "'"); + return true; + } + return false; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java index dfd704f98bbb..3d245146ac1c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java @@ -7,9 +7,11 @@ import com.yahoo.container.jdisc.HttpResponse; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.core5.http.ConnectionRequestTimeoutException; import org.apache.hc.core5.util.Timeout; import java.io.IOException; +import java.io.OutputStream; import java.time.Duration; import java.time.Instant; import java.util.Optional; @@ -34,6 +36,8 @@ public HttpResponse getLogs(HttpURL logServerUri, Optional deployTime) HttpGet get = new HttpGet(logServerUri.asURI()); try { return new ProxyResponse(httpClient.execute(get)); + } catch (ConnectionRequestTimeoutException e) { + return new GatewayTimeoutResponse(504); } catch (IOException e) { if (deployTime.isPresent() && Instant.now().isBefore(deployTime.get().plus(Duration.ofMinutes(5)))) return new EmptyResponse(); @@ -42,4 +46,17 @@ public HttpResponse getLogs(HttpURL logServerUri, Optional deployTime) } } + private static class GatewayTimeoutResponse extends HttpResponse { + + public GatewayTimeoutResponse(int status) { super(status); } + + public GatewayTimeoutResponse() { this(504); } + + @Override + public void render(OutputStream outputStream) { + // NOP + } + + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java index 4a0221fdc2cb..14c54fff1b36 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java @@ -19,21 +19,18 @@ public class FileDistributionMaintainer extends ConfigServerMaintainer { private final FileDirectory fileDirectory; - private final Duration maxUnusedFileReferenceAge; FileDistributionMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval, FileDirectory fileDirectory) { super(applicationRepository, curator, applicationRepository.flagSource(), applicationRepository.clock(), interval, false); - ConfigserverConfig configserverConfig = applicationRepository.configserverConfig(); - this.maxUnusedFileReferenceAge = Duration.ofMinutes(configserverConfig.keepUnusedFileReferencesMinutes()); this.fileDirectory = fileDirectory; } @Override protected double maintain() { - applicationRepository.deleteUnusedFileDistributionReferences(fileDirectory, maxUnusedFileReferenceAge); + applicationRepository.deleteUnusedFileDistributionReferences(fileDirectory); return 1.0; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 2142d5652269..30635fd53df2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -115,6 +115,7 @@ public void setup() throws IOException { .configServerDBDir(temporaryFolder.newFolder().getAbsolutePath()) .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) + .keepUnusedFileReferencesMinutes(1) .build(); InMemoryFlagSource flagSource = new InMemoryFlagSource(); fileDirectory = new FileDirectory(configserverConfig); @@ -276,7 +277,6 @@ public void getLogsForHostname() { @Test public void deleteUnusedFileReferences() { File fileReferencesDir = new File(configserverConfig.fileReferencesDir()); - Duration keepFileReferencesDuration = Duration.ofSeconds(4); // Add file reference that is not in use and should be deleted (older than 'keepFileReferencesDuration') File filereferenceDirOldest = createFileReferenceOnDisk(new File(fileReferencesDir, "bar")); @@ -287,7 +287,7 @@ public void deleteUnusedFileReferences() { createFileReferenceOnDisk(new File(fileReferencesDir, "baz" + i)); clock.advance(Duration.ofSeconds(1)); }); - clock.advance(keepFileReferencesDuration); + clock.advance(Duration.ofMinutes(configserverConfig.keepUnusedFileReferencesMinutes())); // Add file reference that is not in use, but should not be deleted (newer than 'keepFileReferencesDuration') File filereferenceDirNewest = createFileReferenceOnDisk(new File(fileReferencesDir, "foo")); @@ -296,13 +296,14 @@ public void deleteUnusedFileReferences() { .withTenantRepository(tenantRepository) .withOrchestrator(orchestrator) .withClock(clock) + .withConfigserverConfig(configserverConfig) .build(); // TODO: Deploy an app with a bundle or file that will be a file reference, too much missing in test setup to get this working now // PrepareParams prepareParams = new PrepareParams.Builder().applicationId(applicationId()).ignoreValidationErrors(true).build(); // deployApp(new File("src/test/apps/app"), prepareParams); - List deleted = applicationRepository.deleteUnusedFileDistributionReferences(fileDirectory, keepFileReferencesDuration); + List deleted = applicationRepository.deleteUnusedFileDistributionReferences(fileDirectory); List expected = List.of("bar", "baz0", "baz1"); assertEquals(expected.stream().sorted().toList(), deleted.stream().sorted().toList()); // bar, baz0 and baz1 will be deleted and foo is not old enough to be considered diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java index 4d085dddd82d..13392fc7b3dd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java @@ -16,12 +16,22 @@ */ public class MockLogRetriever extends LogRetriever { + private final int statuCode; + private final String logLine; + + public MockLogRetriever() { this(200, "log line"); } + + public MockLogRetriever(int statusCode, String logLine) { + this.statuCode = statusCode; + this.logLine = logLine; + } + @Override public HttpResponse getLogs(HttpURL logServerUri, Optional deployTime) { - return new HttpResponse(200) { + return new HttpResponse(statuCode) { @Override public void render(OutputStream outputStream) throws IOException { - outputStream.write("log line".getBytes(StandardCharsets.UTF_8)); + outputStream.write(logLine.getBytes(StandardCharsets.UTF_8)); } }; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployNodeAllocationTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployNodeAllocationTest.java index 675acf807041..0fbb143d6731 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployNodeAllocationTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployNodeAllocationTest.java @@ -83,7 +83,7 @@ public void testExceedsQuota() { .quota(new Quota(Optional.of(4), Optional.of(valueOf(0))))); fail("Expected to get a QuotaExceededException"); } catch (QuotaExceededException e) { - assertEquals("main: The resources used cost $1.02 but your quota is $0.00: Contact support to upgrade your plan.", e.getMessage()); + assertEquals("main: The resources used cost $1.02 but your remaining quota is $0.00: Contact support to upgrade your plan.", e.getMessage()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index fe573ab5c2f2..58658afcb6f9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -108,7 +108,7 @@ public class ApplicationHandlerTest { private final static ApplicationId myTenantApplicationId = ApplicationId.from(mytenantName, ApplicationName.defaultName(), InstanceName.defaultName()); private final static ApplicationId applicationId = ApplicationId.defaultId(); private final static MockTesterClient testerClient = new MockTesterClient(); - private static final MockLogRetriever logRetriever = new MockLogRetriever(); + private MockLogRetriever logRetriever = new MockLogRetriever(); private static final Version vespaVersion = Version.fromString("7.8.9"); private static final SecretStoreValidator secretStoreValidator = new MockSecretStoreValidator(); @@ -462,6 +462,20 @@ public void testGetLogs() throws IOException { assertEquals("log line", getRenderedString(response)); } + @Test + public void testGetLogsConnectionTimeout() throws IOException { + logRetriever = new MockLogRetriever(504, "Connection error when getting logs"); + setup(); + applicationRepository.deploy(new File("src/test/apps/app-logserver-with-container"), prepareParams(applicationId)); + String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/logs?from=100&to=200"; + ApplicationHandler mockHandler = createApplicationHandler(); + + HttpResponse response = mockHandler.handle(createTestRequest(url, GET)); + assertEquals(504, response.getStatus()); + + assertEquals("Connection error when getting logs", getRenderedString(response)); + } + @Test public void testValidateSecretStore() throws IOException { applicationRepository.deploy(new File("src/test/apps/app-logserver-with-container"), prepareParams(applicationId)); diff --git a/container-core/src/main/java/com/yahoo/restapi/ErrorResponse.java b/container-core/src/main/java/com/yahoo/restapi/ErrorResponse.java index 2f329524c2b7..8050f6afd848 100644 --- a/container-core/src/main/java/com/yahoo/restapi/ErrorResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/ErrorResponse.java @@ -7,6 +7,7 @@ import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; import static com.yahoo.jdisc.Response.Status.CONFLICT; import static com.yahoo.jdisc.Response.Status.FORBIDDEN; +import static com.yahoo.jdisc.Response.Status.GATEWAY_TIMEOUT; import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR; import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED; import static com.yahoo.jdisc.Response.Status.NOT_FOUND; @@ -25,6 +26,7 @@ public enum errorCodes { FORBIDDEN, METHOD_NOT_ALLOWED, INTERNAL_SERVER_ERROR, + GATEWAY_TIMEOUT, UNAUTHORIZED, CONFLICT } @@ -49,6 +51,10 @@ public static ErrorResponse internalServerError(String message) { return new ErrorResponse(INTERNAL_SERVER_ERROR, errorCodes.INTERNAL_SERVER_ERROR.name(), message); } + public static ErrorResponse gatewayTimeout(String message) { + return new ErrorResponse(GATEWAY_TIMEOUT, errorCodes.GATEWAY_TIMEOUT.name(), message); + } + public static ErrorResponse badRequest(String message) { return new ErrorResponse(BAD_REQUEST, errorCodes.BAD_REQUEST.name(), message); } diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiException.java b/container-core/src/main/java/com/yahoo/restapi/RestApiException.java index 3a44bc4da5fd..51964d5fc755 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApiException.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiException.java @@ -69,6 +69,11 @@ public static class InternalServerError extends RestApiException { public InternalServerError(String message, Throwable cause) { super(ErrorResponse::internalServerError, message, cause); } } + public static class GatewayTimeout extends RestApiException { + public GatewayTimeout(String message) { this(message, null); } + public GatewayTimeout(String message, Throwable cause) { super(ErrorResponse::gatewayTimeout, message, cause); } + } + public static class Forbidden extends RestApiException { public Forbidden() { this("Forbidden"); } public Forbidden(String message) { super(ErrorResponse::forbidden, message, null); } diff --git a/container-disc/src/main/java/ai/vespa/secret/model/Role.java b/container-disc/src/main/java/ai/vespa/secret/model/Role.java index 8d8e3baf2dcb..340499abb27b 100644 --- a/container-disc/src/main/java/ai/vespa/secret/model/Role.java +++ b/container-disc/src/main/java/ai/vespa/secret/model/Role.java @@ -19,13 +19,6 @@ public String value() { return value; } - public String forVault(VaultName vault) { - return switch(this) { - case WRITER, READER -> vault.value() + "-" + value; - case TENANT_SECRET_WRITER -> value; - }; - } - @Override public String toString() { return Role.class.getSimpleName() + "." + value; diff --git a/container-disc/src/main/resources/configdefinitions/asm-secret.def b/container-disc/src/main/resources/configdefinitions/asm-secret.def index 3b29b603af4b..268060131aea 100644 --- a/container-disc/src/main/resources/configdefinitions/asm-secret.def +++ b/container-disc/src/main/resources/configdefinitions/asm-secret.def @@ -1,6 +1,11 @@ -# Config for AsmSecretStore +# Config for ASM secret readers package=ai.vespa.secret.config.aws ztsUri string athenzDomain string default="" + +# TODO: move to a separaet config (and remove defaults). Only used by AsmTenantSecretReader +# Used to create the athenz role name when retrieving secrets on behalf of a tenant +tenant string default="" +system string default="" diff --git a/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java b/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java index 3ec40fc9e1cb..2bc74ef70a00 100755 --- a/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java +++ b/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java @@ -279,10 +279,11 @@ void onCompletion() { } String report() { + long generationTime = timeToLastToken - timeToFirstToken; return "Time to first token: " + timeToFirstToken + " ms, " + - "Generation time: " + timeToLastToken + " ms, " + + "Generation time: " + generationTime + " ms, " + "Generated tokens: " + tokens + " " + - String.format("(%.2f tokens/sec)", tokens / (timeToLastToken / 1000.0)); + String.format("(%.2f tokens/sec)", tokens / (generationTime / 1000.0)); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/CoverageAggregator.java b/container-search/src/main/java/com/yahoo/search/dispatch/CoverageAggregator.java index f22dc3f3c5d0..21c3b0ce94d2 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/CoverageAggregator.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/CoverageAggregator.java @@ -86,7 +86,9 @@ public CoverageAggregator adjustedDegradedCoverage(int redundancy, TimeoutHandle if (missingNodes > 0) { clone.adjustActiveDocs(missingNodes); } - clone.timedOut = true; + if (timeoutHandler.reason() == DEGRADED_BY_TIMEOUT) { + clone.timedOut = true; + } return clone; } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java index 933a673c1bc8..6528c5d2ae40 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java @@ -6,18 +6,12 @@ import java.util.Collection; import java.util.Map; import java.util.Set; -import java.util.logging.Logger; - -import static java.util.logging.Level.FINE; /** * @author baldersheim */ public class SearchGroupsImpl implements SearchGroups { - private static final Logger log = Logger.getLogger(SearchGroupsImpl.class.getName()); - - private final Map groups; private final double minActiveDocsPercentage; @@ -41,8 +35,6 @@ public boolean isPartialGroupCoverageSufficient(boolean currentIsGroupCoverageSu public boolean isGroupCoverageSufficient(boolean currentIsGroupCoverageSufficient, long groupDocumentCount, long medianDocumentCount, long maxDocumentCount) { - log.log(FINE, () -> String.format("Group doc count: %d, max doc count: %d, median: %d, minActiveDocsPercentage: %.2f", - groupDocumentCount, maxDocumentCount, medianDocumentCount, minActiveDocsPercentage)); if (medianDocumentCount <= 0) return true; if (currentIsGroupCoverageSufficient) { // To take a group *out of* rotation, require that it has less active documents than the median. diff --git a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java index e319740f7416..ad6205a80506 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java @@ -149,7 +149,7 @@ private static List createHitOrderingClause(Sorting sortingS case DESCENDING -> // When we sort in descending order, the hit with the largest value should come first (and be surfaced). orderingClause.add(new NegFunction(new MaxAggregator(new AttributeValue(fieldOrder.getFieldName())))); - default -> throw new UnsupportedOperationException("Can not handle sort order " + sortOrder + "."); + default -> throw new IllegalStateException("Can not handle sort order " + sortOrder + "."); } } return orderingClause; @@ -170,7 +170,7 @@ private static GroupingExpression createGroupOrderingClause(Sorting sortingSpec) case DESCENDING -> // To sort descending, just take the negative. This is the most common case new NegFunction(new AttributeValue(fieldOrder.getFieldName())); - default -> throw new UnsupportedOperationException("Can not handle sort order " + sortOrder + "."); + default -> throw new IllegalStateException("Can not handle sort order " + sortOrder + "."); }; } return groupingClause; diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/TimeFunctions.java b/container-search/src/main/java/com/yahoo/search/grouping/request/TimeFunctions.java index 577ac72a56bd..11bdc19f23bb 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/TimeFunctions.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/request/TimeFunctions.java @@ -1,6 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.request; +import com.yahoo.processing.IllegalInputException; + /** * This abstract class is a factory for timestamp functions in a {@link GroupingExpression}. Apart from offering * per-function factory methods, this class also contains a {@link #newInstance(com.yahoo.search.grouping.request.TimeFunctions.Type, diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java index c967f9ca1fb3..9fb6bcb7fda5 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.vespa; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.grouping.request.AddFunction; import com.yahoo.search.grouping.request.AggregatorNode; import com.yahoo.search.grouping.request.AndFunction; @@ -206,7 +207,7 @@ public AggregationResult toAggregationResult(GroupingExpression exp) { if (exp instanceof CountAggregator) { return new ExpressionCountAggregationResult(); } - throw new UnsupportedOperationException( + throw new IllegalInputException( "Can not aggregate on " + GroupingOperation.getLevelDesc(level) + "."); } if (exp instanceof AvgAggregator) { @@ -243,7 +244,7 @@ public AggregationResult toAggregationResult(GroupingExpression exp) { return new XorAggregationResult() .setExpression(toExpressionNode(((XorAggregator)exp).getExpression())); } - throw new UnsupportedOperationException("Can not convert '" + exp + "' to an aggregator."); + throw new IllegalInputException("Can not convert '" + exp + "' to an aggregator."); } /** @@ -251,7 +252,7 @@ public AggregationResult toAggregationResult(GroupingExpression exp) { * * @param exp The expression to convert. * @return The corresponding back-end expression. - * @throws UnsupportedOperationException Thrown if the given expression could not be converted. + * @throws IllegalInputException Thrown if the given expression could not be converted. */ public ExpressionNode toExpressionNode(GroupingExpression exp) { if (exp instanceof AddFunction) { @@ -520,8 +521,8 @@ public ExpressionNode toExpressionNode(GroupingExpression exp) { return new XorBitFunctionNode().setNumBits(((XorBitFunction)exp).getNumBits()) .addArg(toExpressionNode(((XorBitFunction)exp).getArg(0))); } - throw new UnsupportedOperationException("Can not convert '" + exp + "' of class " + exp.getClass().getName() + - " to an expression."); + throw new IllegalInputException("Can not convert '" + exp + "' of class " + exp.getClass().getName() + + " to an expression."); } private TimeStampFunctionNode toTime(GroupingExpression arg, TimeStampFunctionNode.TimePart timePart) { @@ -576,7 +577,7 @@ private ResultNodeVector toBucketList(PredefinedFunction fnc) { private BucketResultNode toBucket(GroupingExpression exp) { if (!(exp instanceof BucketValue)) { - throw new UnsupportedOperationException("Can not convert '" + exp + "' to a bucket."); + throw new IllegalInputException("Can not convert '" + exp + "' to a bucket."); } ConstantValue begin = ((BucketValue)exp).getFrom(); ConstantValue end = ((BucketValue)exp).getTo(); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingTransform.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingTransform.java index cac544d63abd..47b332feee0c 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingTransform.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingTransform.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.vespa; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.grouping.Continuation; import java.util.HashMap; @@ -90,8 +91,8 @@ public GroupingTransform putLabel(int parentTag, int tag, String label, String t } else { for (Integer sibling : siblings) { if (label.equals(labels.get(sibling))) { - throw new UnsupportedOperationException("Can not use " + type + " label '" + label + - "' for multiple siblings."); + throw new IllegalInputException("Can not use " + type + " label '" + label + + "' for multiple siblings."); } } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/RequestBuilder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/RequestBuilder.java index 73b68ebff7ba..0aa5d315c687 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/RequestBuilder.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/RequestBuilder.java @@ -116,7 +116,7 @@ public List getRequestList() { * This method might fail due to unsupported constructs in the request, in which case an exception is thrown. * * @throws IllegalStateException If this method is called more than once. - * @throws UnsupportedOperationException If the grouping request contains unsupported constructs. + * @throws IllegalInputException If the grouping request contains unsupported constructs. */ public void build() { if (tag != 0) { @@ -173,7 +173,7 @@ OptionalLong totalGroupsAndSummaries() { private void processRequestNode(BuildFrame frame) { int level = frame.astNode.getLevel(); if (level > 2) { - throw new UnsupportedOperationException("Can not operate on " + + throw new IllegalInputException("Can not operate on " + GroupingOperation.getLevelDesc(level) + "."); } if (frame.astNode instanceof EachOperation) { @@ -243,7 +243,7 @@ private void resolveGroupBy(BuildFrame frame) { GroupingExpression exp = frame.astNode.getGroupBy(); if (exp != null) { if (frame.state.groupBy != null) { - throw new UnsupportedOperationException("Can not group list of groups."); + throw new IllegalInputException("Can not group list of groups."); } frame.state.groupBy = converter.toExpressionNode(exp); frame.state.label = exp.toString(); // label for next each() @@ -255,7 +255,7 @@ private void resolveGroupBy(BuildFrame frame) { } else if (level == 1) { frame.state.label = "hits"; // next each() is hitlist } else { - throw new UnsupportedOperationException("Can not create anonymous " + + throw new IllegalInputException("Can not create anonymous " + GroupingOperation.getLevelDesc(level) + "."); } } @@ -289,7 +289,7 @@ private void resolveOrderBy(BuildFrame frame) { } int reqLevel = frame.astNode.getLevel(); if (reqLevel != 2) { - throw new UnsupportedOperationException( + throw new IllegalInputException( "Can not order " + GroupingOperation.getLevelDesc(reqLevel) + " content."); } for (GroupingExpression exp : lst) { @@ -321,7 +321,7 @@ private AggregationResult toAggregationResult(GroupingExpression exp, Group grou String label = exp.getLabel(); if (result instanceof HitsAggregationResult hits) { if (label != null) { - throw new UnsupportedOperationException("Can not label expression '" + exp + "'."); + throw new IllegalInputException("Can not label expression '" + exp + "'."); } if (frame.state.max != null) { transform.putMax(tag, frame.state.max, "hit list"); @@ -349,7 +349,7 @@ private void resolveWhere(BuildFrame frame) { String where = frame.astNode.getWhere(); if (where != null) { if (!isRootOperation(frame)) { - throw new UnsupportedOperationException("Can not apply 'where' to non-root group."); + throw new IllegalInputException("Can not apply 'where' to non-root group."); } switch (where) { case "true": @@ -359,7 +359,7 @@ private void resolveWhere(BuildFrame frame) { // ignore break; default: - throw new UnsupportedOperationException("Operation 'where' does not support '" + where + "'."); + throw new IllegalInputException("Operation 'where' does not support '" + where + "'."); } } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java index e8b6d4f72aab..a78dcc4ec21d 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java @@ -2,6 +2,7 @@ package com.yahoo.search.grouping.vespa; import com.yahoo.prelude.hitfield.RawBase64; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.grouping.Continuation; import com.yahoo.search.grouping.GroupingRequest; import com.yahoo.search.grouping.result.BoolId; @@ -134,12 +135,12 @@ public Continuation getContinuation() { * Constructs the grouping result tree that corresponds to the parameters given to this builder. This method might * fail due to unsupported constructs in the results, in which case an exception is thrown. * - * @throws UnsupportedOperationException Thrown if the grouping result contains unsupported constructs. + * @throws IllegalInputException Thrown if the grouping result contains unsupported constructs. */ public void build() { int numChildren = rootBuilder.childGroups.size(); if (numChildren != 1) { - throw new UnsupportedOperationException("Expected 1 group, got " + numChildren + "."); + throw new IllegalInputException("Expected 1 group, got " + numChildren + "."); } rootBuilder.childGroups.get(0).fill(root); } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java index fff992583092..21c1e8c7b149 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java @@ -9,6 +9,7 @@ import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.query.NullItem; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -122,7 +123,7 @@ void requireThatAggregationPerHitWithoutGroupingDoesNotWorkYet() { try { execute("each(output(strlen(customer)))"); fail(); - } catch (UnsupportedOperationException e) { + } catch (IllegalInputException e) { } } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingTransformTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingTransformTestCase.java index 7dd4416163f9..77036fd0ffa6 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingTransformTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingTransformTestCase.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.vespa; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.grouping.Continuation; import org.junit.jupiter.api.Test; @@ -40,7 +41,7 @@ void requireThatLabelIsUniqueAmongSiblings() { try { transform.putLabel(0, 2, "foo", "my_type"); fail(); - } catch (UnsupportedOperationException e) { + } catch (IllegalInputException e) { assertEquals("Can not use my_type label 'foo' for multiple siblings.", e.getMessage()); } diff --git a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java index e33cf9d958fc..b812817064ca 100644 --- a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java @@ -536,4 +536,4 @@ public void testQueryWithDocumentFrequencyAndSignificance() { var resultSignificance = firstWord.getSignificance(); assertEquals(querySignificance.get(), resultSignificance); } -} \ No newline at end of file +} diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index fb13d4706e0a..eb5cc01f3762 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -110,7 +110,7 @@ 4.4.0 1.2 4.0.5 - 11.0.22 + 11.0.24 5.0.2 1.0.2 1.3.0 @@ -142,7 +142,7 @@ 4.0.1 4.0.4 2.8.0 - 3.25.4 + 3.25.5 7.4.2 1.3.7 1.1.10.6 diff --git a/document/abi-spec.json b/document/abi-spec.json index 89c0e4e27196..ba2313dcf0b1 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -200,6 +200,7 @@ "public int getCode()", "public com.yahoo.document.FieldPath buildFieldPath(java.lang.String)", "public com.yahoo.document.PrimitiveDataType getPrimitiveType()", + "public com.yahoo.document.DataType getNestedType()", "public void visitMembers(com.yahoo.vespa.objects.ObjectVisitor)", "public int compareTo(com.yahoo.document.DataType)", "public boolean isMultivalue()", diff --git a/document/src/main/java/com/yahoo/document/ArrayDataType.java b/document/src/main/java/com/yahoo/document/ArrayDataType.java index b14e6cd8a424..05b578f74457 100644 --- a/document/src/main/java/com/yahoo/document/ArrayDataType.java +++ b/document/src/main/java/com/yahoo/document/ArrayDataType.java @@ -24,10 +24,12 @@ public ArrayDataType(DataType nestedType, int code) { super("Array<"+nestedType.getName()+">", code, nestedType); } + @Override public ArrayDataType clone() { return (ArrayDataType) super.clone(); } + @Override public Array createFieldValue() { return new Array(this); } diff --git a/document/src/main/java/com/yahoo/document/CollectionDataType.java b/document/src/main/java/com/yahoo/document/CollectionDataType.java index d580e27d4371..98be8d4709f5 100644 --- a/document/src/main/java/com/yahoo/document/CollectionDataType.java +++ b/document/src/main/java/com/yahoo/document/CollectionDataType.java @@ -33,9 +33,8 @@ public CollectionDataType clone() { return type; } - public DataType getNestedType() { - return nestedType; - } + @Override + public DataType getNestedType() { return nestedType; } @Override protected FieldValue createByReflection(Object arg) { return null; } diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java index 64bd462fb34d..7a49938334fa 100644 --- a/document/src/main/java/com/yahoo/document/DataType.java +++ b/document/src/main/java/com/yahoo/document/DataType.java @@ -291,9 +291,10 @@ public FieldPath buildFieldPath(String fieldPathString) { * * @return primitive data type, or null */ - public PrimitiveDataType getPrimitiveType() { - return null; - } + public PrimitiveDataType getPrimitiveType() { return null; } + + /** Returns the nested type of this if it is a collection type, null otherwise. */ + public DataType getNestedType() { return null; } @Override public void visitMembers(ObjectVisitor visitor) { diff --git a/eval/src/apps/eval_expr/eval_expr.cpp b/eval/src/apps/eval_expr/eval_expr.cpp index b66d77cbb5f6..1b7dcb3f7729 100644 --- a/eval/src/apps/eval_expr/eval_expr.cpp +++ b/eval/src/apps/eval_expr/eval_expr.cpp @@ -31,6 +31,8 @@ using vespalib::Slime; using vespalib::slime::JsonFormat; using vespalib::slime::Inspector; using vespalib::slime::Cursor; +using vespalib::Input; +using vespalib::Memory; using CostProfile = std::vector>; @@ -157,20 +159,21 @@ class Context const CTFMetaData &meta() const { return _meta; } const CostProfile &cost() const { return _cost; } - void save(const std::string &name, Value::UP value) { + bool save(const std::string &name, Value::UP value) { REQUIRE(value); for (size_t i = 0; i < _param_names.size(); ++i) { if (_param_names[i] == name) { _param_types[i] = value->type(); _param_values[i] = std::move(value); _param_refs[i] = *_param_values[i]; - return; + return true; } } _param_names.push_back(name); _param_types.push_back(value->type()); _param_values.push_back(std::move(value)); _param_refs.emplace_back(*_param_values.back()); + return false; } bool remove(const std::string &name) { @@ -247,24 +250,92 @@ void handle_message(Context &ctx, const Inspector &req, Cursor &reply) { } } -bool should_ignore(const std::string &str) { +bool is_hash_bang(const std::string &str) { + if (str.size() > 2) { + return str[0] == '#' && str[1] == '!'; + } + return false; +} + +bool is_only_whitespace(const std::string &str) { for (auto c : str) { if (!std::isspace(static_cast(c))) { - return (c == '#'); + return false; } } return true; } -struct Script { - vespalib::MappedFileInput input; - LineReader reader; - Script(const std::string &file_name) - : input(file_name), reader(input) - { - if (!input.valid()) { +class Script { +private: + std::unique_ptr _input; + LineReader _reader; + bool _script_only = false; +public: + Script(std::unique_ptr input) + : _input(std::move(input)), _reader(*_input) {} + static auto empty() { + struct EmptyInput : Input { + Memory obtain() override { return Memory(); } + Input &evict(size_t) override { return *this; } + }; + return std::make_unique