From fdd8c6b44e8a52e3b6f5d94bac4c8a717fccb5f4 Mon Sep 17 00:00:00 2001 From: Frederic Gurr Date: Wed, 6 Dec 2023 19:32:07 +0100 Subject: [PATCH] https://ci.eclipse.org/egit/: ssh key went missing ? https://gitlab.eclipse.org/eclipsefdn/helpdesk/-/issues/4041 --- .../target/k8s/statefulset.json | 11 + .../target/jenkins/plugins.log | 2 +- .../jenkins/tools/.install-plugins.sh.etag | 1 - .../target/jenkins/tools/install-plugins.sh | 292 ------------------ .../target/k8s/statefulset.json | 11 + templates/k8s/statefulset.libsonnet | 16 +- 6 files changed, 31 insertions(+), 302 deletions(-) delete mode 100644 instances/technology.jgit/target/jenkins/tools/.install-plugins.sh.etag delete mode 100755 instances/technology.jgit/target/jenkins/tools/install-plugins.sh diff --git a/instances/technology.egit/target/k8s/statefulset.json b/instances/technology.egit/target/k8s/statefulset.json index 2aaf92c94..69281e484 100644 --- a/instances/technology.egit/target/k8s/statefulset.json +++ b/instances/technology.egit/target/k8s/statefulset.json @@ -141,6 +141,11 @@ "mountPath": "/run/secrets/jenkins/ssh", "name": "gerrit-ssh-keys", "readOnly": true + }, + { + "mountPath": "/run/secrets/jenkins/ssh-gerrithub.io", + "name": "gerrithub-io-ssh-keys", + "readOnly": true } ] } @@ -167,6 +172,12 @@ "secret": { "secretName": "gerrit-ssh-keys" } + }, + { + "name": "gerrithub-io-ssh-keys", + "secret": { + "secretName": "gerrithub.io-ssh-keys" + } } ] } diff --git a/instances/technology.jgit/target/jenkins/plugins.log b/instances/technology.jgit/target/jenkins/plugins.log index 2d1205a2e..02b38dea5 100644 --- a/instances/technology.jgit/target/jenkins/plugins.log +++ b/instances/technology.jgit/target/jenkins/plugins.log @@ -7,7 +7,7 @@ apache-httpcomponents-client-4-api 4.5.14-208.v438351942757 authentication-tokens 1.53.v1c90fd9191a_b_ authorize-project 1.7.1 bootstrap5-api 5.3.2-3 -bouncycastle-api 2.29 +bouncycastle-api 2.30.1.77-225.v26ea_c9455fd9 branch-api 2.1135.v8de8e7899051 build-timeout 1.31 caffeine-api 3.1.8-133.v17b_1ff2e0599 diff --git a/instances/technology.jgit/target/jenkins/tools/.install-plugins.sh.etag b/instances/technology.jgit/target/jenkins/tools/.install-plugins.sh.etag deleted file mode 100644 index 7e988f05a..000000000 --- a/instances/technology.jgit/target/jenkins/tools/.install-plugins.sh.etag +++ /dev/null @@ -1 +0,0 @@ -"c3f0fec63b3f9e55ba4f308072f2c788ca7d844e29527c9d466a8a2d8bad18c4" \ No newline at end of file diff --git a/instances/technology.jgit/target/jenkins/tools/install-plugins.sh b/instances/technology.jgit/target/jenkins/tools/install-plugins.sh deleted file mode 100755 index b2849e376..000000000 --- a/instances/technology.jgit/target/jenkins/tools/install-plugins.sh +++ /dev/null @@ -1,292 +0,0 @@ -#!/bin/bash -eu - -# Resolve dependencies and download plugins given on the command line -# -# FROM jenkins -# RUN install-plugins.sh docker-slaves github-branch-source -# -# Environment variables: -# REF: directory with preinstalled plugins. Default: /usr/share/jenkins/ref/plugins -# JENKINS_WAR: full path to the jenkins.war. Default: /usr/share/jenkins/jenkins.war -# JENKINS_UC: url of the Update Center. Default: "" -# JENKINS_UC_EXPERIMENTAL: url of the Experimental Update Center for experimental versions of plugins. Default: "" -# JENKINS_INCREMENTALS_REPO_MIRROR: url of the incrementals repo mirror. Default: "" -# JENKINS_UC_DOWNLOAD: download url of the Update Center. Default: JENKINS_UC/download -# CURL_OPTIONS When downloading the plugins with curl. Curl options. Default: -sSfL -# CURL_CONNECTION_TIMEOUT When downloading the plugins with curl. Maximum time allowed for connection. Default: 20 -# CURL_RETRY When downloading the plugins with curl. Retry request if transient problems occur. Default: 3 -# CURL_RETRY_DELAY When downloading the plugins with curl. Wait time between retries. Default: 0 -# CURL_RETRY_MAX_TIME When downloading the plugins with curl. Retry only within this period. Default: 60 - -set -o pipefail - -echo "WARN: install-plugins.sh is deprecated, please switch to jenkins-plugin-cli" - -JENKINS_WAR=${JENKINS_WAR:-/usr/share/jenkins/jenkins.war} - -. /usr/local/bin/jenkins-support - -REF_DIR="${REF}/plugins" -FAILED="$REF_DIR/failed-plugins.txt" - -getLockFile() { - printf '%s' "$REF_DIR/${1}.lock" -} - -getArchiveFilename() { - printf '%s' "$REF_DIR/${1}.jpi" -} - -download() { - local plugin originalPlugin version lock ignoreLockFile url - plugin="$1" - version="${2:-latest}" - ignoreLockFile="${3:-}" - url="${4:-}" - lock="$(getLockFile "$plugin")" - - if [[ $ignoreLockFile ]] || mkdir "$lock" &>/dev/null; then - if ! doDownload "$plugin" "$version" "$url"; then - # some plugin don't follow the rules about artifact ID - # typically: docker-plugin - originalPlugin="$plugin" - plugin="${plugin}-plugin" - if ! doDownload "$plugin" "$version" "$url"; then - echo "Failed to download plugin: $originalPlugin or $plugin" >&2 - echo "Not downloaded: ${originalPlugin}" >> "$FAILED" - return 1 - fi - fi - - if ! checkIntegrity "$plugin"; then - echo "Downloaded file is not a valid ZIP: $(getArchiveFilename "$plugin")" >&2 - echo "Download integrity: ${plugin}" >> "$FAILED" - return 1 - fi - - resolveDependencies "$plugin" - fi -} - -doDownload() { - local plugin version url jpi - plugin="$1" - version="$2" - url="$3" - jpi="$(getArchiveFilename "$plugin")" - - # If plugin already exists and is the same version do not download - if test -f "$jpi" && unzip -p "$jpi" META-INF/MANIFEST.MF | tr -d '\r' | grep "^Plugin-Version: ${version}$" > /dev/null; then - echo "Using provided plugin: $plugin" - return 0 - fi - - if [[ -n $url ]] ; then - echo "Will use url=$url" - elif [[ "$version" == "latest" && -n "$JENKINS_UC_LATEST" ]]; then - # If version-specific Update Center is available, which is the case for LTS versions, - # use it to resolve latest versions. - url="$JENKINS_UC_LATEST/latest/${plugin}.hpi" - elif [[ "$version" == "experimental" && -n "$JENKINS_UC_EXPERIMENTAL" ]]; then - # Download from the experimental update center - url="$JENKINS_UC_EXPERIMENTAL/latest/${plugin}.hpi" - elif [[ "$version" == incrementals* ]] ; then - # Download from Incrementals repo: https://jenkins.io/blog/2018/05/15/incremental-deployment/ - # Example URL: https://repo.jenkins-ci.org/incrementals/org/jenkins-ci/plugins/workflow/workflow-support/2.19-rc289.d09828a05a74/workflow-support-2.19-rc289.d09828a05a74.hpi - local groupId incrementalsVersion - # add a trailing ; so the \n gets added to the end - readarray -t "-d;" arrIN <<<"${version};"; - unset 'arrIN[-1]'; - groupId=${arrIN[1]} - incrementalsVersion=${arrIN[2]} - url="${JENKINS_INCREMENTALS_REPO_MIRROR}/$(echo "${groupId}" | tr '.' '/')/${plugin}/${incrementalsVersion}/${plugin}-${incrementalsVersion}.hpi" - else - JENKINS_UC_DOWNLOAD=${JENKINS_UC_DOWNLOAD:-"$JENKINS_UC/download"} - url="$JENKINS_UC_DOWNLOAD/plugins/$plugin/$version/${plugin}.hpi" - fi - - echo "Downloading plugin: $plugin from $url" - # We actually want to allow variable value to be split into multiple options passed to curl. - # This is needed to allow long options and any options that take value. - # shellcheck disable=SC2086 - retry_command curl ${CURL_OPTIONS:--sSfL} --connect-timeout "${CURL_CONNECTION_TIMEOUT:-20}" --retry "${CURL_RETRY:-3}" --retry-delay "${CURL_RETRY_DELAY:-0}" --retry-max-time "${CURL_RETRY_MAX_TIME:-60}" "$url" -o "$jpi" - return $? -} - -checkIntegrity() { - local plugin jpi - plugin="$1" - jpi="$(getArchiveFilename "$plugin")" - - unzip -t -qq "$jpi" >/dev/null - return $? -} - -resolveDependencies() { - local plugin jpi dependencies - plugin="$1" - jpi="$(getArchiveFilename "$plugin")" - - dependencies="$(unzip -p "$jpi" META-INF/MANIFEST.MF | tr -d '\r' | tr '\n' '|' | sed -e 's#| ##g' | tr '|' '\n' | grep "^Plugin-Dependencies: " | sed -e 's#^Plugin-Dependencies: ##')" - - if [[ ! $dependencies ]]; then - echo " > $plugin has no dependencies" - return - fi - - echo " > $plugin depends on $dependencies" - - IFS=',' read -r -a array <<< "$dependencies" - - for d in "${array[@]}" - do - plugin="$(cut -d':' -f1 - <<< "$d")" - if [[ $d == *"resolution:=optional"* ]]; then - echo "Skipping optional dependency $plugin" - else - local pluginInstalled - if pluginInstalled="$(echo -e "${bundledPlugins}\n${installedPlugins}" | grep "^${plugin}:")"; then - pluginInstalled="${pluginInstalled//[$'\r']}" - local versionInstalled; versionInstalled=$(versionFromPlugin "${pluginInstalled}") - local minVersion; minVersion=$(versionFromPlugin "${d}") - if versionLT "${versionInstalled}" "${minVersion}"; then - echo "Upgrading bundled dependency $d ($minVersion > $versionInstalled)" - download "$plugin" & - else - echo "Skipping already installed dependency $d ($minVersion <= $versionInstalled)" - fi - else - download "$plugin" & - fi - fi - done - wait -} - -bundledPlugins() { - if [ -f "$JENKINS_WAR" ] - then - TEMP_PLUGIN_DIR=/tmp/plugintemp.$$ - for i in $(jar tf "$JENKINS_WAR" | grep -E '[^detached-]plugins.*\..pi' | sort) - do - rm -fr $TEMP_PLUGIN_DIR - mkdir -p $TEMP_PLUGIN_DIR - PLUGIN=$(basename "$i"|cut -f1 -d'.') - (cd $TEMP_PLUGIN_DIR;jar xf "$JENKINS_WAR" "$i";jar xvf "$TEMP_PLUGIN_DIR/$i" META-INF/MANIFEST.MF >/dev/null 2>&1) - VER=$(grep -E -i Plugin-Version "$TEMP_PLUGIN_DIR/META-INF/MANIFEST.MF"|cut -d: -f2|sed 's/ //') - echo "$PLUGIN:$VER" - done - rm -fr $TEMP_PLUGIN_DIR - else - echo "war not found, installing all plugins: $JENKINS_WAR" - fi -} - -versionFromPlugin() { - local plugin=$1 - if [[ $plugin =~ .*:.* ]]; then - echo "${plugin##*:}" - else - echo "latest" - fi - -} - -installedPlugins() { - for f in "$REF_DIR"/*.jpi; do - echo "$(basename "$f" | sed -e 's/\.jpi//'):$(get_plugin_version "$f")" - done -} - -jenkinsMajorMinorVersion() { - if [[ -f "$JENKINS_WAR" ]]; then - local version major minor - version="$(java -jar "$JENKINS_WAR" --version)" - major="$(echo "$version" | cut -d '.' -f 1)" - minor="$(echo "$version" | cut -d '.' -f 2)" - echo "$major.$minor" - else - echo "" - fi -} - -main() { - local plugin jenkinsVersion - local plugins=() - - mkdir -p "$REF_DIR" || exit 1 - rm -f "$FAILED" - - # Read plugins from stdin or from the command line arguments - if [[ ($# -eq 0) ]]; then - while read -r line || [ "$line" != "" ]; do - # Remove leading/trailing spaces, comments, and empty lines - plugin=$(echo "${line}" | tr -d '\r' | sed -e 's/^[ \t]*//g' -e 's/[ \t]*$//g' -e 's/[ \t]*#.*$//g' -e '/^[ \t]*$/d') - - # Avoid adding empty plugin into array - if [ ${#plugin} -ne 0 ]; then - plugins+=("${plugin}") - fi - done - else - plugins=("$@") - fi - - # Create lockfile manually before first run to make sure any explicit version set is used. - echo "Creating initial locks..." - for plugin in "${plugins[@]}"; do - mkdir "$(getLockFile "${plugin%%:*}")" - done - - echo "Analyzing war $JENKINS_WAR..." - bundledPlugins="$(bundledPlugins)" - - echo "Registering preinstalled plugins..." - installedPlugins="$(installedPlugins)" - - # Get the update center URL based on the jenkins version - jenkinsVersion="$(jenkinsMajorMinorVersion)" - # shellcheck disable=SC2086 - jenkinsUcJson=$(curl ${CURL_OPTIONS:--sSfL} -o /dev/null -w "%{url_effective}" "${JENKINS_UC}/update-center.json?version=${jenkinsVersion}") - if [ -n "${jenkinsUcJson}" ]; then - JENKINS_UC_LATEST=${jenkinsUcJson//update-center.json/} - echo "Using version-specific update center: $JENKINS_UC_LATEST..." - else - JENKINS_UC_LATEST= - fi - - echo "Downloading plugins..." - for plugin in "${plugins[@]}"; do - local reg='^([^:]+):?([^:]+)?:?([^:]+)?:?(http.+)?' - if [[ $plugin =~ $reg ]]; then - local pluginId="${BASH_REMATCH[1]}" - local version="${BASH_REMATCH[2]}" - local lock="${BASH_REMATCH[3]}" - local url="${BASH_REMATCH[4]}" - download "$pluginId" "$version" "${lock:-true}" "${url}" & - else - echo "Skipping the line '${plugin}' as it does not look like a reference to a plugin" - fi - done - wait - - echo - echo "WAR bundled plugins:" - echo "${bundledPlugins}" - echo - echo "Installed plugins:" - installedPlugins - - if [[ -f $FAILED ]]; then - echo "Some plugins failed to download!" "$(<"$FAILED")" >&2 - exit 1 - fi - - echo "Cleaning up locks" - find "$REF_DIR" -regex ".*.lock" | while read -r filepath; do - rm -r "$filepath" - done - -} - -main "$@" diff --git a/instances/technology.jgit/target/k8s/statefulset.json b/instances/technology.jgit/target/k8s/statefulset.json index 493e9932b..226dcf2d8 100644 --- a/instances/technology.jgit/target/k8s/statefulset.json +++ b/instances/technology.jgit/target/k8s/statefulset.json @@ -141,6 +141,11 @@ "mountPath": "/run/secrets/jenkins/ssh", "name": "gerrit-ssh-keys", "readOnly": true + }, + { + "mountPath": "/run/secrets/jenkins/ssh-gerrithub.io", + "name": "gerrithub-io-ssh-keys", + "readOnly": true } ] } @@ -167,6 +172,12 @@ "secret": { "secretName": "gerrit-ssh-keys" } + }, + { + "name": "gerrithub-io-ssh-keys", + "secret": { + "secretName": "gerrithub.io-ssh-keys" + } } ] } diff --git a/templates/k8s/statefulset.libsonnet b/templates/k8s/statefulset.libsonnet index bcbfa2bd4..4fa25ed9f 100644 --- a/templates/k8s/statefulset.libsonnet +++ b/templates/k8s/statefulset.libsonnet @@ -107,21 +107,21 @@ local Kube = import "kube.libsonnet"; name: "jenkins-config", readOnly: true, }, - ] + if std.objectHas(config.secrets, "gerrit-trigger-plugin") then [ + ] + (if std.objectHas(config.secrets, "gerrit-trigger-plugin") then [ { mountPath: "/run/secrets/jenkins/ssh", name: "gerrit-ssh-keys", readOnly: true, } - ] else [] - + if std.objectHas(config, "gerrit") then [ + ] else []) + + (if std.objectHas(config, "gerrit") then [ { mountPath: "/run/secrets/jenkins/ssh-" + gerrit.site, name: std.strReplace(gerrit.site, '.', '-') + "-ssh-keys", readOnly: true, } for gerrit in config.gerrit - ] else [], + ] else []), env: [ { name: "JAVA_OPTS", @@ -215,15 +215,15 @@ local Kube = import "kube.libsonnet"; name: "jenkins-config", }, }, - ] + if std.objectHas(config.secrets, "gerrit-trigger-plugin") then [ + ] + (if std.objectHas(config.secrets, "gerrit-trigger-plugin") then [ { name: "gerrit-ssh-keys", secret: { secretName: "gerrit-ssh-keys", }, } - ] else [] - + if std.objectHas(config, "gerrit") then [ + ] else []) + + (if std.objectHas(config, "gerrit") then [ { name: std.strReplace(gerrit.site, '.', '-') + "-ssh-keys", secret: { @@ -231,7 +231,7 @@ local Kube = import "kube.libsonnet"; }, } for gerrit in config.gerrit - ] else [], + ] else []), }, }, volumeClaimTemplates: [