CLOUD-64700: [kms] Автоматизировать процедуру обновления docker и com…

…pute образов Vault+KMS

yandex/scripts/common.sh

@@ -0,0 +1,78 @@
+init() {
+  START_DIR=$(pwd)
+  trap 'cd $START_DIR' EXIT
+
+  SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
+  cd $SCRIPT_DIR
+  SCRIPT_DIR=$(pwd)
+
+  if [[ -n $WORK_DIR ]]; then
+    mkdir -p $WORK_DIR
+    cd $WORK_DIR
+  fi
+  WORK_DIR=$(pwd)
+  YCKMS_VERSION=${PATCH_BRANCH:-"$BASE_VERSION+yckms"}
+}
+
+go_to_work_dir() {
+  cd $WORK_DIR
+}
+
+cleanup() {
+  go_to_work_dir
+  rm -rf vault
+  rm -rf vault-kms-wrapper
+}
+
+init_vault() {
+  go_to_work_dir
+  if [[ ! -d "vault" ]]; then
+    echo "Cloning vault"
+    git clone [email protected]:yandex-cloud/vault.git
+    cd vault
+    git remote add upstream [email protected]:hashicorp/vault.git
+  else
+    echo "Vault already cloned"
+    cd vault
+    git reset --hard
+  fi
+
+  echo "Synchronizing vault with upstream"
+  git checkout main
+  git pull upstream main
+  echo "Fetching tags"
+  git fetch upstream --tags
+
+  go_to_work_dir
+}
+
+init_vault_kms_wrapper() {
+  go_to_work_dir
+  if [[ ! -d "vault-kms-wrapper" ]]; then
+    echo "Cloning vault-kms-wrapper"
+    git clone [email protected]:yandex-cloud/vault-kms-wrapper.git
+    cd vault-kms-wrapper
+  else
+    echo "Vault already cloned"
+    cd vault-kms-wrapper
+  fi
+
+  echo "Refreshing main"
+  git reset --hard
+  git checkout main
+  git pull
+  git fetch -p
+
+  go_to_work_dir
+}
+
+get_kms_wrapper_version() {
+  go_to_work_dir
+  cd vault
+  CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+  git checkout $BASE_VERSION
+  KMS_WRAPPER_VERSION=$(go list -m github.com/hashicorp/go-kms-wrapping/v2 | cut -f 2 -d " ")
+  echo "Current go-kms-wrapping version: $KMS_WRAPPER_VERSION"
+  git checkout $CURRENT_BRANCH
+  go_to_work_dir
+}

yandex/scripts/push_compute/push_compute_image.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -e
+
+SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
+. $SCRIPT_PATH/../common.sh
+. $SCRIPT_PATH/push_compute.cfg
+
+init
+init_vault
+cd vault
+
+if ! git checkout $YCKMS_VERSION; then
+    echo >&2 "Cannot find branch $YCKMS_VERSION !"
+    exit 1
+else
+    echo "Checkout to branch '$YCKMS_VERSION' succeeded"
+fi
+
+TOKEN=$(ycp --profile $PROFILE iam create-token)
+
+cd yandex/compute
+
+if [[ $REPLACE_DEFAULT_ENDPOINT == 1 ]]; then
+  echo "$(jq --arg ENDPOINT "$ENDPOINT" --arg ZONE "$ZONE" --arg PLATFORM_ID "$PLATFORM_ID" \
+   '.builders[0] += {"endpoint":$ENDPOINT, "zone":$ZONE, "platform_id":$PLATFORM_ID}' vault.packer.json)" > vault.packer.json
+fi
+
+echo "$(jq --arg SOURCE_IMAGE_FAMILY "ubuntu-2004-lts" '.builders[0].source_image_family = $SOURCE_IMAGE_FAMILY' vault.packer.json)" > vault.packer.json
+
+FOLDER_ID="$FOLDER_ID" TOKEN="$TOKEN" $SCRIPT_PATH/packer build vault.packer.json

yandex/scripts/push_compute/push_compute_sample.cfg

@@ -0,0 +1,22 @@
+#Requirements:
+# - yc and ycp
+# - jq
+# - packer (from https://wiki.yandex-team.ru/cloud/devel/assembly-workshop/#packer)
+#   packer should be placed inside directory with script
+# Github already contains release version of $BASE_VERSION ($BASE_VERSION+$WRAPPER_SUFFIX)
+
+#Version of patched vault from which will be build image(from git branch $BASE_VERSION+yckms)
+BASE_VERSION="v1.14.1"
+#Directory inside which patched version will be pulled from git
+WORK_DIR=./../tmp
+#If need to replace default endpoint(default is cloud.yandex.ru)
+REPLACE_DEFAULT_ENDPOINT=1
+ENDPOINT=api.il.nebius.cloud:443
+ZONE=il1-a
+PLATFORM_ID=standard-v3
+#Folder inside which image will be pulled
+FOLDER_ID=b48ans97du1s547k9e9m
+PROFILE=israel
+#Branch of git inside https://github.com/yandex-cloud/vault
+#from which will be build image(if not set $BASE_VERSION+yckms will be used)
+PATCH_BRANCH=test

yandex/scripts/push_docker/push_docker_image.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+set -e
+
+SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
+  . $SCRIPT_PATH/../common.sh
+  . $SCRIPT_PATH/push_docker.cfg
+
+init
+init_vault
+cd vault
+
+if ! git checkout $YCKMS_VERSION; then
+    echo >&2 "Cannot find branch $YCKMS_VERSION !"
+    exit 1
+else
+    echo "Checkout to branch '$YCKMS_VERSION' succeeded"
+fi
+
+cd yandex
+cd docker
+
+IAM_TOKEN=$(ycp --profile="$PROFILE" iam create-token)
+DOMAIN="cr.yandex"
+
+if [[ "$PROFILE" == "israel" ]]; then
+  DOMAIN="cr.cloudil.com"
+fi
+
+if [[ "$PROFILE" == "preprod" ]]; then
+  DOMAIN="cr.cloud-preprod.yandex.net"
+fi
+
+docker login --username iam --password $IAM_TOKEN $DOMAIN
+docker build --platform linux/amd64 -t $DOMAIN/$REGISTRY_ID/vault:$BASE_VERSION-yckms -t $DOMAIN/$REGISTRY_ID/vault .
+docker push $DOMAIN/$REGISTRY_ID/vault:$BASE_VERSION-yckms
+docker logout

yandex/scripts/push_docker/push_docker_sample.cfg

@@ -0,0 +1,15 @@
+#Requirements:
+# - yc and ycp
+# - docker
+# Github already contains release version of $BASE_VERSION ($BASE_VERSION+$WRAPPER_SUFFIX)
+
+#Version of patched vault from which will be build image(from git branch $BASE_VERSION+yckms)
+BASE_VERSION="v1.14.1"
+#Directory inside which patched version will be pulled from git
+WORK_DIR=./../tmp
+PROFILE=israel
+#Registry to which docker image will be pushed
+REGISTRY_ID=crls41le9juekjee2rv4
+#Branch of git inside https://github.com/yandex-cloud/vault
+#from which will be build image(if not set $BASE_VERSION+yckms will be used)
+PATCH_BRANCH=test

yandex/scripts/push_helm_chart/helm_chart_sample.cfg

@@ -0,0 +1,15 @@
+#Requirements:
+# - yc and ycp
+# - docker
+# - helm
+# - yq
+# - diff, patch, perl (should be included in default distributive)
+# - Build and pulled docker image of corresponding vault version(BASE_VERSION parameter)
+
+# Docker image version of patched version that will be used for helm (Version is $BASE_VERSION-yckms)
+BASE_VERSION="v1.14.1"
+#Diractory in will be pulled vault-csi-provider and vault-k8s
+WORK_DIR=./../tmp
+PROFILE=israel
+#Registry inside which will be pushed helm image
+REGISTRY_ID=crls41le9juekjee2rv4

yandex/scripts/push_helm_chart/push_helm_chart.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+set -e
+
+SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
+  . $SCRIPT_PATH/helm_chart.cfg
+
+if [[ -n $WORK_DIR ]]; then
+  mkdir -p $WORK_DIR
+  cd $WORK_DIR
+fi
+
+IAM_TOKEN=$(ycp --profile="$PROFILE" iam create-token)
+DOMAIN="cr.yandex"
+
+if [[ "$PROFILE" == "israel" ]]; then
+  DOMAIN="cr.cloudil.com"
+fi
+
+if [[ "$PROFILE" == "preprod" ]]; then
+  DOMAIN="cr.cloud-preprod.yandex.net"
+fi
+
+docker login --username iam --password $IAM_TOKEN $DOMAIN
+
+git clone https://github.com/hashicorp/vault-helm.git ./vault-helm
+git clone ssh://[email protected]/cloud/mk8s-marketplace-helm.git ./mk8s-marketplace-helm
+
+cd vault-helm
+
+TAGS=$(git tag --sort="-version:refname")
+echo $TAGS
+
+VERSION=$(echo "$BASE_VERSION" | cut -c 2-)
+CUT_VERSION="${VERSION%.*}"
+ACTUAL_TAG=''
+
+for TAG in $TAGS
+do
+  git checkout tags/$TAG values.yaml
+  ACTUAL_VERSION=$(yq '.server.image.tag' values.yaml)
+  ACTUAL_VERSION="${ACTUAL_VERSION%.*}"
+  echo "Actual version = $ACTUAL_VERSION and version = $CUT_VERSION"
+  if [ $CUT_VERSION = $ACTUAL_VERSION ]
+  then
+    ACTUAL_TAG=$TAG
+    echo "Found actual version $ACTUAL_VERSION"
+    break
+  fi
+done
+
+git checkout tags/$ACTUAL_TAG
+
+sudo rm -r ./.git
+
+CSI_PROVIDER_ACTUAL_TAG=$(yq '.csi.image.tag' values.yaml)
+K8S_ACTUAL_TAG=$(yq '.injector.image.tag' values.yaml)
+
+cd ../
+
+docker pull hashicorp/vault-csi-provider:$CSI_PROVIDER_ACTUAL_TAG --platform amd64
+docker pull hashicorp/vault-k8s:$K8S_ACTUAL_TAG --platform amd64
+
+CSI_PROVIDER_IMAGE_ID=$(docker images hashicorp/vault-csi-provider:$CSI_PROVIDER_ACTUAL_TAG -q)
+K8S_IMAGE_ID=$(docker images hashicorp/vault-k8s:$K8S_ACTUAL_TAG -q)
+
+docker tag $K8S_IMAGE_ID $DOMAIN/$REGISTRY_ID/vault/vault-k8s:$K8S_ACTUAL_TAG
+docker tag $K8S_IMAGE_ID $DOMAIN/$REGISTRY_ID/vault/vault-k8s:latest
+
+docker tag $CSI_PROVIDER_IMAGE_ID $DOMAIN/$REGISTRY_ID/vault/vault-csi-provider:$CSI_PROVIDER_ACTUAL_TAG
+docker tag $CSI_PROVIDER_IMAGE_ID $DOMAIN/$REGISTRY_ID/vault/vault-csi-provider:latest
+
+docker push $DOMAIN/$REGISTRY_ID/vault/vault-k8s:$K8S_ACTUAL_TAG
+docker push $DOMAIN/$REGISTRY_ID/vault/vault-k8s:latest
+
+docker push $DOMAIN/$REGISTRY_ID/vault/vault-csi-provider:$CSI_PROVIDER_ACTUAL_TAG
+docker push $DOMAIN/$REGISTRY_ID/vault/vault-csi-provider:latest
+
+rm -r ./mk8s-marketplace-helm/products/hashicorp-vault/chart
+cp -r ./vault-helm ./mk8s-marketplace-helm/products/hashicorp-vault/chart
+rm -r ./mk8s-marketplace-helm/products/hashicorp-vault/chart/test
+
+CHART_VERSION="$(yq '.version' ./vault-helm/Chart.yaml)-1"
+replacement=$CHART_VERSION yq -i '.version = strenv(replacement)' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/Chart.yaml
+
+cat >./mk8s-marketplace-helm/products/hashicorp-vault/chart/templates/kms-creds-secret.yaml <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kms-creds
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+type: Opaque
+data:
+  "credentials.json": {{ .Values.yandexKmsAuthJson | b64enc }}
+EOF
+
+/bin/bash $SCRIPT_PATH/update_values.sh
+
+cd mk8s-marketplace-helm/products/hashicorp-vault/chart
+helm package .
+helm push ./vault-$(echo "$ACTUAL_TAG" | cut -c 2-)-1.tgz oci://$DOMAIN/$REGISTRY_ID/vault/chart

yandex/scripts/push_helm_chart/update_values.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
+  . $SCRIPT_PATH/release_sample.cfg
+
+CSI_PROVIDER_ACTUAL_TAG=$(yq '.csi.image.tag' ./vault-helm/values.yaml)
+K8S_ACTUAL_TAG=$(yq '.injector.image.tag' ./vault-helm/values.yaml)
+
+yq '.' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml > values.yaml.new
+
+DOMAIN="cr.yandex"
+if [[ "$PROFILE" == "israel" ]]; then
+  DOMAIN="cr.cloudil.com"
+fi
+if [[ "$PROFILE" == "preprod" ]]; then
+  DOMAIN="cr.cloud-preprod.yandex.net"
+fi
+VAULT_VERSION="$(echo "$BASE_VERSION" | cut -c 2-)-yckms"
+
+replacement=$VAULT_VERSION yq -i '(.. | select(key == "repository" and . == "hashicorp/vault") | parent).tag |= strenv(replacement)' values.yaml.new
+replacement="$DOMAIN/$REGISTRY_ID/vault/vault" yq -i '(.. | select(key == "repository" and . == "hashicorp/vault") | parent).repository |= strenv(replacement)' values.yaml.new
+replacement=$CSI_PROVIDER_ACTUAL_TAG yq -i '(.. | select(key == "repository" and . == "hashicorp/vault-csi-provider") | parent).tag |= strenv(replacement)' values.yaml.new
+replacement="$DOMAIN/$REGISTRY_ID/vault/vault-csi-provider" yq -i '(.. | select(key == "repository" and . == "hashicorp/vault-csi-provider") | parent).repository |= strenv(replacement)' values.yaml.new
+replacement=$K8S_ACTUAL_TAG yq -i '(.. | select(key == "repository" and . == "hashicorp/vault-k8s") | parent).tag |= strenv(replacement)' values.yaml.new
+replacement="$DOMAIN/$REGISTRY_ID/vault/vault-k8s" yq -i '(.. | select(key == "repository" and . == "hashicorp/vault-k8s") | parent).repository |= strenv(replacement)' values.yaml.new
+yq -i '(.. | select(key == "extraVolumes")) |= {"type": "secret", "name": "kms-creds"} ' values.yaml.new
+
+yq '.' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml > values.yaml.noblanks
+diff -B values.yaml.noblanks values.yaml.new > ./patch.file
+patch ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml ./patch.file
+
+#patch works bad, so it's needed to patch twice
+yq '.' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml > values.yaml.new
+
+read -rd '' replacement << EOF
+
+seal "yandexcloudkms" {
+    kms_key_id = "{{ .Values.yandexKmsKeyId}}"
+    service_account_key_file = "/vault/userconfig/kms-creds/credentials.json"
+}
+
+EOF
+replacement=$replacement yq -i '(.. | select(key == "standalone")).config |= . + strenv(replacement)' values.yaml.new
+
+yq '.' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml > values.yaml.noblanks
+diff -B values.yaml.noblanks values.yaml.new > ./patch.file
+patch ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml ./patch.file
+
+echo 'yandexKmsAuthJson: ""' >> ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml
+echo 'yandexKmsKeyId: ""' >> ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml
+
+perl -pe 'chomp if eof' ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml >tmp.file
+mv tmp.file ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml
+
+rm patch.file
+rm values.yaml.new
+rm values.yaml.noblanks
+rm ./mk8s-marketplace-helm/products/hashicorp-vault/chart/values.yaml.orig

yandex/scripts/release/cleanup.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
+. $SCRIPT_PATH/common.sh
+. $SCRIPT_PATH/release_sample.cfg
+
+init
+cleanup

yandex/scripts/release/patch_version_base.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+START_DIR=$(pwd)
+trap 'cd $START_DIR' EXIT
+
+SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
+cd "$SCRIPT_PATH"/..
+
+sed -i '' 's/.*VersionMetadata.*=.*""/VersionMetadata = "yckms"/' version/version_base.go
+go fmt version/version_base.go