diff --git a/.github/actions/build-and-test-feature/action.yml b/.github/actions/build-and-test-feature/action.yml index 30562ee8..70d11f73 100644 --- a/.github/actions/build-and-test-feature/action.yml +++ b/.github/actions/build-and-test-feature/action.yml @@ -5,7 +5,6 @@ description: Test feature inputs: args: {type: string, required: true} gh_token: {type: string, defaut: '', required: false} - vault_host: {type: string, defaut: '', required: false} aws_role_arn: {type: string, defaut: '', required: false} rw_sccache_bucket: {type: string, defaut: '', required: false} rw_sccache_region: {type: string, defaut: '', required: false} @@ -30,7 +29,6 @@ runs: NODE_NO_WARNINGS: 1 VAULT_S3_TTL: "900" # 15 minutes gh_token: "${{ inputs.gh_token }}" - vault_host: "${{ inputs.vault_host }}" aws_role_arn: "${{ inputs.aws_role_arn }}" rw_sccache_bucket: "${{ inputs.rw_sccache_bucket }}" rw_sccache_region: "${{ inputs.rw_sccache_region }}" diff --git a/.github/workflows/build-and-test-feature.yml b/.github/workflows/build-and-test-feature.yml index a0d514f8..b15b78fe 100644 --- a/.github/workflows/build-and-test-feature.yml +++ b/.github/workflows/build-and-test-feature.yml @@ -37,7 +37,6 @@ jobs: with: args: "${{ inputs.args }}" gh_token: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN }}" - vault_host: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'https://vault.ops.k8s.rapids.ai' || '' }}" aws_role_arn: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs' || '' }}" rw_sccache_bucket: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'rapids-sccache-devs' || '' }}" rw_sccache_region: "${{ vars.AWS_REGION }}" diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json index a006abb0..c4d79c49 100644 --- a/features/src/utils/devcontainer-feature.json +++ b/features/src/utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "devcontainer-utils", "id": "utils", - "version": "24.12.2", + "version": "24.12.3", "description": "A feature to install RAPIDS devcontainer utility scripts", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/utils/install.sh b/features/src/utils/install.sh index 11a79001..e7abb4a8 100644 --- a/features/src/utils/install.sh +++ b/features/src/utils/install.sh @@ -66,10 +66,9 @@ chgrp crontab "$(realpath -m "$(which cron)")"; chmod u+s "$(realpath -m "$(which cron)")"; # shellcheck disable=SC2174 -mkdir -m 0775 -p /var/log/devcontainer-utils; +mkdir -m 0777 -p /var/log/devcontainer-utils; touch /var/log/devcontainer-utils/creds-s3.log; -chmod 0664 /var/log/devcontainer-utils/creds-s3.log; -chgrp crontab /var/log/devcontainer-utils/creds-s3.log; +chmod 0777 /var/log/devcontainer-utils/creds-s3.log; # Install Devcontainer utility scripts to /opt/devcontainer cp -ar ./opt/devcontainer /opt/; @@ -105,13 +104,6 @@ declare -a commands_and_sources=( "init-gitlab-cli gitlab/cli/init.sh" "clone-gitlab-repo gitlab/repo/clone.sh" "print-missing-gitlab-token-warning gitlab/print-missing-token-warning.sh" - "vault-auth-github vault/auth/github.sh" - "vault-s3-init vault/s3/init.sh" - "vault-s3-creds-generate vault/s3/creds/generate.sh" - "vault-s3-creds-persist vault/s3/creds/persist.sh" - "vault-s3-creds-propagate vault/s3/creds/propagate.sh" - "vault-s3-creds-schedule vault/s3/creds/schedule.sh" - "vault-s3-creds-test vault/s3/creds/test.sh" ) # Install alternatives diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh index 7e248f6a..2a14ad09 100755 --- a/features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh +++ b/features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh @@ -13,8 +13,10 @@ _creds_github_generate() { exit 1; fi - # Remove existing credentials in case vault declines to issue new ones. - rm -rf ~/.aws/{stamp,config,credentials}; + # Remove existing credentials in case nv-gha-aws declines to issue new ones. + if test -w ~/.aws; then + rm -rf ~/.aws/{stamp,config,credentials}; + fi SCCACHE_REGION="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}"; @@ -57,7 +59,9 @@ _creds_github_generate() { generated_at="$(date '+%s')"; if gh nv-gha-aws org "${org}" "${nv_gha_aws_args[@]}" >"${HOME}/.aws/credentials" 2>>/var/log/devcontainer-utils/creds-s3.log; then if devcontainer-utils-creds-s3-propagate 2>&1 | tee -a /var/log/devcontainer-utils/creds-s3.log; then - echo "${generated_at}" > ~/.aws/stamp; + if test -w ~/.aws; then + echo "${generated_at}" > ~/.aws/stamp; + fi return 0; fi fi diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh index b89992b7..2f8448ef 100755 --- a/features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh +++ b/features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh @@ -41,61 +41,73 @@ _creds_s3_persist() { # Reset envvars reset_envvar "SCCACHE_BUCKET"; reset_envvar "SCCACHE_REGION"; - reset_envvar "AWS_ACCESS_KEY_ID"; - reset_envvar "AWS_SESSION_TOKEN"; - reset_envvar "AWS_SECRET_ACCESS_KEY"; mkdir -p ~/.aws; - rm -f ~/.aws/{config,credentials}; - if test -n "${stamp:-}"; then - echo "${stamp:-}" > ~/.aws/stamp; + if test -w ~/.aws; then + local name; + for name in config credentials; do + echo > ~/".aws/${name}" + done + if test -n "${stamp:-}"; then + echo "${stamp:-}" > ~/.aws/stamp; + fi fi if ! grep -qE "^$" <<< "${no_bucket-}"; then unset_envvar "SCCACHE_BUCKET"; elif ! grep -qE "^$" <<< "${bucket:-}"; then export_envvar "SCCACHE_BUCKET" "${bucket}"; - cat <<________EOF >> ~/.aws/config -bucket=${bucket:-} -________EOF + if test -w ~/.aws/config; then + cat <<< "bucket=${bucket:-}" >> ~/.aws/config + fi fi if ! grep -qE "^$" <<< "${no_region-}"; then unset_envvar "SCCACHE_REGION"; elif ! grep -qE "^$" <<< "${region:-}"; then export_envvar "SCCACHE_REGION" "${region}"; - cat <<________EOF >> ~/.aws/config -region=${region:-} -________EOF + if test -w ~/.aws/config; then + cat <<< "region=${region:-}" >> ~/.aws/config + fi fi - if test -f ~/.aws/config; then + if test -w ~/.aws && test -w ~/.aws/config; then cat <<________EOF > ~/.aws/config2 && mv ~/.aws/config{2,} [default] $(cat ~/.aws/config) ________EOF + chmod 0644 ~/.aws/config; fi if ! grep -qE "^$" <<< "${aws_access_key_id:-}"; then - cat <<________EOF >> ~/.aws/credentials -aws_access_key_id=${aws_access_key_id} -________EOF + if test -w ~/.aws/credentials; then + reset_envvar "AWS_ACCESS_KEY_ID"; + cat <<< "aws_access_key_id=${aws_access_key_id}" >> ~/.aws/credentials + else + export_envvar "AWS_ACCESS_KEY_ID" "${aws_access_key_id}"; + fi fi if ! grep -qE "^$" <<< "${aws_secret_access_key:-}"; then - cat <<________EOF >> ~/.aws/credentials -aws_secret_access_key=${aws_secret_access_key} -________EOF + if test -w ~/.aws/credentials; then + reset_envvar "AWS_SESSION_TOKEN"; + cat <<< "aws_secret_access_key=${aws_secret_access_key}" >> ~/.aws/credentials + else + export_envvar "AWS_SESSION_TOKEN" "${aws_secret_access_key}"; + fi fi if ! grep -qE "^$" <<< "${aws_session_token:-}"; then - cat <<________EOF >> ~/.aws/credentials -aws_session_token=${aws_session_token} -________EOF + if test -w ~/.aws/credentials; then + reset_envvar "AWS_SECRET_ACCESS_KEY"; + cat <<< "aws_session_token=${aws_session_token}" >> ~/.aws/credentials + else + export_envvar "AWS_SECRET_ACCESS_KEY" "${aws_session_token}"; + fi fi - if test -f ~/.aws/credentials; then + if test -w ~/.aws && test -w ~/.aws/credentials; then cat <<________EOF > ~/.aws/credentials2 && mv ~/.aws/credentials{2,} [default] $(cat ~/.aws/credentials) diff --git a/features/src/utils/opt/devcontainer/bin/creds/s3/vault/generate.sh b/features/src/utils/opt/devcontainer/bin/creds/s3/vault/generate.sh index 76323e61..03278553 100755 --- a/features/src/utils/opt/devcontainer/bin/creds/s3/vault/generate.sh +++ b/features/src/utils/opt/devcontainer/bin/creds/s3/vault/generate.sh @@ -17,11 +17,14 @@ _creds_vault_generate() { SCCACHE_REGION="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}"; # Remove existing credentials in case vault declines to issue new ones. - rm -rf ~/.aws/{stamp,config,credentials}; + if test -w ~/.aws; then + rm -rf ~/.aws/{stamp,config,credentials}; + fi - devcontainer-utils-creds-s3-persist - <<< \ - --bucket="${SCCACHE_BUCKET:-}" \ - --region="${SCCACHE_REGION:-}" ; + devcontainer-utils-creds-s3-persist - <<< " \ + --bucket '${SCCACHE_BUCKET:-}' \ + --region '${SCCACHE_REGION:-}' \ + "; # Initialize the GitHub CLI with the appropriate user scopes # shellcheck disable=SC1091 diff --git a/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh index 43ca2aee..2c17051b 100755 --- a/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh +++ b/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh @@ -77,6 +77,8 @@ clone_git_repo() { git -C "${directory}" remote set-url --push upstream read_only 2>/dev/null || true; if test "${upstream}" == "${origin}"; then git -C "${directory}" remote set-url --push origin read_only 2>/dev/null || true; + else + git -C "${directory}" remote set-url --push origin "${origin}" 2>/dev/null || true; fi git -C "${directory}" fetch "${fqj[@]}" --all; diff --git a/features/src/utils/opt/devcontainer/bin/update-envvars.sh b/features/src/utils/opt/devcontainer/bin/update-envvars.sh index 711eced8..c867e636 100755 --- a/features/src/utils/opt/devcontainer/bin/update-envvars.sh +++ b/features/src/utils/opt/devcontainer/bin/update-envvars.sh @@ -3,7 +3,7 @@ export_envvar() { if [ -n "${1:-}" ]; then for file in ~/.bashrc /etc/profile.d/*-devcontainer-utils.sh; do - echo "export ${1}=\"${2:-}\";" | sudo tee -a "${file}" >/dev/null; + cat <<< "export ${1}=\"${2:-}\";" | sudo tee -a "${file}" >/dev/null; done; fi } @@ -11,7 +11,7 @@ export_envvar() { unset_envvar() { if [ -n "${1:-}" ]; then for file in ~/.bashrc /etc/profile.d/*-devcontainer-utils.sh; do - echo "unset ${1};" | sudo tee -a "${file}" >/dev/null; + cat <<< "unset ${1};" | sudo tee -a "${file}" >/dev/null; done; fi } diff --git a/features/src/utils/opt/devcontainer/bin/vault/auth/github.sh b/features/src/utils/opt/devcontainer/bin/vault/auth/github.sh deleted file mode 100755 index 53a826a5..00000000 --- a/features/src/utils/opt/devcontainer/bin/vault/auth/github.sh +++ /dev/null @@ -1,40 +0,0 @@ -#! /usr/bin/env bash - - -get_vault_token() { - local -; - set -euo pipefail; - - # shellcheck disable=SC1091 - . devcontainer-utils-debug-output 'devcontainer_utils_debug' 'vault-s3 vault-auth-github'; - - local -r VAULT_HOST="$1"; - local -r user_orgs=("${@:2}"); - local -r gh_token="$(gh auth token)"; - local vault_token=null; - local o; - local org; - - for o in "${user_orgs[@]}"; do - for org in $(echo -e "${o}\n${o,,}\n${o^^}" | sort -su); do - vault_token="$( \ - curl -s \ - -X POST \ - -H "Content-Type: application/json" \ - -d "{\"token\": \"${gh_token}\"}" \ - "${VAULT_HOST}/v1/auth/github-${org}/login" \ - | jq -r '.auth.client_token' \ - )"; - if test "${vault_token:-null}" != null; then - break; - fi - done - if test "${vault_token:-null}" != null; then - break; - fi - done - - echo "vault_token='$vault_token'"; -} - -get_vault_token "$@";