Chore: Daily Tasks #342
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Chore: Daily Tasks" | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" | |
| workflow_dispatch: | |
| permissions: | |
| issues: write | |
| pull-requests: write | |
| concurrency: | |
| group: lock | |
| jobs: | |
| generate-readme: | |
| runs-on: ubuntu-latest | |
| name: "Generate readme files" | |
| container: | |
| image: ghcr.io/truecharts/devcontainer:3.1.10@sha256:c239addf725eb5cedf79517f8089fdafdc32b5270d1893ee87ae6e511b9bcae3 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| token: ${{ secrets.BOT_TOKEN }} | |
| fetch-depth: 1 | |
| - name: Setting repo parent dir as safe safe.directory | |
| run: git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| - name: generate readme.md | |
| shell: bash | |
| run: | | |
| for train in stable operators SCALE incubator games enterprise develop non-free deprecated dependency core; do | |
| for chart in charts/${train}/*; do | |
| if [ -d "${chart}" ]; then | |
| echo "Generating readme.md for ${train}/${chart}" | |
| cp "templates/README.md.tpl" "${chart}/README.md" | |
| sed -i "s/TRAINPLACEHOLDER/${train}/" "${chart}/README.md" | |
| sed -i "s/CHARTPLACEHOLDER/${chartname}/" "${chart}/README.md" | |
| fi | |
| done | |
| done | |
| - name: generate HelmIgnore | |
| shell: bash | |
| run: | | |
| for train in stable operators SCALE incubator games enterprise develop non-free deprecated dependency core; do | |
| for chart in charts/${train}/*; do | |
| if [ -d "${chart}" ]; then | |
| echo "Attempting to sync HelmIgnore file for: ${chartname}" | |
| rm -rf ${chart}/.helmignore | |
| cp templates/chart/.helmignore ${chart}/ | |
| fi | |
| done | |
| done | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| repository: truecharts/website | |
| path: website | |
| fetch-depth: 1 | |
| token: ${{ secrets.BOT_TOKEN }} | |
| - name: Bump and Sync | |
| shell: bash | |
| run: | | |
| # Designed to ensure the appversion in Chart.yaml is in sync with the primary Chart tag if found | |
| # Also makes sure that home link is pointing to the correct url | |
| sync_tag() { | |
| local chart="$1" | |
| local chartname="$2" | |
| local train="$3" | |
| echo "Attempting to sync primary tag with appversion for: ${chartname}" | |
| local tag="$(cat ${chart}/values.yaml | grep '^ tag: ' | awk -F" " '{ print $2 }' | head -1)" | |
| tag="${tag%%@*}" | |
| tag="${tag##\"}" | |
| tag="${tag:-auto}" | |
| tag=$(echo $tag | sed "s/release-//g") | |
| tag=$(echo $tag | sed "s/release_//g") | |
| tag=$(echo $tag | sed "s/version-//g") | |
| tag=$(echo $tag | sed "s/version_//g") | |
| tag="${tag#*V.}" | |
| tag="${tag#*v-}" | |
| tag="${tag#*v}" | |
| tag="${tag%-*}" | |
| tag="${tag:0:10}" | |
| tag="${tag%-}" | |
| tag="${tag%_}" | |
| tag="${tag%.}" | |
| echo "Updating tag of ${chartname} to ${tag}..." | |
| sed -i -e "s|appVersion: .*|appVersion: \"${tag}\"|" "${chart}/Chart.yaml" | |
| echo "Updating icon of ${chartname}..." | |
| sed -i -e "s|icon: .*|icon: https:\/\/truecharts.org\/img\/hotlink-ok\/chart-icons\/${chartname}.png|" "${chart}/Chart.yaml" | |
| echo "Updating home of ${chartname}..." | |
| sed -i -e "s|home: .*|home: https:\/\/truecharts.org\/charts\/${train}\/${chartname}|" "${chart}/Chart.yaml" | |
| echo "Attempting to update sources of ${chartname}..." | |
| echo "Using go-yq verion: <$(go-yq -V)>" | |
| # Get current sources, exluding those that may have been added automatically. | |
| curr_sources=$( | |
| go-yq ' | |
| .sources[] | | |
| select( | |
| . != "https://github.com/truecharts*" and | |
| . != "https://ghcr*" and | |
| . != "docker.io*" and | |
| . != "https://docker.io*" and | |
| . != "https://hub.docker*" and | |
| . != "https://fleet.*" and | |
| . != "https://github.com/truecharts/containers/tree/master/mirror/*" and | |
| . != "https://public.ecr.aws*" and | |
| . != "https://ocir.io*" and | |
| . != "https://gcr*" and | |
| . != "https://azurecr*" and | |
| . != "https://quay*" and | |
| . != "https://lscr*" and | |
| . != "https://github.com/truecharts/containers*" and | |
| . == "http*" | |
| ) | |
| ' \ | |
| "${chart}/Chart.yaml" | |
| ) | |
| # Empty sources list in-place | |
| go-yq -i 'del(.sources.[])' "${chart}/Chart.yaml" | |
| # Add truechart source | |
| tcsource="https://github.com/truecharts/charts/tree/master/charts/$train/$chartname" go-yq -i '.sources += env(tcsource)' "${chart}/Chart.yaml" | |
| # Get the container image name that was parsed out of the Dockerfile for the website. | |
| container=$(cat website/docs/charts/description_list.md | grep "\[${chartname}\]" | cut -f3 -d '|' | grep -v 'Not Found' || echo "" || echo "failed-container-fetch") | |
| # Convert the container image name to a URL. | |
| if [ ! -z "$container" ]; then | |
| prefix="" | |
| case "$container" in | |
| lscr.io/linuxserver/*) | |
| prefix="https://fleet.linuxserver.io/image?name=" | |
| container=${container#lscr.io/} | |
| ;; | |
| tccr.io/truecharts/*) | |
| prefix="https://github.com/truecharts/containers/tree/master/mirror" | |
| container=${container#tccr.io/truecharts/} | |
| ;; | |
| mcr.microsoft.com/*) | |
| prefix="" | |
| ;; | |
| public.ecr.aws/*) | |
| prefix="https://gallery.ecr.aws/" | |
| container=${container#public.ecr.aws/} | |
| ;; | |
| ghcr.io/*) | |
| prefix="https://" | |
| ;; | |
| quay.io/*) | |
| prefix="https://" | |
| ;; | |
| gcr.io/*) | |
| prefix="https://" | |
| ;; | |
| *.azurecr.io/*) | |
| prefix="" | |
| ;; | |
| *.ocir.io/*) | |
| prefix="" | |
| ;; | |
| # There have been a number of domains used for the Docker Hub registry over the years. | |
| # NOTE: This is also the default case! | |
| docker.io/*|index.docker.io/*|registry-1.docker.io/*|registry.hub.docker.com/*|*) | |
| prefix="https://hub.docker.com/r/" | |
| container=${container#docker.io/} | |
| container=${container#index.docker.io/} | |
| container=${container#registry-1.docker.io/} | |
| container=${container#registry.hub.docker.com/} | |
| # If the image name does not contain a slash it is a Docker Official Image. | |
| if [ "$container" == "${container////}" ]; then | |
| prefix="https://hub.docker.com/_/" | |
| # If the user name is library it is a Docker Official Image. | |
| elif [ "${container%%/*}" == "library" ]; then | |
| prefix="https://hub.docker.com/_/" | |
| container=${container#library/} | |
| fi | |
| # Avoid creating a bad link since an unsupported registry may have been used. | |
| slashes=${container//[^\/]/} | |
| # Bail out if the image name has more than 1 slash. | |
| if [ ${#slashes} -gt 1 ]; then | |
| prefix="" | |
| echo "WARNING: Not assuming '$container' is a Docker Hub image" | |
| fi | |
| ;; | |
| esac | |
| if [ -n "${prefix}" ]; then | |
| container="${prefix}${container}" go-yq -i '.sources += env(container) | .sources |= unique' "${chart}/Chart.yaml" | |
| fi | |
| fi | |
| # Add the rest of the sources | |
| while IFS= read -r line; do | |
| src="$line" go-yq -i '.sources += env(src)' "${chart}/Chart.yaml" || echo "src set error" | |
| done <<< "$curr_sources" | |
| echo "Sources of ${chartname} updated!" | |
| } | |
| export -f sync_tag | |
| for train in enterprise stable operators incubator dependency; do | |
| echo "Correcting Chart.yaml for Train: ${train}..." | |
| for chart in $(ls "charts/${train}"); do | |
| echo "Correcting Chart.yaml for Chart: ${chart}..." | |
| sync_tag "charts/${train}/${chart}" "${chart}" "${train}" || echo "failed" | |
| done | |
| done | |
| - name: Fix Fixable Pre-Commit issues | |
| shell: bash | |
| if: inputs.chartChangesDetected == 'true' | |
| run: | | |
| echo "Running pre-commit test-and-cleanup..." | |
| pre-commit run --all ||: | |
| # Fix sh files to always be executable | |
| find . -name '*.sh' | xargs chmod +x | |
| - name: Cleanup | |
| run: | | |
| rm -rf changes.json | |
| rm -rf master | |
| - name: Commit changes | |
| run: | | |
| git config user.name "TrueCharts-Bot" | |
| git config user.email "[email protected]" | |
| git pull | |
| git add --all | |
| git commit -sm "Commit daily changes" || exit 0 | |
| git push | |
| generate-security-reports: | |
| runs-on: ubuntu-latest | |
| name: "Generate Security Reports" | |
| container: | |
| image: ghcr.io/truecharts/devcontainer:3.1.10@sha256:c239addf725eb5cedf79517f8089fdafdc32b5270d1893ee87ae6e511b9bcae3 | |
| steps: | |
| - name: Install Kubernetes tools | |
| uses: yokawasa/action-setup-kube-tools@af4ebb1af1efd30c5bd84a2e9773355ad6362a33 # v0.9.3 | |
| with: | |
| setup-tools: | | |
| helmv3 | |
| helm: "3.8.0" | |
| - name: Prep Helm | |
| run: | | |
| helm repo add truecharts https://charts.truecharts.org | |
| helm repo add truecharts-library https://library-charts.truecharts.org | |
| helm repo add truecharts-deps https://deps.truecharts.org | |
| helm repo add jetstack https://charts.jetstack.io | |
| helm repo add vmwaretanzu https://vmware-tanzu.github.io/helm-charts | |
| helm repo add cnpg https://cloudnative-pg.github.io/charts | |
| helm repo add metallb https://metallb.github.io/metallb | |
| helm repo add prometheus-community https://prometheus-community.github.io/helm-charts | |
| helm repo update | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| token: ${{ secrets.BOT_TOKEN }} | |
| fetch-depth: 1 | |
| - name: Setting repo parent dir as safe safe.directory | |
| run: git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| - name: Checkout website | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| fetch-depth: 1 | |
| repository: truecharts/website | |
| token: ${{ secrets.BOT_TOKEN }} | |
| path: website | |
| - name: fetch dependencies | |
| shell: bash | |
| run: | | |
| .github/scripts/fetch_helm_deps.sh | |
| - name: generate security reports | |
| shell: bash | |
| run: | | |
| #!/bin/bash | |
| render() { | |
| local chart="$1" | |
| local chartname="$2" | |
| local train="$3" | |
| echo "Rendering helm-template for ${chartname}" | |
| mkdir -p ${chart}/render | |
| helm template ${chart} >> ${chart}/render/app.yaml || echo "Helm template failed..." | |
| } | |
| helm_sec_scan() { | |
| local chart="$1" | |
| local chartname="$2" | |
| local train="$3" | |
| echo "Scanning helm security for ${chartname}" | |
| mkdir -p ${chart}/render | |
| rm -rf website/docs/charts/${train}/${chartname}/helm-security.md || echo "removing old helm-security file failed..." | |
| echo "# Helm Security" >> website/docs/charts/${train}/${chartname}/helm-security.md | |
| echo "" >> website/docs/charts/${train}/${chartname}/helm-security.md | |
| echo "## Helm-Chart" >> website/docs/charts/${train}/${chartname}/helm-security.md | |
| echo "" >> website/docs/charts/${train}/${chartname}/helm-security.md | |
| echo "##### Scan Results" >> website/docs/charts/${train}/${chartname}/helm-security.md | |
| echo "" >> website/docs/charts/${train}/${chartname}/helm-security.md | |
| trivy config --namespaces builtin.kubernetes.* -f template --template "@./templates/trivy-config.tpl" ${chart}/render >> website/docs/charts/${train}/${chartname}/helm-security.md || echo "trivy scan failed..." | |
| } | |
| container_sec_scan() { | |
| local chart="$1" | |
| local chartname="$2" | |
| local train="$3" | |
| echo "Scanning container security for ${chartname}" | |
| mkdir -p ${chart}/render | |
| rm -rf website/docs/charts/${train}/${chartname}/container-security.md || echo "removing old container-security file failed..." | |
| echo "# Container Security" >> website/docs/charts/${train}/${chartname}/container-security.md | |
| echo "" >> website/docs/charts/${train}/${chartname}/container-security.md | |
| echo "##### Detected Containers" >> website/docs/charts/${train}/${chartname}/container-security.md | |
| echo "" >> website/docs/charts/${train}/${chartname}/container-security.md | |
| find ${chart}/render/ -name '*.yaml' -type f -exec cat {} \; | grep image: | sed "s/image: //g" | sed "s/\"//g" >> ${chart}/render/containers.tmp | |
| cat ${chart}/render/containers.tmp >> website/docs/charts/${train}/${chartname}/container-security.md | |
| echo "" >> website/docs/charts/${train}/${chartname}/container-security.md | |
| echo "##### Scan Results" >> website/docs/charts/${train}/${chartname}/container-security.md | |
| echo "" >> website/docs/charts/${train}/${chartname}/container-security.md | |
| for container in $(cat ${chart}/render/containers.tmp | sort | uniq); do | |
| if [[ "$container" == *"truecharts/alpine"* || "$container" == *"truecharts/ubuntu"* || "$container" == *"truecharts/kubectl"* ]]; then | |
| echo "Skipping ${container}, as it's a shared common container..." | |
| else | |
| echo "**Container: ${container}**" >> website/docs/charts/${train}/${chartname}/container-security.md | |
| echo "" >> website/docs/charts/${train}/${chartname}/container-security.md | |
| trivy image --security-checks vuln -f template --template "@./templates/trivy-container.tpl" ${container} >> website/docs/charts/${train}/${chartname}/container-security.md || echo "trivy container scan failed..." | |
| echo "" >> website/docs/charts/${train}/${chartname}/container-security.md | |
| fi | |
| done | |
| } | |
| cleanfiles() { | |
| local chart="$1" | |
| local chartname="$2" | |
| local train="$3" | |
| echo "sanitising website output for ${chartname}..." | |
| rm -rf ${chart}/render | |
| sed -i 's|<br>|<br />|g' website/docs/charts/${train}/${chartname}/helm-security.md ||: | |
| sed -i 's|<br>|<br />|g' website/docs/charts/${train}/${chartname}/container-security.md ||: | |
| sed -i 's|<hr>|<hr />|g' website/docs/charts/${train}/${chartname}/helm-security.md ||: | |
| sed -i 's|<hr>|<hr />|g' website/docs/charts/${train}/${chartname}/container-security.md ||: | |
| } | |
| for train in enterprise stable operators incubator dependency; do | |
| echo "Processing Charts for Train: ${train}..." | |
| for chart in $(ls "charts/${train}"); do | |
| render "charts/${train}/${chart}" ${chart} ${train} || echo "rendering failed for ${chart}" | |
| helm_sec_scan "charts/${train}/${chart}" ${chart} ${train} || echo "helm chart processing failed for ${chart}" | |
| if [ ${train} == "enterprise" ]; then | |
| container_sec_scan "charts/${train}/${chart}" ${chart} ${train} || echo "container processing failed for ${chart}" | |
| fi | |
| cleanfiles "charts/${train}/${chart}" ${chart} ${train} || echo "cleaning failed for ${chart}" | |
| done | |
| done | |
| echo "finsihed security scan" | |
| - name: Commit Website Changes | |
| run: | | |
| cd website | |
| git config user.name "TrueCharts-Bot" | |
| git config user.email "[email protected]" | |
| git pull | |
| git add --all | |
| git commit -sm "Commit released docs for TrueCharts" || exit 0 | |
| git push | |
| lock-threads: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: dessant/lock-threads@1bf7ec25051fe7c00bdd17e6a7cf3d7bfb7dc771 # v5 | |
| with: | |
| github-token: ${{ secrets.BOT_TOKEN }} | |
| issue-inactive-days: "7" | |
| exclude-any-issue-labels: "" | |
| issue-comment: "This issue is locked to prevent necro-posting on closed issues. Please create a new issue or contact staff on discord of the problem persists" | |
| issue-lock-reason: "" | |
| pr-inactive-days: "7" | |
| pr-comment: "This PR is locked to prevent necro-posting on closed PRs. Please create a issue or contact staff on discord if you want to further discuss this" | |
| pr-lock-reason: "resolved" | |
| log-output: true | |
| check-contributors: | |
| name: Check Contributors | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 | |
| with: | |
| token: ${{ secrets.BOT_TOKEN }} | |
| fetch-depth: 1 | |
| - uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4 | |
| with: | |
| node-version: 18 | |
| - uses: borales/actions-yarn@97ba8bebfe5b549bb7999261698a52a81fd62f1b # v4.2.0 | |
| with: | |
| cmd: install --frozen-lockfile | |
| - name: List missing and unknown contributors | |
| env: | |
| PRIVATE_TOKEN: ${{ secrets.BOT_TOKEN }} | |
| run: | | |
| awk -F', ' '{ for( i=1; i<=NF; i++ ) print $i }' <<<$(yarn all-contributors check) |