Skip to content
Test all functions, build marketplace

Merge pull request #816 from mlrun/nashpaz123-patch-1-1 #548

Sign in to view logs

Merge pull request #816 from mlrun/nashpaz123-patch-1-1

Merge pull request #816 from mlrun/nashpaz123-patch-1-1 #548

Workflow file for this run

.github/workflows/test-all.yaml at e4d5ce0
name: Test all functions, build marketplace
on:
push:
branches:
- development
- master
jobs:
build_strategy_matrix:
runs-on: ubuntu-latest
steps:
- name: Get the current branch name
shell: bash
run: echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
id: myref
- uses: actions/checkout@v3
- id: set-matrix
# This is very hacky, but it goes like that:
# 1) Associate base_ref with origin/base_ref since actions/checkout doesn't do it, if we don't do that we won't be able to check the actual diff
# 2) Build JSON string
# 2.1) Add beginning of JSON
# 2.2) Get diff between origin/base_ref and the checked-out repo => git diff ${{ github.base_ref }} --name-only
# 2.3) Clean the file name and leave us only with directories => sed 's,/*[^/]\+/*$,,'
# 2.4) Sort and keep only unique directories => sort | uniq
# 2.5) Remove directories starting with '.' => grep -v '^\.'
# 2.6) Add quotation marks to all strings => sed 's/.*/"&"/'
# 2.7) Add comma suffix to all strings excluding the last one => sed '$!s/.*/&,/'
# 2.8) Close JSON
# 3) Save matrix JSON to output
# This is old fetch command it cant work cause base_ref is only avaliable on pull request actions: git fetch origin ${{ github.base_ref }}:${{ github.base_ref }}
run: |
git fetch --no-tags --prune --depth=1 origin +refs/heads/${{ steps.myref.outputs.branch }}:refs/remotes/origin/${{ steps.myref.outputs.branch }}
matrix=$((
echo '{ "package" : ['
git ls-files *[^cli] | grep '/' | sort | uniq| sed 's,/*[^/]\+/*$,,' | sort | uniq | grep -v '^\.' | sed 's/.*/"&"/' | sed '$!s/.*/&,/'
echo " ]}"
) | jq -c .)
echo "::set-output name=matrix::$matrix"
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
check_matrix:
runs-on: ubuntu-latest
needs: build_strategy_matrix
steps:
- name: Install json2yaml
run: |
sudo npm install -g json2yaml
- name: Check matrix definition
run: |
matrix='${{ needs.build_strategy_matrix.outputs.matrix }}'
echo $matrix
echo $matrix | jq .
echo $matrix | json2yaml
run_monorepo_tests:
needs: build_strategy_matrix
runs-on: ubuntu-latest
strategy:
# matrix: [{"package": some package that changed}, {...}, ...]
matrix: ${{fromJson(needs.build_strategy_matrix.outputs.matrix)}}
steps:
# Source
- name: Checkout current repo
uses: actions/checkout@v3
with:
path: functions
# Install python 3.9
- name: Install python 3.9
uses: actions/setup-python@v3
with:
python-version: 3.9
# Install dependencies
- uses: actions/cache@v3
id: cache
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('functions/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install requirements
run: |
pip install --upgrade pip
pip install -r functions/requirements.txt
- name: Run py tests
run: python functions/functions.py run-tests -r functions -s py -fn ${{ matrix.package }}
continue-on-error: true
# - name: Run ipynb tests
# run: python functions/functions.py run-tests -r functions -s ipynb
- name: Clean
run: |
rm -rf functions
build-marketplace:
name: Build marketplace
if: github.repository == 'mlrun/functions' && github.event_name != 'pull_request'
runs-on: ubuntu-latest
needs: run_monorepo_tests
continue-on-error: false
steps:
- name: Get the current branch name
shell: bash
run: echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
id: branch
- name: Checkout current repo
uses: actions/checkout@v3
with:
path: functions
- name: Checkout Marketplace
uses: actions/checkout@v3
with:
repository: mlrun/marketplace
path: marketplace
- name: Install python 3.9
uses: actions/setup-python@v3
with:
python-version: 3.9
- name: Install requirements
run: |
cd functions
pip install --upgrade pip
pip install -r requirements.txt
- name: Build marketplace docs
env:
CHANNEL: ${{ steps.branch.outputs.branch }}
run: |
cd marketplace
pwd
git pull origin
cd ..
python functions/functions.py build-marketplace -s functions -m marketplace -c $CHANNEL -v -f
- name: Publish marketplace release
env:
GITHUB_TOKEN: ${{ secrets.MARKETPLACE_ACCESS_TOKEN_V4 }}
USERNAME: iguazio-cicd
REPO_PATH: marketplace
BASE_REPO: mlrun
BASE_BRANCH: master
run: |
cd marketplace
pwd
COMMIT_SHA=$(git rev-parse --short "$GITHUB_SHA")
echo "commit sha: $COMMIT_SHA"
echo "github sha: $GITHUB_SHA"
BRANCH_NAME=marketplace-doc-gen-$COMMIT_SHA
REMOTE=https://$USERNAME:[email protected]/$BASE_REPO/$REPO_PATH.git
echo "Validating environment params...";
[ -z "${GITHUB_TOKEN}" ] && {
echo 'Missing input "GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}".';
exit 1;
};
git config --local user.name $USERNAME
git branch --set-upstream-to origin/master
git remote -v
echo "Checking out [$BRANCH_NAME]..."
git checkout -b $BRANCH_NAME
echo "Checking out [$BASE_BRANCH]..."
git checkout $BASE_BRANCH
git pull
echo "Checking out [$BRANCH_NAME]..."
git checkout $BRANCH_NAME
git merge $BASE_BRANCH
git status
git status --ignored
find . -type f | xargs ls -artl
git add --all
git status
git status --ignored
echo "Commiting changes..."
git rev-parse --show-toplevel
git commit -a -m "Automatically generated by github-worflow[bot] for commit: $COMMIT_SHA"
git status
git status --ignored
# https://stackoverflow.com/questions/64270867/auth-error-trying-to-copy-a-repo-with-github-actions
git config -l | grep 'http\..*\.extraheader' | cut -d= -f1 | xargs -L1 git config --unset-all
echo "Pushing [$BRANCH_NAME] to remote [$REMOTE]"
git push -f $REMOTE $BRANCH_NAME
echo "Submiting pull request..."
gh pr create --title "Marketplace update from $BRANCH_NAME" --body "github-workflow" --base $BASE_BRANCH --head $BRANCH_NAME --repo $BASE_REPO/$REPO_PATH