[Mlflow_utils] - mlflow model server (#811) #551
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test all functions, build marketplace | |
| on: | |
| push: | |
| branches: | |
| - development | |
| - master | |
| jobs: | |
| build_strategy_matrix: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Get the current branch name | |
| shell: bash | |
| run: echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" | |
| id: myref | |
| - uses: actions/checkout@v4 | |
| - id: set-matrix | |
| # This is very hacky, but it goes like that: | |
| # 1) Associate base_ref with origin/base_ref since actions/checkout doesn't do it, if we don't do that we won't be able to check the actual diff | |
| # 2) Build JSON string | |
| # 2.1) Add beginning of JSON | |
| # 2.2) Get diff between origin/base_ref and the checked-out repo => git diff ${{ github.base_ref }} --name-only | |
| # 2.3) Clean the file name and leave us only with directories => sed 's,/*[^/]\+/*$,,' | |
| # 2.4) Sort and keep only unique directories => sort | uniq | |
| # 2.5) Remove directories starting with '.' => grep -v '^\.' | |
| # 2.6) Add quotation marks to all strings => sed 's/.*/"&"/' | |
| # 2.7) Add comma suffix to all strings excluding the last one => sed '$!s/.*/&,/' | |
| # 2.8) Close JSON | |
| # 3) Save matrix JSON to output | |
| # This is old fetch command it cant work cause base_ref is only avaliable on pull request actions: git fetch origin ${{ github.base_ref }}:${{ github.base_ref }} | |
| run: | | |
| git fetch --no-tags --prune --depth=1 origin +refs/heads/${{ steps.myref.outputs.branch }}:refs/remotes/origin/${{ steps.myref.outputs.branch }} | |
| matrix=$(( | |
| echo '{ "package" : [' | |
| git ls-files *[^cli] | grep '/' | sort | uniq| sed 's,/*[^/]\+/*$,,' | sort | uniq | grep -v '^\.' | sed 's/.*/"&"/' | sed '$!s/.*/&,/' | |
| echo " ]}" | |
| ) | jq -c .) | |
| echo "::set-output name=matrix::$matrix" | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| check_matrix: | |
| runs-on: ubuntu-latest | |
| needs: build_strategy_matrix | |
| steps: | |
| - name: Install json2yaml | |
| run: | | |
| sudo npm install -g json2yaml | |
| - name: Check matrix definition | |
| run: | | |
| matrix='${{ needs.build_strategy_matrix.outputs.matrix }}' | |
| echo $matrix | |
| echo $matrix | jq . | |
| echo $matrix | json2yaml | |
| run_monorepo_tests: | |
| needs: build_strategy_matrix | |
| runs-on: ubuntu-latest | |
| strategy: | |
| # matrix: [{"package": some package that changed}, {...}, ...] | |
| matrix: ${{fromJson(needs.build_strategy_matrix.outputs.matrix)}} | |
| steps: | |
| # Source | |
| - name: Checkout current repo | |
| uses: actions/checkout@v4 | |
| with: | |
| path: functions | |
| # Install python 3.9 | |
| - name: Install python 3.9 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: 3.9 | |
| # Install dependencies | |
| - uses: actions/cache@v3 | |
| id: cache | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('functions/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install requirements | |
| run: | | |
| pip install --upgrade pip | |
| pip install -r functions/requirements.txt | |
| - name: Run py tests | |
| run: python functions/functions.py run-tests -r functions -s py -fn ${{ matrix.package }} | |
| continue-on-error: true | |
| # - name: Run ipynb tests | |
| # run: python functions/functions.py run-tests -r functions -s ipynb | |
| - name: Clean | |
| run: | | |
| rm -rf functions | |
| build-marketplace: | |
| name: Build marketplace | |
| if: github.repository == 'mlrun/functions' && github.event_name != 'pull_request' | |
| runs-on: ubuntu-latest | |
| needs: run_monorepo_tests | |
| continue-on-error: false | |
| steps: | |
| - name: Get the current branch name | |
| shell: bash | |
| run: echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" | |
| id: branch | |
| - name: Checkout current repo | |
| uses: actions/checkout@v4 | |
| with: | |
| path: functions | |
| - name: Checkout Marketplace | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: mlrun/marketplace | |
| path: marketplace | |
| - name: Install python 3.9 | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: 3.9 | |
| - name: Install requirements | |
| run: | | |
| cd functions | |
| pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Build marketplace docs | |
| env: | |
| CHANNEL: ${{ steps.branch.outputs.branch }} | |
| run: | | |
| cd marketplace | |
| pwd | |
| git pull origin | |
| cd .. | |
| python functions/functions.py build-marketplace -s functions -m marketplace -c $CHANNEL -v -f | |
| - name: Publish marketplace release | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.MARKETPLACE_ACCESS_TOKEN_V3 }} | |
| USERNAME: iguazio-cicd | |
| USEREMAIL: [email protected] | |
| REPO_PATH: marketplace | |
| BASE_REPO: mlrun | |
| BASE_BRANCH: master | |
| run: | | |
| cd marketplace | |
| pwd | |
| COMMIT_SHA=$(git rev-parse --short "$GITHUB_SHA") | |
| echo "commit sha: $COMMIT_SHA" | |
| echo "github sha: $GITHUB_SHA" | |
| BRANCH_NAME=marketplace-doc-gen-$COMMIT_SHA | |
| REMOTE=https://$USERNAME:[email protected]/$BASE_REPO/$REPO_PATH.git | |
| echo "Validating environment params..."; | |
| [ -z "${GITHUB_TOKEN}" ] && { | |
| echo 'Missing input "GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}".'; | |
| exit 1; | |
| }; | |
| git config --local user.name $USERNAME | |
| git config --local user.email $USEREMAIL | |
| git branch --set-upstream-to origin/master | |
| git remote -v | |
| echo "1. Checking out [$BRANCH_NAME]..." | |
| git checkout -b $BRANCH_NAME | |
| echo "2. Checking out [$BASE_BRANCH]..." | |
| git checkout $BASE_BRANCH | |
| git pull | |
| echo "3. Checking out [$BRANCH_NAME]..." | |
| git checkout $BRANCH_NAME | |
| echo "3a. merging" | |
| git merge $BASE_BRANCH | |
| echo "3b. status" | |
| git status | |
| git status --ignored | |
| find . -type f | xargs ls -artl | |
| echo "3b. add" | |
| git add --all | |
| git status | |
| git status --ignored | |
| echo "4. Commiting changes..." | |
| echo "4a. git rev-parse" | |
| git rev-parse --show-toplevel | |
| echo "4b. git commit" | |
| git commit -a -m "Automatically generated by github-worflow[bot] for commit: $COMMIT_SHA" | |
| git status | |
| git status --ignored | |
| # https://stackoverflow.com/questions/64270867/auth-error-trying-to-copy-a-repo-with-github-actions | |
| git config -l | grep 'http\..*\.extraheader' | cut -d= -f1 | xargs -L1 git config --unset-all | |
| echo "Pushing [$BRANCH_NAME] to remote [$REMOTE]" | |
| git push -f $REMOTE $BRANCH_NAME | |
| echo "Submiting pull request..." | |
| gh pr create --title "Marketplace update from $BRANCH_NAME" --body "github-workflow" --base $BASE_BRANCH --head $BRANCH_NAME --repo $BASE_REPO/$REPO_PATH |