-
Notifications
You must be signed in to change notification settings - Fork 54
189 lines (182 loc) · 7.04 KB
/
test-all.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
name: Test all functions, build marketplace
on:
push:
branches:
- development
- master
jobs:
build_strategy_matrix:
runs-on: ubuntu-latest
steps:
- name: Get the current branch name
shell: bash
run: echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
id: myref
- uses: actions/checkout@v4
- id: set-matrix
# This is very hacky, but it goes like that:
# 1) Associate base_ref with origin/base_ref since actions/checkout doesn't do it, if we don't do that we won't be able to check the actual diff
# 2) Build JSON string
# 2.1) Add beginning of JSON
# 2.2) Get diff between origin/base_ref and the checked-out repo => git diff ${{ github.base_ref }} --name-only
# 2.3) Clean the file name and leave us only with directories => sed 's,/*[^/]\+/*$,,'
# 2.4) Sort and keep only unique directories => sort | uniq
# 2.5) Remove directories starting with '.' => grep -v '^\.'
# 2.6) Add quotation marks to all strings => sed 's/.*/"&"/'
# 2.7) Add comma suffix to all strings excluding the last one => sed '$!s/.*/&,/'
# 2.8) Close JSON
# 3) Save matrix JSON to output
# This is old fetch command it cant work cause base_ref is only avaliable on pull request actions: git fetch origin ${{ github.base_ref }}:${{ github.base_ref }}
run: |
git fetch --no-tags --prune --depth=1 origin +refs/heads/${{ steps.myref.outputs.branch }}:refs/remotes/origin/${{ steps.myref.outputs.branch }}
matrix=$((
echo '{ "package" : ['
git ls-files *[^cli] | grep '/' | sort | uniq| sed 's,/*[^/]\+/*$,,' | sort | uniq | grep -v '^\.' | sed 's/.*/"&"/' | sed '$!s/.*/&,/'
echo " ]}"
) | jq -c .)
echo "::set-output name=matrix::$matrix"
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
check_matrix:
runs-on: ubuntu-latest
needs: build_strategy_matrix
steps:
- name: Install json2yaml
run: |
sudo npm install -g json2yaml
- name: Check matrix definition
run: |
matrix='${{ needs.build_strategy_matrix.outputs.matrix }}'
echo $matrix
echo $matrix | jq .
echo $matrix | json2yaml
run_monorepo_tests:
needs: build_strategy_matrix
runs-on: ubuntu-latest
strategy:
# matrix: [{"package": some package that changed}, {...}, ...]
matrix: ${{fromJson(needs.build_strategy_matrix.outputs.matrix)}}
steps:
# Source
- name: Checkout current repo
uses: actions/checkout@v4
with:
path: functions
# Install python 3.9
- name: Install python 3.9
uses: actions/setup-python@v3
with:
python-version: 3.9
# Install dependencies
- uses: actions/cache@v3
id: cache
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('functions/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install requirements
run: |
pip install --upgrade pip
pip install -r functions/requirements.txt
- name: Run py tests
run: python functions/functions.py run-tests -r functions -s py -fn ${{ matrix.package }}
continue-on-error: true
# - name: Run ipynb tests
# run: python functions/functions.py run-tests -r functions -s ipynb
- name: Clean
run: |
rm -rf functions
build-marketplace:
name: Build marketplace
if: github.repository == 'mlrun/functions' && github.event_name != 'pull_request'
runs-on: ubuntu-latest
needs: run_monorepo_tests
continue-on-error: false
steps:
- name: Get the current branch name
shell: bash
run: echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
id: branch
- name: Checkout current repo
uses: actions/checkout@v4
with:
path: functions
- name: Checkout Marketplace
uses: actions/checkout@v4
with:
repository: mlrun/marketplace
path: marketplace
- name: Install python 3.9
uses: actions/setup-python@v3
with:
python-version: 3.9
- name: Install requirements
run: |
cd functions
pip install --upgrade pip
pip install -r requirements.txt
- name: Build marketplace docs
env:
CHANNEL: ${{ steps.branch.outputs.branch }}
run: |
cd marketplace
pwd
git pull origin
cd ..
python functions/functions.py build-marketplace -s functions -m marketplace -c $CHANNEL -v -f
- name: Publish marketplace release
env:
GITHUB_TOKEN: ${{ secrets.MARKETPLACE_ACCESS_TOKEN_V3 }}
USERNAME: iguazio-cicd
USEREMAIL: [email protected]
REPO_PATH: marketplace
BASE_REPO: mlrun
BASE_BRANCH: master
run: |
cd marketplace
pwd
COMMIT_SHA=$(git rev-parse --short "$GITHUB_SHA")
echo "commit sha: $COMMIT_SHA"
echo "github sha: $GITHUB_SHA"
BRANCH_NAME=marketplace-doc-gen-$COMMIT_SHA
REMOTE=https://$USERNAME:[email protected]/$BASE_REPO/$REPO_PATH.git
echo "Validating environment params...";
[ -z "${GITHUB_TOKEN}" ] && {
echo 'Missing input "GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}".';
exit 1;
};
git config --local user.name $USERNAME
git config --local user.email $USEREMAIL
git branch --set-upstream-to origin/master
git remote -v
echo "1. Checking out [$BRANCH_NAME]..."
git checkout -b $BRANCH_NAME
echo "2. Checking out [$BASE_BRANCH]..."
git checkout $BASE_BRANCH
git pull
echo "3. Checking out [$BRANCH_NAME]..."
git checkout $BRANCH_NAME
echo "3a. merging"
git merge $BASE_BRANCH
echo "3b. status"
git status
git status --ignored
find . -type f | xargs ls -artl
echo "3b. add"
git add --all
git status
git status --ignored
echo "4. Commiting changes..."
echo "4a. git rev-parse"
git rev-parse --show-toplevel
echo "4b. git commit"
git commit -a -m "Automatically generated by github-worflow[bot] for commit: $COMMIT_SHA"
git status
git status --ignored
# https://stackoverflow.com/questions/64270867/auth-error-trying-to-copy-a-repo-with-github-actions
git config -l | grep 'http\..*\.extraheader' | cut -d= -f1 | xargs -L1 git config --unset-all
echo "Pushing [$BRANCH_NAME] to remote [$REMOTE]"
git push -f $REMOTE $BRANCH_NAME
echo "Submiting pull request..."
gh pr create --title "Marketplace update from $BRANCH_NAME" --body "github-workflow" --base $BASE_BRANCH --head $BRANCH_NAME --repo $BASE_REPO/$REPO_PATH