CI: fix save benchmark (#4812)

juicedata · Nov 27, 2024 · 0a22d52 · 0a22d52
1 parent f96dd47
commit 0a22d52
Show file tree

Hide file tree

Showing 6 changed files with 72 additions and 21 deletions.
diff --git a/.github/actions/sysbench/action.yml b/.github/actions/sysbench/action.yml
@@ -13,8 +13,12 @@ inputs:
     description: 'file test mode'
     required: true
     default: 'seqrd'
-  mysql_password:
-    description: 'mysql password'
+  awk_access_key_id:
+    description: 'aws access key id'
+    required: true
+    default: ''
+  awk_access_key_secret:
+    description: 'aws access key secret'
     required: true
     default: ''
   meta:
@@ -52,6 +56,5 @@ runs:
         runtime=$((end-start))
         echo "cost $runtime seconds"
         cd -
-        export MYSQL_PASSWORD=${{inputs.mysql_password}}
         version=$(./juicefs -V|cut -b 17- | sed 's/:/-/g')
-        .github/scripts/save_benchmark.sh --name ${{inputs.name}} --result $runtime --meta ${{inputs.meta}} --storage ${{inputs.storage}} --extra ${{inputs.compress}}
+        AWS_ACCESS_KEY_ID=${{inputs.aws_access_key_id}} AWS_SECRET_ACCESS_KEY=${{inputs.aws_access_key_secret}} .github/scripts/save_benchmark.sh --name ${{inputs.name}} --result $runtime --meta ${{inputs.meta}} --storage ${{inputs.storage}} --extra ${{inputs.compress}}
diff --git a/.github/scripts/command/load_dump_bench.sh b/.github/scripts/command/load_dump_bench.sh
@@ -22,14 +22,14 @@ test_load_dump_with_small_dir(){
   end=`date +%s`
   runtime=$((end-start))
   version=$(./juicefs -V|cut -b 17- | sed 's/:/-/g')
-  .github/scripts/save_benchmark.sh --name load_small_dir --result $runtime --meta $META --storage file
+  .github/scripts/save_benchmark.sh --category load_dump --name load_small_dir --result $runtime --meta $META --storage file
   echo "load cost $runtime seconds"
   start=`date +%s`
   ./juicefs dump $META_URL dump.json
   end=`date +%s`
   runtime=$((end-start))
   echo "dump cost $runtime seconds"
-  .github/scripts/save_benchmark.sh --name dump_small_dir --result $runtime --meta $META --storage file
+  .github/scripts/save_benchmark.sh --category load_dump --name dump_small_dir --result $runtime --meta $META --storage file
   ./juicefs mount $META_URL /jfs -d --no-usage-report
   inode=$(df -i /jfs | grep JuiceFS |awk -F" " '{print $3}')
   if [ "$inode" -ne "2233313" ]; then 
@@ -60,7 +60,7 @@ do_load_dump_with_big_dir(){
   runtime=$((end-start))
   echo "load cost $runtime seconds"
   version=$(./juicefs -V|cut -b 17- | sed 's/:/-/g')
-  .github/scripts/save_benchmark.sh --name load_big_dir --result $runtime --meta $META --storage file
+  .github/scripts/save_benchmark.sh --category load_dump --name load_big_dir --result $runtime --meta $META --storage file
   start=`date +%s`
   if [ "$with_subdir" = true ] ; then
     ./juicefs dump $META_URL dump.json --subdir test
@@ -70,7 +70,7 @@ do_load_dump_with_big_dir(){
   end=`date +%s`
   runtime=$((end-start))
   echo "dump cost $runtime seconds"
-  .github/scripts/save_benchmark.sh --name dump_big_dir --result $runtime --meta $META --storage file
+  .github/scripts/save_benchmark.sh --category load_dump --name dump_big_dir --result $runtime --meta $META --storage file
   ./juicefs mount $META_URL /jfs -d --no-usage-report
   df -i /jfs
   inode=$(df -i /jfs | grep JuiceFS |awk -F" " '{print $3}')
@@ -89,7 +89,7 @@ test_list_with_big_dir(){
   runtime=$((end-start))
   echo "list cost $runtime seconds"
   version=$(./juicefs -V|cut -b 17- | sed 's/:/-/g')
-  .github/scripts/save_benchmark.sh --name list_big_dir --result $runtime --meta $META --storage file
+  .github/scripts/save_benchmark.sh --category list --name list_big_dir --result $runtime --meta $META --storage file
   if [ "$file_count" -ne "1000001" ]; then 
     echo "<FATAL>: file_count error: $file_count"
     exit 1

diff --git a/.github/scripts/save_benchmark.sh b/.github/scripts/save_benchmark.sh
@@ -4,6 +4,10 @@ save_benchmark(){
     while [[ $# -gt 0 ]]; do
         key="$1"
         case $key in
+            --category)
+                category="$2"
+                shift
+                ;;
             --name)
                 name="$2"
                 shift
@@ -39,6 +43,7 @@ save_benchmark(){
     created_date=$(date +"%Y-%m-%d")
     cat <<EOF > result.json
     {
+        "category": "$category",
         "name": "$name",
         "result": "$result",
         "meta": "$meta",
@@ -54,7 +59,7 @@ save_benchmark(){
     }
 EOF
     cat result.json
-    # AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_ACCESS_KEY_SECRET=$AWS_ACCESS_KEY_SECRET ./juicefs sync --force-update result.json s3://juicefs-ci-aws.s3.us-east-1.amazonaws.com/ci-report/fio-test/$created_date/$name
+    AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY ./juicefs sync --force-update result.json s3://juicefs-ci-aws.s3.us-east-1.amazonaws.com/ci-report/$category/$name/$created_date/$meta-$storage.json
 }
 
 save_benchmark $@
diff --git a/.github/workflows/fio_benchmark.yml b/.github/workflows/fio_benchmark.yml
@@ -126,6 +126,9 @@ jobs:
           echo "fio_job=$fio_job" >> $GITHUB_OUTPUT
 
       - name: Fio Benchmark 
+        env:
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
         run: |
           # set -o pipefail
           echo "Fio Benchmark"

diff --git a/.github/workflows/load.yml b/.github/workflows/load.yml
@@ -35,7 +35,7 @@ jobs:
           if [ "${{github.event_name}}" == "schedule"  ] || [ "${{github.event_name}}" == "workflow_dispatch"  ]; then
             echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
           else
-            echo 'meta_matrix=["redis", "mysql", "tikv"]' >> $GITHUB_OUTPUT
+            echo 'meta_matrix=["redis", "mysql"]' >> $GITHUB_OUTPUT
             # echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT
           fi
     outputs:
@@ -85,24 +85,44 @@ jobs:
 
       - name: Load and dump with small directory
         timeout-minutes: 30
+        env:
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
+          META: ${{matrix.meta}}
+          START_META: true
         run: |
-          sudo MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} META=${{matrix.meta}} START_META=true .github/scripts/command/load_dump_bench.sh test_load_dump_with_small_dir
+          sudo -E .github/scripts/command/load_dump_bench.sh test_load_dump_with_small_dir
 
       - name: Load and dump with big directory
         timeout-minutes: 30
+        env:
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
+          META: ${{matrix.meta}}
+          START_META: false
         run: |
-          sudo MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} META=${{matrix.meta}} START_META=false .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir
+          sudo -E .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir
       
       - name: Load and dump subdir with big directory
         if: false
         timeout-minutes: 30
+        env:
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
+          META: ${{matrix.meta}} 
+          START_META: false 
         run: |
-          sudo MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} META=${{matrix.meta}} START_META=false .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir_subdir
+          sudo -E .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir_subdir
           
       - name: List big directory
         timeout-minutes: 30
+        env:
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
+          META: ${{matrix.meta}} 
+          START_META: false 
         run: | 
-          sudo MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} META=${{matrix.meta}} START_META=false .github/scripts/command/load_dump_bench.sh test_list_with_big_dir
+          sudo -E .github/scripts/command/load_dump_bench.sh test_list_with_big_dir
 
       - name: log
         if: ${{ always() }}

diff --git a/.github/workflows/sysbench.yml b/.github/workflows/sysbench.yml
@@ -1,6 +1,20 @@
 name: "sysbench"
 
 on:
+  push:
+    branches:
+      - 'main'
+      - 'release-**'
+    paths:
+      - '**/sysbench.yml'
+      - '**/sysbench/action.yml'
+  pull_request:
+    branches:
+      - 'main'
+      - 'release-**'
+    paths:
+      - '**/sysbench.yml'
+      - '**/sysbench/action.yml'
   schedule:
     - cron:  '30 20 * * *'
   workflow_dispatch:
@@ -84,7 +98,8 @@ jobs:
       - name: seq read 1M
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          aws_access_key_id: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          aws_access_key_secret: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 1000
           file_total_size: '1G'
           file_test_mode: 'seqrd'
@@ -96,7 +111,8 @@ jobs:
       - name: seq write 1M
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          aws_access_key_id: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          aws_access_key_secret: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 1000
           file_total_size: '1G'
           file_test_mode: 'seqwr'
@@ -108,7 +124,8 @@ jobs:
       - name: seq read 100K
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          aws_access_key_id: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          aws_access_key_secret: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 10000
           file_total_size: '1G'
           file_test_mode: 'seqrd'
@@ -120,7 +137,8 @@ jobs:
       - name: seq write 100K
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          aws_access_key_id: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          aws_access_key_secret: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 10000
           file_total_size: '1G'
           file_test_mode: 'seqwr'
@@ -132,7 +150,8 @@ jobs:
       - name: random readwrite 1M
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          aws_access_key_id: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          aws_access_key_secret: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 1000
           file_total_size: '1G'
           file_test_mode: 'rndrw'
@@ -144,7 +163,8 @@ jobs:
       - name: random readwrite 100K
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          aws_access_key_id: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          aws_access_key_secret: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 10000
           file_total_size: '1G'
           file_test_mode: 'rndrw'