fix save benchmark

juicedata · May 6, 2024 · 744840e · 744840e
1 parent 3207f90
commit 744840e
Show file tree

Hide file tree

Showing 9 changed files with 94 additions and 16 deletions.
diff --git a/.github/scripts/hypo/command.py b/.github/scripts/hypo/command.py
@@ -50,9 +50,9 @@ class JuicefsCommandMachine(JuicefsMachine):
     MP2 = '/tmp/jfs2'
     ROOT_DIR1=os.path.join(MP1, 'fsrand')
     ROOT_DIR2=os.path.join(MP2, 'fsrand')
-    EXCLUDE_RULES = ['rebalance_dir', 'rebalance_file', 'config']
+    EXCLUDE_RULES = ['rebalance_dir', 'rebalance_file', 'config', 'dump_load_dump', 'dump']
     # EXCLUDE_RULES = []
-    INCLUDE_RULES = ['dump_load_dump', 'mkdir', 'create_file', 'set_xattr']
+    INCLUDE_RULES = ['dump_load_dump', 'mkdir', 'create_file', 'set_xattr', 'dump']
     cmd1 = CommandOperation('cmd1', MP1, ROOT_DIR1)
     cmd2 = CommandOperation('cmd2', MP2, ROOT_DIR2)
     fsop1 = FsOperation('fs1', ROOT_DIR1)
@@ -257,7 +257,7 @@ def teardown(self):
     settings.register_profile("dev", max_examples=MAX_EXAMPLE, verbosity=Verbosity.debug, 
         print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \
         report_multiple_bugs=False, 
-        phases=[Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain])
+        phases=[Phase.reuse, Phase.generate, Phase.target, Phase.explain])
     profile = os.environ.get('PROFILE', 'dev')
     settings.load_profile(profile)
 

diff --git a/.github/scripts/hypo/command_test.py b/.github/scripts/hypo/command_test.py
@@ -35,5 +35,11 @@ def test_config(self):
         state.config(capacity=1, enable_acl=True, encrypt_secret=True, force=False, inodes=81, trash_days=0, user='root', yes=True)
         state.teardown()
 
+    def test_dump(self):
+        state = JuicefsCommandMachine()
+        v1 = state.init_folders()
+        state.dump_load_dump(fast=True, folder=v1, keep_secret_key=True, skip_trash=False, threads=10, user='root')
+        state.teardown()
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/.github/scripts/hypo/fs.py b/.github/scripts/hypo/fs.py
@@ -508,7 +508,7 @@ def teardown(self):
     settings.register_profile("dev", max_examples=MAX_EXAMPLE, verbosity=Verbosity.debug, 
         print_blob=True, stateful_step_count=STEP_COUNT, deadline=None, \
         report_multiple_bugs=False, 
-        phases=[Phase.reuse, Phase.generate, Phase.target, Phase.shrink, Phase.explain])
+        phases=[Phase.reuse, Phase.generate, Phase.target, Phase.explain])
     settings.register_profile("schedule", max_examples=1000, verbosity=Verbosity.debug, 
         print_blob=True, stateful_step_count=200, deadline=None, \
         report_multiple_bugs=False, 

diff --git a/.github/scripts/hypo/fs_test.py b/.github/scripts/hypo/fs_test.py
@@ -53,5 +53,13 @@ def test_clone_918(self):
         v5 = state.clone_cp_file(entry=v3, new_entry_name='mbbb', parent=v1, preserve=True, user='root')
         state.teardown()
 
+    def test_xattr(self):
+        state = JuicefsMachine()
+        v1 = state.init_folders()
+        v19 = state.create_file(content=b'\xb5y', file_name='wums', mode='x', parent=v1, umask=16, user='root')
+        v29 = state.set_xattr(file=v19, flag=2, name='R', user='root', value=b'(9')
+        state.set_xattr(file=v19, flag=2, name='\x8ar', user='root', value=b'E$\xfe')
+        state.teardown()
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/.github/scripts/hypo/s3_test.py b/.github/scripts/hypo/s3_test.py
@@ -84,5 +84,43 @@ def test_issue_4682(self):
         state.get_bucket_policy(alias=v1, bucket_name=v6)
         state.teardown()
 
+    def test_issue_xxx(self):
+        state = S3Machine()
+        policies_0, policies_1, policies_2, policies_3 = state.init_policies()
+        aliases_0 = state.init_aliases()
+        state.list_groups(alias=aliases_0)
+        state.set_alias(alias='fxdc', url1='localhost:9000', url2='localhost:9005', user_name='user1')
+        users_0 = state.add_user(alias=aliases_0, user_name='user2')
+        state.set_alias(alias='cjxe', url1='localhost:9000', url2='localhost:9005', user_name='user3')
+        state.list_groups(alias=aliases_0)
+        buckets_0 = state.create_bucket(alias=aliases_0, bucket_name='zesf')
+        user_policy_0 = state.set_policy_to_user(alias=aliases_0, policy_name=policies_2, user_name=users_0)
+        user_policy_1 = state.set_policy_to_user(alias=aliases_0, policy_name=policies_2, user_name=users_0)
+        state.list_buckets(alias=aliases_0)
+        state.list_groups(alias=aliases_0)
+        buckets_1 = state.create_bucket(alias=aliases_0, bucket_name='lzzy')
+        groups_0 = state.add_group(alias=aliases_0, group_name='group1', members=[users_0])
+        state.set_alias(alias='zciu', url1='localhost:9000', url2='localhost:9005', user_name='user3')
+        state.list_buckets(alias=aliases_0)
+        state.list_buckets(alias=aliases_0)
+        groups_1 = state.add_group(alias=aliases_0, group_name='group2', members=[users_0])
+        user_policy_2 = state.set_policy_to_user(alias=aliases_0, policy_name=policies_2, user_name=users_0)
+        user_policy_3 = state.set_policy_to_user(alias=aliases_0, policy_name=policies_0, user_name=users_0)
+        groups_2 = state.add_group(alias=aliases_0, group_name='group3', members=[users_0])
+        state.set_alias(alias='yjjd', url1='localhost:9000', url2='localhost:9005', user_name='user1')
+        state.list_groups(alias=aliases_0)
+        state.list_buckets(alias=aliases_0)
+        aliases_1 = state.set_alias(alias='rbkk', url1='localhost:9000', url2='localhost:9005', user_name=users_0)
+        user_policy_4 = state.set_policy_to_user(alias=aliases_0, policy_name=policies_3, user_name=users_0)
+        users_1 = state.add_user(alias=aliases_1, user_name=users_0)
+        user_policy_5 = state.set_policy_to_user(alias=aliases_0, policy_name=policies_2, user_name=users_0)
+        users_2 = state.add_user(alias=aliases_1, user_name=users_0)
+        state.list_buckets(alias=aliases_0)
+        user_policy_6 = state.set_policy_to_user(alias=aliases_0, policy_name=policies_3, user_name=users_0)
+        buckets_2 = state.create_bucket(alias=aliases_0, bucket_name='ktvl')
+        state.create_bucket(alias=aliases_1, bucket_name='wopa')
+        state.get_bucket_policy(alias=aliases_1, bucket_name=buckets_1)
+        state.teardown()
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/.github/scripts/save_benchmark.sh b/.github/scripts/save_benchmark.sh
@@ -54,7 +54,7 @@ save_benchmark(){
     }
 EOF
     cat result.json
-    # AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_ACCESS_KEY_SECRET=$AWS_ACCESS_KEY_SECRET ./juicefs sync --force-update result.json s3://juicefs-ci-aws.s3.us-east-1.amazonaws.com/ci-report/fio-test/$created_date/$name
+    AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY ./juicefs sync --force-update result.json s3://juicefs-ci-aws.s3.us-east-1.amazonaws.com/ci-report/fio-test/$created_date/$name
 }
 
 save_benchmark $@
diff --git a/.github/workflows/load.yml b/.github/workflows/load.yml
@@ -87,24 +87,44 @@ jobs:
 
       - name: Load and dump with small directory
         timeout-minutes: 30
+        env:
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
+          META: ${{matrix.meta}}
+          START_META: true
         run: |
-          sudo MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} META=${{matrix.meta}} START_META=true .github/scripts/command/load_dump_bench.sh test_load_dump_with_small_dir
+          sudo -E .github/scripts/command/load_dump_bench.sh test_load_dump_with_small_dir
 
       - name: Load and dump with big directory
         timeout-minutes: 30
+        env:
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
+          META: ${{matrix.meta}}
+          START_META: false
         run: |
-          sudo MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} META=${{matrix.meta}} START_META=false .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir
+          sudo -E .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir
       
       - name: Load and dump subdir with big directory
         if: false
         timeout-minutes: 30
+        env:
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
+          META: ${{matrix.meta}} 
+          START_META: false 
         run: |
-          sudo MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} META=${{matrix.meta}} START_META=false .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir_subdir
+          sudo -E .github/scripts/command/load_dump_bench.sh test_load_dump_with_big_dir_subdir
           
       - name: List big directory
         timeout-minutes: 30
+        env:
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
+          META: ${{matrix.meta}} 
+          START_META: false 
         run: | 
-          sudo MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} META=${{matrix.meta}} START_META=false .github/scripts/command/load_dump_bench.sh test_list_with_big_dir
+          sudo -E .github/scripts/command/load_dump_bench.sh test_list_with_big_dir
 
       - name: log
         if: always()

diff --git a/.github/workflows/resources/vdbench_long_run.conf b/.github/workflows/resources/vdbench_long_run.conf
@@ -1,4 +1,4 @@
 data_errors=1
 fsd=fsd1,anchor=/tmp/jfs,depth=1,width=2,files=2,sizes=(30m,0),shared=yes,openflags=o_direct
 fwd=fwd1,fsd=fsd1,threads=4,xfersize=(512,20,4k,20,64k,20,512k,20,1024k,20),fileio=random,fileselect=random,rdpct=70
-rd=rd1,fwd=fwd*,fwdrate=max,format=restart,elapsed=1800,interval=1
+rd=rd1,fwd=fwd*,fwdrate=max,format=restart,elapsed=7200,interval=1
diff --git a/.github/workflows/sysbench.yml b/.github/workflows/sysbench.yml
@@ -86,7 +86,8 @@ jobs:
       - name: seq read 1M
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 1000
           file_total_size: '1G'
           file_test_mode: 'seqrd'
@@ -98,7 +99,8 @@ jobs:
       - name: seq write 1M
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 1000
           file_total_size: '1G'
           file_test_mode: 'seqwr'
@@ -110,7 +112,8 @@ jobs:
       - name: seq read 100K
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 10000
           file_total_size: '1G'
           file_test_mode: 'seqrd'
@@ -122,7 +125,8 @@ jobs:
       - name: seq write 100K
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 10000
           file_total_size: '1G'
           file_test_mode: 'seqwr'
@@ -134,7 +138,8 @@ jobs:
       - name: random readwrite 1M
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 1000
           file_total_size: '1G'
           file_test_mode: 'rndrw'
@@ -146,7 +151,8 @@ jobs:
       - name: random readwrite 100K
         uses: ./.github/actions/sysbench
         with:
-          mysql_password: ${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}}
+          AWS_ACCESS_KEY_ID: ${{secrets.CI_AWS_ACCESS_KEY_ID}}
+          AWS_SECRET_ACCESS_KEY: ${{secrets.CI_AWS_ACCESS_KEY_SECRET}}
           file_num: 10000
           file_total_size: '1G'
           file_test_mode: 'rndrw'