From 29ae6a378a890a6ac1b5a45d3937048297387524 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Thu, 18 Apr 2024 11:24:21 +0800
Subject: [PATCH 01/13] refactor test

---
 .github/scripts/hypo/command.py    | 35 +++++++-------
 .github/scripts/hypo/command_op.py |  4 +-
 .github/scripts/hypo/common.py     | 11 ++++-
 .github/scripts/hypo/fs.py         | 73 ++++++++++++------------------
 .github/scripts/hypo/fs_op.py      | 16 ++++++-
 .github/scripts/hypo/s3.py         | 17 ++-----
 6 files changed, 76 insertions(+), 80 deletions(-)

diff --git a/.github/scripts/hypo/command.py b/.github/scripts/hypo/command.py
index 7421bc128152..bc4fd97de00a 100644
--- a/.github/scripts/hypo/command.py
+++ b/.github/scripts/hypo/command.py
@@ -34,6 +34,7 @@
 from fs_op import FsOperation
 from command_op import CommandOperation
 from fs import JuicefsMachine
+import common
 
 SEED=int(os.environ.get('SEED', random.randint(0, 1000000000)))
 
@@ -50,8 +51,17 @@ class JuicefsCommandMachine(JuicefsMachine):
     EXCLUDE_RULES = ['rebalance_dir', 'rebalance_file']
     # EXCLUDE_RULES = []
     INCLUDE_RULES = ['dump_load_dump', 'mkdir', 'create_file', 'set_xattr']
-    cmd1 = CommandOperation('cmd1', mp=MP1)
-    cmd2 = CommandOperation('cmd2', mp=MP2)
+
+    def __init__(self):
+        super().__init__()
+        self.cmd1 = CommandOperation('cmd1', self.MP1, self.get_default_rootdir1())
+        self.cmd2 = CommandOperation('cmd2', self.MP2, self.get_default_rootdir2())
+
+    def get_default_rootdir1(self):
+        return os.path.join(self.MP1, 'fsrand')
+    
+    def get_default_rootdir2(self):
+        return os.path.join(self.MP2, 'fsrand')
 
     def equal(self, result1, result2):
         if type(result1) != type(result2):
@@ -59,27 +69,16 @@ def equal(self, result1, result2):
         if isinstance(result1, Exception):
             if 'panic:' in str(result1) or 'panic:' in str(result2):
                 return False
-            r1 = str(result1).replace(self.MP1, '')
-            r2 = str(result2).replace(self.MP2, '')
-            return r1 == r2
-        elif isinstance(result1, str):
-            r1 = str(result1).replace(self.MP1, '')
-            r2 = str(result2).replace(self.MP2, '')
-            return r1 == r2
-        elif isinstance(result1, tuple):
-            r1 = [str(item).replace(self.MP1, '') for item in result1]
-            r2 = [str(item).replace(self.MP2, '') for item in result2]
-            return r1 == r2
-        else:
-            return  result1 == result2
+            result1 = str(result1)
+            result2 = str(result2)
+        result1 = common.replace(result1, self.MP1, '***')
+        result2 = common.replace(result2, self.MP2, '***')
+        return result1 == result2
 
     def get_client_version(self, mount):
         output = run_cmd(f'{mount} version')
         return output.split()[2]
 
-    def __init__(self):
-        super().__init__()
-
     def should_run(self, rule):
         if len(self.EXCLUDE_RULES) > 0:
             return rule not in self.EXCLUDE_RULES
diff --git a/.github/scripts/hypo/command_op.py b/.github/scripts/hypo/command_op.py
index c4456732dc3b..caac231ab166 100644
--- a/.github/scripts/hypo/command_op.py
+++ b/.github/scripts/hypo/command_op.py
@@ -22,10 +22,10 @@
 class CommandOperation:
     JFS_CONTROL_FILES=['.accesslog', '.config', '.stats']
     stats = Statistics()
-    def __init__(self, name, mp):
+    def __init__(self, name, mp, root_dir):
         self.logger = common.setup_logger(f'./{name}.log', name, os.environ.get('LOG_LEVEL', 'INFO'))
         self.mp = mp
-        self.root_dir = self.mp+'/fsrand'
+        self.root_dir = root_dir
         self.meta_url = self.get_meta_url(mp)
                 
     def get_meta_url(self, mp):
diff --git a/.github/scripts/hypo/common.py b/.github/scripts/hypo/common.py
index 0c77db184d8f..5fad9e42cc23 100644
--- a/.github/scripts/hypo/common.py
+++ b/.github/scripts/hypo/common.py
@@ -6,8 +6,15 @@
 import pwd
 import subprocess
 import sys
-
-
+def replace(self, src, old, new):
+    if isinstance(src, str):
+        return src.replace(old, new).replace(old, new)
+    elif isinstance(src, list):
+        return [self.replace(x, old, new) for x in src]
+    elif isinstance(src, dict):
+        return {k: self.replace(v, old, new) for k, v in src.items()}
+    else:
+        return src
 def run_cmd(command: str) -> str:
     print('run_cmd:'+command)
     if '|' in command or '>' in command:
diff --git a/.github/scripts/hypo/fs.py b/.github/scripts/hypo/fs.py
index 8f89f0fea44e..312c4e3c3eba 100644
--- a/.github/scripts/hypo/fs.py
+++ b/.github/scripts/hypo/fs.py
@@ -30,13 +30,6 @@ class JuicefsMachine(RuleBasedStateMachine):
     EntryWithACL = Bundle('entry_with_acl')
     FilesWithXattr = Bundle('files_with_xattr')
     start = time.time()
-    ROOT_DIR1=os.environ.get('ROOT_DIR1', '/tmp/fsrand').rstrip('/')
-    ROOT_DIR2=os.environ.get('ROOT_DIR2', '/tmp/jfs/fsrand').rstrip('/')
-    
-    fsop1 = FsOperation('fs1', ROOT_DIR1)
-    fsop2 = FsOperation('fs2', ROOT_DIR2)
-
-    ZONES = {ROOT_DIR1:common.get_zones(ROOT_DIR1), ROOT_DIR2:common.get_zones(ROOT_DIR2)}
     SUDO_USERS = ['root']
     USERS=['root', 'user1', 'user2','user3']
     GROUPS = USERS+['group1', 'group2', 'group3', 'group4']
@@ -47,13 +40,8 @@ class JuicefsMachine(RuleBasedStateMachine):
     
     @initialize(target=Folders)
     def init_folders(self):
-        if not os.path.exists(self.ROOT_DIR1):
-            os.makedirs(self.ROOT_DIR1)
-        if not os.path.exists(self.ROOT_DIR2):
-            os.makedirs(self.ROOT_DIR2)
-        if os.environ.get('PROFILE', 'dev') != 'generate':
-            common.clean_dir(self.ROOT_DIR1)
-            common.clean_dir(self.ROOT_DIR2)
+        self.fsop1.init_rootdir()
+        self.fsop2.init_rootdir()
         return ''
     
     def create_users(self, users):
@@ -61,11 +49,17 @@ def create_users(self, users):
             if user != 'root':
                 common.create_user(user)
 
+    def get_default_rootdir1(self):
+        return '/tmp/fsrand'
+    
+    def get_default_rootdir2(self):
+        return '/tmp/jfs/fsrand'
+
     def __init__(self):
         super(JuicefsMachine, self).__init__()
         print(f'__init__')
-        if os.environ.get('EXCLUDE_RULES') is not None:
-            self.EXCLUDE_RULES = os.environ.get('EXCLUDE_RULES').split(',')
+        self.fsop1 = FsOperation('fs1', os.environ.get('ROOT_DIR1', self.get_default_rootdir1()))
+        self.fsop2 = FsOperation('fs2', os.environ.get('ROOT_DIR2', self.get_default_rootdir2()))
         if not self.group_created:
             for group in self.GROUPS:
                 common.create_group(group)
@@ -83,17 +77,13 @@ def equal(self, result1, result2):
         if type(result1) != type(result2):
             return False
         if isinstance(result1, Exception):
-            r1 = str(result1).replace(self.ROOT_DIR1, '')
-            r2 = str(result2).replace(self.ROOT_DIR2, '')
-            return r1 == r2
-        elif isinstance(result1, tuple):
-            return result1 == result2
-        elif isinstance(result1, str):
-            r1 = str(result1).replace(self.ROOT_DIR1, '')
-            r2 = str(result2).replace(self.ROOT_DIR2, '')
-            return  r1 == r2
-        else:
-            return result1 == result2
+            if 'panic:' in str(result1) or 'panic:' in str(result2):
+                return False
+            result1 = str(result1)
+            result2 = str(result2)
+        result1 = common.replace(result1, self.fsop1.root_dir, '***')
+        result2 = common.replace(result2, self.fsop2.root_dir, '***')
+        return result1 == result2
 
     def seteuid(self, user):
         os.seteuid(pwd.getpwnam(user).pw_uid)
@@ -472,38 +462,37 @@ def chown(self, entry, owner, user='root'):
      
     @rule( dir =Folders, vdirs = st.integers(min_value=2, max_value=31) )
     @precondition(lambda self: self.should_run('split_dir') \
-                  and (common.is_jfs(self.ROOT_DIR1) or common.is_jfs(self.ROOT_DIR2))
+                  and (self.fsop1.is_jfs() or self.fsop2.is_jfs())
     )
     def split_dir(self, dir, vdirs):
         self.fsop1.do_split_dir(dir, vdirs)
         self.fsop2.do_split_dir(dir, vdirs)
-    
 
     @rule(dir = Folders)
     @precondition(lambda self: self.should_run('merge_dir') \
-                   and (common.is_jfs(self.ROOT_DIR1) or common.is_jfs(self.ROOT_DIR2))
+                 and (self.fsop1.is_jfs() or self.fsop2.is_jfs())
     )
     def merge_dir(self, dir):
         self.fsop1.do_merge_dir(dir)
         self.fsop2.do_merge_dir(dir)
     
-    @rule(dir = Folders,
-          zone1=st.sampled_from(ZONES[ROOT_DIR1]),
-          zone2=st.sampled_from(ZONES[ROOT_DIR2]),
-          is_vdir=st.booleans())
+    # @rule(dir = Folders,
+    #       zone1=st.sampled_from(self.fsop1.get_zones()),
+    #       zone2=st.sampled_from(self.fsop2.get_zones()),
+    #       is_vdir=st.booleans())
     @precondition(lambda self: self.should_run('rebalance_dir') \
-                   and (common.is_jfs(self.ROOT_DIR1) or common.is_jfs(self.ROOT_DIR2))
+                   and (self.fsop1.is_jfs() or self.fsop2.is_jfs())
     )
     def rebalance_dir(self, dir, zone1, zone2, is_vdir):
         self.fsop1.do_rebalance(dir, zone1, is_vdir)
         self.fsop2.do_rebalance(dir, zone2, is_vdir)
 
-    @rule(file = Files, 
-          zone1=st.sampled_from(ZONES[ROOT_DIR1]),
-          zone2=st.sampled_from(ZONES[ROOT_DIR2]),
-          )
+    # @rule(file = Files, 
+    #       zone1=st.sampled_from(self.fsop1.get_zones()),
+    #       zone2=st.sampled_from(self.fsop2.get_zones()),
+    #       )
     @precondition(lambda self: self.should_run('rebalance_file') \
-                   and (common.is_jfs(self.ROOT_DIR1) or common.is_jfs(self.ROOT_DIR2))
+                   and (self.fsop1.is_jfs() or self.fsop2.is_jfs())
     )
     def rebalance_file(self, file, zone1, zone2):
         self.fsop1.do_rebalance(file, zone1, False)
@@ -511,10 +500,6 @@ def rebalance_file(self, file, zone1, zone2):
 
     def teardown(self):
         pass
-        # if COMPARE and os.path.exists(ROOT_DIR1):
-        #     common.compare_content(ROOT_DIR1, ROOT_DIR2)
-        #     common.compare_stat(ROOT_DIR1, ROOT_DIR2)
-        #     common.compare_acl(ROOT_DIR1, ROOT_DIR2)
 
 if __name__ == '__main__':
     MAX_EXAMPLE=int(os.environ.get('MAX_EXAMPLE', '100'))
diff --git a/.github/scripts/hypo/fs_op.py b/.github/scripts/hypo/fs_op.py
index 41bf775f8f68..b43fe994aa02 100644
--- a/.github/scripts/hypo/fs_op.py
+++ b/.github/scripts/hypo/fs_op.py
@@ -23,9 +23,9 @@
 class FsOperation:
     JFS_CONTROL_FILES=['.accesslog', '.config', '.stats']
     stats = Statistics()
-    def __init__(self, name, root_dir):
+    def __init__(self, name, root_dir:str):
         self.logger =common.setup_logger(f'./{name}.log', name, os.environ.get('LOG_LEVEL', 'INFO'))
-        self.root_dir = root_dir
+        self.root_dir = root_dir.rstrip('/')
 
     def run_cmd(self, command:str) -> str:
         self.logger.info(f'run_cmd: {command}')
@@ -41,6 +41,18 @@ def run_cmd(self, command:str) -> str:
             raise e
         return output.stdout.decode()
 
+    def get_zones(self):
+        return common.get_zones(self.root_dir)
+
+    def is_jfs(self):
+        return common.is_jfs(self.root_dir)
+
+    def init_rootdir(self):
+        if not os.path.exists(self.root_dir):
+            os.makedirs(self.root_dir)
+        if os.environ.get('PROFILE', 'dev') != 'generate':
+            common.clean_dir(self.root_dir)
+
     def seteuid(self, user):
         os.seteuid(pwd.getpwnam(user).pw_uid)
         os.setegid(pwd.getpwnam(user).pw_gid)
diff --git a/.github/scripts/hypo/s3.py b/.github/scripts/hypo/s3.py
index f0195ee73c00..c074d2d6915c 100644
--- a/.github/scripts/hypo/s3.py
+++ b/.github/scripts/hypo/s3.py
@@ -14,6 +14,7 @@
 from s3_op import S3Client
 from s3_strategy import *
 from s3_contant import *
+import common
 # minio client: https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-04-22T17-40-00Z
 # minio server: minio/minio:RELEASE.2021-04-22T15-44-28Z
 # docker run -d -p 9000:9000 --name minio -e "MINIO_ACCESS_KEY=minioadmin" -e "MINIO_SECRET_KEY=minioadmin" minio/minio:RELEASE.2021-04-22T15-44-28Z server /data
@@ -62,16 +63,6 @@ def init_aliases(self):
     @initialize(target=policies)
     def init_policies(self):
         return multiple(*BUILD_IN_POLICIES)
-            
-    def replace(self, result, prefix, url):
-        if isinstance(result, str):
-            return result.replace(prefix, '***').replace(url, '***')
-        elif isinstance(result, list):
-            return [self.replace(x, prefix, url) for x in result]
-        elif isinstance(result, dict):
-            return {k: self.replace(v, prefix, url) for k, v in result.items()}
-        else:
-            return result
 
     def equal(self, result1, result2):
         if os.getenv('PROFILE', 'dev') == 'generate':
@@ -81,8 +72,10 @@ def equal(self, result1, result2):
         if isinstance(result1, Exception):
             result1 = str(result1)
             result2 = str(result2)
-        result1 = self.replace(result1, self.PREFIX1, self.URL1)
-        result2 = self.replace(result2, self.PREFIX2, self.URL2)
+        result1 = common.replace(result1, self.PREFIX1, '***')
+        result1 = common.replace(result1, self.URL1, '***')
+        result2 = common.replace(result2, self.PREFIX2, '***')
+        result2 = common.replace(result2, self.URL2, '***')
         # print(f'result1 is {result1}\nresult2 is {result2}')
         return result1 == result2
 

From eca4df7a89948f538b5347a71221b5ff05af685f Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Thu, 18 Apr 2024 11:35:00 +0800
Subject: [PATCH 02/13] wip

---
 .github/scripts/hypo/common.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/scripts/hypo/common.py b/.github/scripts/hypo/common.py
index 5fad9e42cc23..ef94f7c1f02f 100644
--- a/.github/scripts/hypo/common.py
+++ b/.github/scripts/hypo/common.py
@@ -6,13 +6,13 @@
 import pwd
 import subprocess
 import sys
-def replace(self, src, old, new):
+def replace(src, old, new):
     if isinstance(src, str):
-        return src.replace(old, new).replace(old, new)
+        return src.replace(old, new)
     elif isinstance(src, list):
-        return [self.replace(x, old, new) for x in src]
+        return [replace(x, old, new) for x in src]
     elif isinstance(src, dict):
-        return {k: self.replace(v, old, new) for k, v in src.items()}
+        return {k: replace(v, old, new) for k, v in src.items()}
     else:
         return src
 def run_cmd(command: str) -> str:

From 0876ae82f089dd08d8cb7a8e675db3ff5e20d7d3 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Thu, 18 Apr 2024 11:54:01 +0800
Subject: [PATCH 03/13] wip

---
 .github/workflows/fsrand.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/fsrand.yml b/.github/workflows/fsrand.yml
index e693e7dc4970..a9f7b5762a8b 100644
--- a/.github/workflows/fsrand.yml
+++ b/.github/workflows/fsrand.yml
@@ -103,7 +103,7 @@ jobs:
           else
             PROFILE='pull_request'
           fi
-          sudo -E PROFILE=${PROFILE} python3 .github/scripts/hypo/fs.py 2>&1 | tee fsrand.log
+          sudo -E PROFILE=${PROFILE} LOG_LEVEL=WARNING python3 .github/scripts/hypo/fs.py 2>&1 | tee fsrand.log
           exit ${PIPESTATUS[0]}
 
       - name: check fsrand.log

From dd698d772707cc16030d4b087ada7dcc776142cf Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Thu, 18 Apr 2024 12:01:18 +0800
Subject: [PATCH 04/13] wip

---
 .github/scripts/hypo/command.py | 11 +++++++----
 .github/scripts/hypo/fs.py      |  7 ++++---
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/.github/scripts/hypo/command.py b/.github/scripts/hypo/command.py
index bc4fd97de00a..2effcb21269e 100644
--- a/.github/scripts/hypo/command.py
+++ b/.github/scripts/hypo/command.py
@@ -48,15 +48,18 @@ class JuicefsCommandMachine(JuicefsMachine):
     Entries = Files | Folders
     MP1 = '/tmp/jfs1'
     MP2 = '/tmp/jfs2'
+    ROOT_DIR1=os.path.join(MP1, 'fsrand')
+    ROOT_DIR2=os.path.join(MP2, 'fsrand')
     EXCLUDE_RULES = ['rebalance_dir', 'rebalance_file']
     # EXCLUDE_RULES = []
     INCLUDE_RULES = ['dump_load_dump', 'mkdir', 'create_file', 'set_xattr']
-
+    cmd1 = CommandOperation('cmd1', MP1, ROOT_DIR1)
+    cmd2 = CommandOperation('cmd2', MP2, ROOT_DIR2)
+    fsop1 = FsOperation('fs1', ROOT_DIR1)
+    fsop2 = FsOperation('fs2', ROOT_DIR2)
     def __init__(self):
         super().__init__()
-        self.cmd1 = CommandOperation('cmd1', self.MP1, self.get_default_rootdir1())
-        self.cmd2 = CommandOperation('cmd2', self.MP2, self.get_default_rootdir2())
-
+        
     def get_default_rootdir1(self):
         return os.path.join(self.MP1, 'fsrand')
     
diff --git a/.github/scripts/hypo/fs.py b/.github/scripts/hypo/fs.py
index 312c4e3c3eba..d378105dd34f 100644
--- a/.github/scripts/hypo/fs.py
+++ b/.github/scripts/hypo/fs.py
@@ -37,7 +37,9 @@ class JuicefsMachine(RuleBasedStateMachine):
     INCLUDE_RULES = []
     EXCLUDE_RULES = ['rebalance_dir', 'rebalance_file', \
                         'clone_cp_file', 'clone_cp_dir']
-    
+    fsop1 = FsOperation('fs1', os.environ.get('ROOT_DIR1', '/tmp/fsrand'))
+    fsop2 = FsOperation('fs2', os.environ.get('ROOT_DIR2', '/tmp/jfs/fsrand'))
+
     @initialize(target=Folders)
     def init_folders(self):
         self.fsop1.init_rootdir()
@@ -58,8 +60,7 @@ def get_default_rootdir2(self):
     def __init__(self):
         super(JuicefsMachine, self).__init__()
         print(f'__init__')
-        self.fsop1 = FsOperation('fs1', os.environ.get('ROOT_DIR1', self.get_default_rootdir1()))
-        self.fsop2 = FsOperation('fs2', os.environ.get('ROOT_DIR2', self.get_default_rootdir2()))
+        
         if not self.group_created:
             for group in self.GROUPS:
                 common.create_group(group)

From 628a6d3c0556e2e268ba843c89fa99c3b2367e6f Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Thu, 18 Apr 2024 12:07:38 +0800
Subject: [PATCH 05/13] wip

---
 .github/scripts/hypo/command.py      |  1 +
 .github/scripts/hypo/command_op.py   |  2 +-
 .github/scripts/hypo/command_test.py | 35 +++++++++++++++++++++++++++-
 .github/scripts/hypo/common.py       |  2 +-
 4 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/.github/scripts/hypo/command.py b/.github/scripts/hypo/command.py
index 2effcb21269e..3039986c4848 100644
--- a/.github/scripts/hypo/command.py
+++ b/.github/scripts/hypo/command.py
@@ -76,6 +76,7 @@ def equal(self, result1, result2):
             result2 = str(result2)
         result1 = common.replace(result1, self.MP1, '***')
         result2 = common.replace(result2, self.MP2, '***')
+        print(f'result1 is {result1}\nresult2 is {result2}')
         return result1 == result2
 
     def get_client_version(self, mount):
diff --git a/.github/scripts/hypo/command_op.py b/.github/scripts/hypo/command_op.py
index caac231ab166..25ffff34b54c 100644
--- a/.github/scripts/hypo/command_op.py
+++ b/.github/scripts/hypo/command_op.py
@@ -102,7 +102,7 @@ def parse_info(self, info: str):
                     paths.append(li[i].strip())
                 else:
                     break
-        paths = ','.join(paths)
+        paths = ','.join(sorted(paths))
         return filename, files, dirs, length, size, paths
 
     def do_info(self, entry, strict=True, user='root', raw=True, recuisive=False):
diff --git a/.github/scripts/hypo/command_test.py b/.github/scripts/hypo/command_test.py
index ace37a38c78b..659a87ed1a48 100644
--- a/.github/scripts/hypo/command_test.py
+++ b/.github/scripts/hypo/command_test.py
@@ -1,5 +1,5 @@
 import unittest
-from . import cmd 
+from command import JuicefsCommandMachine
 
 class TestCommand(unittest.TestCase):
     def test_dump(self):
@@ -21,5 +21,38 @@ def test_info(self):
         state.info(entry=folders_0, raw=True, recuisive=True, user='user1')
         state.teardown()
 
+    def test_info2(self):
+        state = JuicefsCommandMachine()
+        v1 = state.init_folders()
+        state.info(entry=v1, raw=True, recuisive=False, strict=True, user='user1')
+        v2 = state.create_file(content=b'*', file_name='fzlc', mode='a', parent=v1, umask=368, user='root')
+        state.fallocate(file=v2, length=98993, mode=0, offset=354, user='root')
+        v3 = state.hardlink(dest_file=v2, link_file_name='xtkb', parent=v1, umask=196, user='root')
+        state.truncate(file=v3, size=117319, user='root')
+        v4 = state.create_file(content=b'\xf7', file_name='nfyb', mode='w', parent=v1, umask=289, user='root')
+        state.truncate(file=v2, size=109323, user='root')
+        v5 = state.create_file(content=b'e\xc8p', file_name='aujn', mode='x', parent=v1, umask=292, user='root')
+        state.info(entry=v1, raw=True, recuisive=True, strict=True, user='root')
+        v6 = state.create_file(content=b'\x06', file_name='cgac', mode='w', parent=v1, umask=257, user='root')
+        state.fsck(entry=v1, recuisive=True, repair=False, user='root')
+        v7 = state.create_file(content=b'^\xda', file_name='litc', mode='w', parent=v1, umask=16, user='root')
+        state.fsck(entry=v1, recuisive=True, repair=True, user='user1')
+        state.fsck(entry=v1, recuisive=True, repair=True, user='user1')
+        state.fsck(entry=v1, recuisive=True, repair=False, user='user1')
+        state.fsck(entry=v6, recuisive=True, repair=True, user='user1')
+        v8 = state.create_file(content=b'', file_name='bbab', mode='w', parent=v1, umask=257, user='root')
+        v9 = state.create_file(content=b'\x01\x00\x01\x01\x01', file_name='bbbb', mode='x', parent=v1, umask=257, user='root')
+        state.fsck(entry=v1, recuisive=True, repair=True, user='user1')
+        state.truncate(file=v8, size=65793, user='root')
+        state.fsck(entry=v8, recuisive=True, repair=True, user='user1')
+        state.fsck(entry=v8, recuisive=True, repair=True, user='user1')
+        state.fsck(entry=v1, recuisive=False, repair=True, user='root')
+        state.truncate(file=v3, size=4, user='root')
+        state.fsck(entry=v8, recuisive=True, repair=True, user='user1')
+        v10 = state.hardlink(dest_file=v7, link_file_name='hgea', parent=v1, umask=57, user='root')
+        state.truncate(file=v3, size=32516, user='root')
+        state.info(entry=v10, raw=True, recuisive=True, strict=True, user='user1')
+        state.teardown()
+        
 if __name__ == '__main__':
     unittest.main()
\ No newline at end of file
diff --git a/.github/scripts/hypo/common.py b/.github/scripts/hypo/common.py
index ef94f7c1f02f..69349c336301 100644
--- a/.github/scripts/hypo/common.py
+++ b/.github/scripts/hypo/common.py
@@ -9,7 +9,7 @@
 def replace(src, old, new):
     if isinstance(src, str):
         return src.replace(old, new)
-    elif isinstance(src, list):
+    elif isinstance(src, list) or isinstance(src, tuple):
         return [replace(x, old, new) for x in src]
     elif isinstance(src, dict):
         return {k: replace(v, old, new) for k, v in src.items()}

From 52b4b1ce7212d9d95ac8e013650e9009e6567269 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Thu, 18 Apr 2024 14:33:57 +0800
Subject: [PATCH 06/13] wip

---
 .github/scripts/command/random.sh    |  42 ++++++++++
 .github/scripts/hypo/command_test.py |  35 ++-------
 .github/workflows/command2.yml       | 112 +++++++++++++++++++++++++++
 3 files changed, 159 insertions(+), 30 deletions(-)
 create mode 100755 .github/scripts/command/random.sh
 create mode 100644 .github/workflows/command2.yml

diff --git a/.github/scripts/command/random.sh b/.github/scripts/command/random.sh
new file mode 100755
index 000000000000..74ae65a292ac
--- /dev/null
+++ b/.github/scripts/command/random.sh
@@ -0,0 +1,42 @@
+#!/bin/bash -e
+source .github/scripts/common/common.sh
+
+[[ -z "$META1" ]] && META1=sqlite3
+source .github/scripts/start_meta_engine.sh
+start_meta_engine $META1
+META_URL1=$(get_meta_url $META1)
+
+[[ -z "$META2" ]] && META2=redis
+source .github/scripts/start_meta_engine.sh
+start_meta_engine $META2
+META_URL2=$(get_meta_url $META2)
+
+prepare_test()
+{
+    meta_url=$1
+    mp=$2
+    volume=$3
+    umount_jfs $mp $meta_url
+    python3 .github/scripts/flush_meta.py $meta_url
+    rm -rf /var/jfs/$volume || true
+    rm -rf /var/jfsCache/$volume || true
+    ./juicefs format $meta_url $volume --enable-acl --trash-days 0
+    ./juicefs mount -d $meta_url $mp
+}
+
+test_run_examples()
+{
+    prepare_test $META_URL1 /tmp/jfs1 myjfs1
+    prepare_test $META_URL2 /tmp/jfs2 myjfs2
+    python3 .github/scripts/hypo/command_test.py
+}
+
+test_run_all()
+{
+    prepare_test $META_URL1 /tmp/jfs1 myjfs1
+    prepare_test $META_URL2 /tmp/jfs2 myjfs2
+    python3 .github/scripts/hypo/command.py
+}
+
+source .github/scripts/common/run_test.sh && run_test $@
+
diff --git a/.github/scripts/hypo/command_test.py b/.github/scripts/hypo/command_test.py
index 659a87ed1a48..ea4b55f101e7 100644
--- a/.github/scripts/hypo/command_test.py
+++ b/.github/scripts/hypo/command_test.py
@@ -21,38 +21,13 @@ def test_info(self):
         state.info(entry=folders_0, raw=True, recuisive=True, user='user1')
         state.teardown()
 
-    def test_info2(self):
+    def test_clone(self):
         state = JuicefsCommandMachine()
         v1 = state.init_folders()
-        state.info(entry=v1, raw=True, recuisive=False, strict=True, user='user1')
-        v2 = state.create_file(content=b'*', file_name='fzlc', mode='a', parent=v1, umask=368, user='root')
-        state.fallocate(file=v2, length=98993, mode=0, offset=354, user='root')
-        v3 = state.hardlink(dest_file=v2, link_file_name='xtkb', parent=v1, umask=196, user='root')
-        state.truncate(file=v3, size=117319, user='root')
-        v4 = state.create_file(content=b'\xf7', file_name='nfyb', mode='w', parent=v1, umask=289, user='root')
-        state.truncate(file=v2, size=109323, user='root')
-        v5 = state.create_file(content=b'e\xc8p', file_name='aujn', mode='x', parent=v1, umask=292, user='root')
-        state.info(entry=v1, raw=True, recuisive=True, strict=True, user='root')
-        v6 = state.create_file(content=b'\x06', file_name='cgac', mode='w', parent=v1, umask=257, user='root')
-        state.fsck(entry=v1, recuisive=True, repair=False, user='root')
-        v7 = state.create_file(content=b'^\xda', file_name='litc', mode='w', parent=v1, umask=16, user='root')
-        state.fsck(entry=v1, recuisive=True, repair=True, user='user1')
-        state.fsck(entry=v1, recuisive=True, repair=True, user='user1')
-        state.fsck(entry=v1, recuisive=True, repair=False, user='user1')
-        state.fsck(entry=v6, recuisive=True, repair=True, user='user1')
-        v8 = state.create_file(content=b'', file_name='bbab', mode='w', parent=v1, umask=257, user='root')
-        v9 = state.create_file(content=b'\x01\x00\x01\x01\x01', file_name='bbbb', mode='x', parent=v1, umask=257, user='root')
-        state.fsck(entry=v1, recuisive=True, repair=True, user='user1')
-        state.truncate(file=v8, size=65793, user='root')
-        state.fsck(entry=v8, recuisive=True, repair=True, user='user1')
-        state.fsck(entry=v8, recuisive=True, repair=True, user='user1')
-        state.fsck(entry=v1, recuisive=False, repair=True, user='root')
-        state.truncate(file=v3, size=4, user='root')
-        state.fsck(entry=v8, recuisive=True, repair=True, user='user1')
-        v10 = state.hardlink(dest_file=v7, link_file_name='hgea', parent=v1, umask=57, user='root')
-        state.truncate(file=v3, size=32516, user='root')
-        state.info(entry=v10, raw=True, recuisive=True, strict=True, user='user1')
+        v2 = state.create_file(content=b'\x9bcR\xba', file_name='ygbl', mode='x', parent=v1, umask=466, user='root')
+        state.chmod(entry=v1, mode=715, user='root')
+        state.clone(entry=v2, new_entry_name='drqj', parent=v1, preserve=False, user='user1')
         state.teardown()
-        
+
 if __name__ == '__main__':
     unittest.main()
\ No newline at end of file
diff --git a/.github/workflows/command2.yml b/.github/workflows/command2.yml
new file mode 100644
index 000000000000..14a1abaf288f
--- /dev/null
+++ b/.github/workflows/command2.yml
@@ -0,0 +1,112 @@
+name: "command-random-test"
+
+on:
+  push:
+    branches:
+      - 'main'
+      - 'release-**'
+    paths:
+      - '.github/scripts/command/random.sh'
+      - '.github/scripts/hypo/command*.py'
+      - '**/command2.yml'
+  pull_request:
+    branches:
+      - 'main'
+      - 'release-**'
+    paths:
+      - '.github/scripts/command/random.sh'
+      - '.github/scripts/hypo/command*.py'
+      - '**/command2.yml'
+  schedule:
+    - cron:  '30 20 * * *'
+
+  workflow_dispatch:
+    inputs:
+      debug:
+        type: boolean
+        description: "Run the build with tmate debugging enabled"
+        required: false
+        default: false
+
+jobs:
+  build-matrix:
+    runs-on: ubuntu-20.04
+    steps:
+      - id: set-matrix
+        run: |
+          echo "github.event_name is ${{github.event_name}}"
+          echo "GITHUB_REF_NAME is ${GITHUB_REF_NAME}"
+          if [ "${{github.event_name}}" == "schedule"  ]; then
+            echo 'meta_matrix=["mysql", "redis", "tikv"]' >> $GITHUB_OUTPUT
+          elif [ "${{github.event_name}}" == "pull_request"  ]; then
+            echo 'meta_matrix=["mysql", "redis", "tikv"]' >> $GITHUB_OUTPUT
+          elif [ "${{github.event_name}}" == "workflow_dispatch"  ]; then
+            echo 'meta_matrix=["mysql", "redis", "tikv"]' >> $GITHUB_OUTPUT
+          else
+            echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT
+          fi
+    outputs:
+      meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}
+
+  test:
+    needs: [build-matrix]
+    strategy:
+      fail-fast: false
+      matrix:
+        meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }}
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+
+      - name: Build 
+        uses: ./.github/actions/build
+
+      - name: Run Example
+        timeout-minutes: 60
+        run: |
+          sudo META1=redis META2=${{matrix.meta}} .github/scripts/command/random.sh test_run_examples
+      
+      - name: Run All
+        timeout-minutes: 60
+        run: |
+          sudo META1=redis META2=${{matrix.meta}} .github/scripts/command/random.sh test_run_all
+  
+
+      - name: Log
+        if: always()
+        run: |
+          echo "juicefs log"
+          sudo tail -n 1000 /var/log/juicefs.log
+          grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
+          
+      - name: Setup upterm session
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
+        # if: failure()
+        timeout-minutes: 60
+        uses: lhotari/action-upterm@v1
+
+  success-all-test:
+    runs-on: ubuntu-latest
+    needs: [test]
+    if: always()
+    steps:
+      - uses: technote-space/workflow-conclusion-action@v3
+      - uses: actions/checkout@v3
+
+      - name: Check Failure
+        if: env.WORKFLOW_CONCLUSION == 'failure'
+        run: exit 1
+
+      - name: Send Slack Notification
+        if: failure() && github.event_name != 'workflow_dispatch'
+        uses: juicedata/slack-notify-action@main
+        with:
+          channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
+          slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
+
+      - name: Success
+        if: success()
+        run: echo "All Done"

From 822ad914c37c4e76bc405d930249985f1abcd891 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Thu, 18 Apr 2024 14:44:18 +0800
Subject: [PATCH 07/13] wip

---
 .github/workflows/command2.yml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/command2.yml b/.github/workflows/command2.yml
index 14a1abaf288f..9f7043263101 100644
--- a/.github/workflows/command2.yml
+++ b/.github/workflows/command2.yml
@@ -39,7 +39,7 @@ jobs:
           if [ "${{github.event_name}}" == "schedule"  ]; then
             echo 'meta_matrix=["mysql", "redis", "tikv"]' >> $GITHUB_OUTPUT
           elif [ "${{github.event_name}}" == "pull_request"  ]; then
-            echo 'meta_matrix=["mysql", "redis", "tikv"]' >> $GITHUB_OUTPUT
+            echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
           elif [ "${{github.event_name}}" == "workflow_dispatch"  ]; then
             echo 'meta_matrix=["mysql", "redis", "tikv"]' >> $GITHUB_OUTPUT
           else
@@ -67,14 +67,19 @@ jobs:
       - name: Run Example
         timeout-minutes: 60
         run: |
+          if [ "${{matrix.meta}}" == "redis" ]; then
+            echo "skip test for redis"
+          fi
           sudo META1=redis META2=${{matrix.meta}} .github/scripts/command/random.sh test_run_examples
       
       - name: Run All
         timeout-minutes: 60
         run: |
+          if [ "${{matrix.meta}}" == "redis" ]; then
+            echo "skip test for redis"
+          fi
           sudo META1=redis META2=${{matrix.meta}} .github/scripts/command/random.sh test_run_all
   
-
       - name: Log
         if: always()
         run: |

From 3fe3ca04de9e99832f9dae609320303ec7082356 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Thu, 18 Apr 2024 14:48:05 +0800
Subject: [PATCH 08/13] wip

---
 .github/scripts/hypo/command_op.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/scripts/hypo/command_op.py b/.github/scripts/hypo/command_op.py
index 25ffff34b54c..3e31b99a23b6 100644
--- a/.github/scripts/hypo/command_op.py
+++ b/.github/scripts/hypo/command_op.py
@@ -140,7 +140,7 @@ def do_rmr(self, entry, user='root'):
     
     def do_status(self):
         try:
-            result = self.run_cmd(f'./juicefs status {self.meta_url}', stderr=subprocess.DEVNULL)
+            result = self.run_cmd(f'./juicefs status {self.meta_url} --log-level error', stderr=subprocess.DEVNULL)
             result = json.loads(result)['Setting']
         except subprocess.CalledProcessError as e:
             return self.handleException(e, 'do_status', '')

From b85a5ffd098fb4698713ed28b4c624ab54a01caa Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Thu, 18 Apr 2024 14:49:38 +0800
Subject: [PATCH 09/13] wip

---
 .github/workflows/command2.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/command2.yml b/.github/workflows/command2.yml
index 9f7043263101..b04663ad2402 100644
--- a/.github/workflows/command2.yml
+++ b/.github/workflows/command2.yml
@@ -37,13 +37,13 @@ jobs:
           echo "github.event_name is ${{github.event_name}}"
           echo "GITHUB_REF_NAME is ${GITHUB_REF_NAME}"
           if [ "${{github.event_name}}" == "schedule"  ]; then
-            echo 'meta_matrix=["mysql", "redis", "tikv"]' >> $GITHUB_OUTPUT
+            echo 'meta_matrix=["mysql", "tikv"]' >> $GITHUB_OUTPUT
           elif [ "${{github.event_name}}" == "pull_request"  ]; then
-            echo 'meta_matrix=["sqlite3", "redis", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
+            echo 'meta_matrix=["sqlite3", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
           elif [ "${{github.event_name}}" == "workflow_dispatch"  ]; then
-            echo 'meta_matrix=["mysql", "redis", "tikv"]' >> $GITHUB_OUTPUT
+            echo 'meta_matrix=["mysql", "tikv"]' >> $GITHUB_OUTPUT
           else
-            echo 'meta_matrix=["redis"]' >> $GITHUB_OUTPUT
+            echo 'meta_matrix=["mysql", "tikv"]' >> $GITHUB_OUTPUT
           fi
     outputs:
       meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}

From 540b309fbbe612e257afca52c86bcbdee6f77c24 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Thu, 18 Apr 2024 14:50:15 +0800
Subject: [PATCH 10/13] wip

---
 .github/workflows/command2.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/command2.yml b/.github/workflows/command2.yml
index b04663ad2402..bd48f8c076af 100644
--- a/.github/workflows/command2.yml
+++ b/.github/workflows/command2.yml
@@ -67,17 +67,11 @@ jobs:
       - name: Run Example
         timeout-minutes: 60
         run: |
-          if [ "${{matrix.meta}}" == "redis" ]; then
-            echo "skip test for redis"
-          fi
           sudo META1=redis META2=${{matrix.meta}} .github/scripts/command/random.sh test_run_examples
       
       - name: Run All
         timeout-minutes: 60
         run: |
-          if [ "${{matrix.meta}}" == "redis" ]; then
-            echo "skip test for redis"
-          fi
           sudo META1=redis META2=${{matrix.meta}} .github/scripts/command/random.sh test_run_all
   
       - name: Log

From 07f267b13f9516616e70186e7dabf3caae11848f Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Thu, 18 Apr 2024 14:54:25 +0800
Subject: [PATCH 11/13] wip

---
 .github/workflows/command2.yml | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/command2.yml b/.github/workflows/command2.yml
index bd48f8c076af..e2259a959e69 100644
--- a/.github/workflows/command2.yml
+++ b/.github/workflows/command2.yml
@@ -61,8 +61,19 @@ jobs:
         with:
           fetch-depth: 1
 
-      - name: Build 
+      - name: Set Variable
+        id: vars
+        run: |
+          if [ "${{matrix.meta}}" == "fdb" ]; then
+            echo "target=juicefs.fdb" >> $GITHUB_OUTPUT
+          else
+            echo "target=juicefs" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build
         uses: ./.github/actions/build
+        with: 
+          target: ${{steps.vars.outputs.target}}
 
       - name: Run Example
         timeout-minutes: 60

From c6afd0236ecc3c3031be79af42a6cfc92e0f84c7 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Thu, 18 Apr 2024 15:09:43 +0800
Subject: [PATCH 12/13] wip

---
 .github/scripts/hypo/command.py      | 1 +
 .github/scripts/hypo/command_test.py | 2 +-
 .github/workflows/command2.yml       | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/hypo/command.py b/.github/scripts/hypo/command.py
index 3039986c4848..0dd5d70114a8 100644
--- a/.github/scripts/hypo/command.py
+++ b/.github/scripts/hypo/command.py
@@ -190,6 +190,7 @@ def dump_load_dump(self, folder, fast=False, skip_trash=False, threads=10, keep_
         result1 = self.cmd1.do_dump_load_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key)
         result2 = self.cmd2.do_dump_load_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key)
         print(result1)
+        print(result2)
         result1 = self.clean_dump(result1)
         result2 = self.clean_dump(result2)
         d=self.diff(result1, result2)
diff --git a/.github/scripts/hypo/command_test.py b/.github/scripts/hypo/command_test.py
index ea4b55f101e7..184a62115d37 100644
--- a/.github/scripts/hypo/command_test.py
+++ b/.github/scripts/hypo/command_test.py
@@ -21,7 +21,7 @@ def test_info(self):
         state.info(entry=folders_0, raw=True, recuisive=True, user='user1')
         state.teardown()
 
-    def test_clone(self):
+    def skip_test_clone(self):
         state = JuicefsCommandMachine()
         v1 = state.init_folders()
         v2 = state.create_file(content=b'\x9bcR\xba', file_name='ygbl', mode='x', parent=v1, umask=466, user='root')
diff --git a/.github/workflows/command2.yml b/.github/workflows/command2.yml
index e2259a959e69..cac27d04fd14 100644
--- a/.github/workflows/command2.yml
+++ b/.github/workflows/command2.yml
@@ -39,7 +39,7 @@ jobs:
           if [ "${{github.event_name}}" == "schedule"  ]; then
             echo 'meta_matrix=["mysql", "tikv"]' >> $GITHUB_OUTPUT
           elif [ "${{github.event_name}}" == "pull_request"  ]; then
-            echo 'meta_matrix=["sqlite3", "mysql", "tikv", "tidb", "postgres", "badger", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
+            echo 'meta_matrix=["sqlite3", "mysql", "tikv", "tidb", "postgres", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
           elif [ "${{github.event_name}}" == "workflow_dispatch"  ]; then
             echo 'meta_matrix=["mysql", "tikv"]' >> $GITHUB_OUTPUT
           else

From 5a2a61783fc98105e40482a5352399b882bd40c8 Mon Sep 17 00:00:00 2001
From: Cheng Zhou <zhoucheng@juicedata.io>
Date: Thu, 18 Apr 2024 16:11:48 +0800
Subject: [PATCH 13/13] wip

---
 .github/workflows/command2.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/command2.yml b/.github/workflows/command2.yml
index cac27d04fd14..2a56618cd52b 100644
--- a/.github/workflows/command2.yml
+++ b/.github/workflows/command2.yml
@@ -37,13 +37,13 @@ jobs:
           echo "github.event_name is ${{github.event_name}}"
           echo "GITHUB_REF_NAME is ${GITHUB_REF_NAME}"
           if [ "${{github.event_name}}" == "schedule"  ]; then
-            echo 'meta_matrix=["mysql", "tikv"]' >> $GITHUB_OUTPUT
-          elif [ "${{github.event_name}}" == "pull_request"  ]; then
             echo 'meta_matrix=["sqlite3", "mysql", "tikv", "tidb", "postgres", "mariadb", "fdb"]' >> $GITHUB_OUTPUT
+          elif [ "${{github.event_name}}" == "pull_request"  ]; then
+            echo 'meta_matrix=["sqlite3", "tikv"]' >> $GITHUB_OUTPUT
           elif [ "${{github.event_name}}" == "workflow_dispatch"  ]; then
-            echo 'meta_matrix=["mysql", "tikv"]' >> $GITHUB_OUTPUT
+            echo 'meta_matrix=["sqlite3", "tikv"]' >> $GITHUB_OUTPUT
           else
-            echo 'meta_matrix=["mysql", "tikv"]' >> $GITHUB_OUTPUT
+            echo 'meta_matrix=["sqlite3", "tikv"]' >> $GITHUB_OUTPUT
           fi
     outputs:
       meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }}