CI: refactor random test (#4726)

.github/scripts/command/random.sh

@@ -0,0 +1,42 @@
+#!/bin/bash -e
+source .github/scripts/common/common.sh
+
+[[ -z "$META1" ]] && META1=sqlite3
+source .github/scripts/start_meta_engine.sh
+start_meta_engine $META1
+META_URL1=$(get_meta_url $META1)
+
+[[ -z "$META2" ]] && META2=redis
+source .github/scripts/start_meta_engine.sh
+start_meta_engine $META2
+META_URL2=$(get_meta_url $META2)
+
+prepare_test()
+{
+    meta_url=$1
+    mp=$2
+    volume=$3
+    umount_jfs $mp $meta_url
+    python3 .github/scripts/flush_meta.py $meta_url
+    rm -rf /var/jfs/$volume || true
+    rm -rf /var/jfsCache/$volume || true
+    ./juicefs format $meta_url $volume --enable-acl --trash-days 0
+    ./juicefs mount -d $meta_url $mp
+}
+
+test_run_examples()
+{
+    prepare_test $META_URL1 /tmp/jfs1 myjfs1
+    prepare_test $META_URL2 /tmp/jfs2 myjfs2
+    python3 .github/scripts/hypo/command_test.py
+}
+
+test_run_all()
+{
+    prepare_test $META_URL1 /tmp/jfs1 myjfs1
+    prepare_test $META_URL2 /tmp/jfs2 myjfs2
+    python3 .github/scripts/hypo/command.py
+}
+
+source .github/scripts/common/run_test.sh && run_test $@
+

.github/scripts/hypo/command.py

@@ -34,6 +34,7 @@
 from fs_op import FsOperation
 from command_op import CommandOperation
 from fs import JuicefsMachine
+import common
 
 SEED=int(os.environ.get('SEED', random.randint(0, 1000000000)))
 
@@ -47,39 +48,41 @@ class JuicefsCommandMachine(JuicefsMachine):
     Entries = Files | Folders
     MP1 = '/tmp/jfs1'
     MP2 = '/tmp/jfs2'
+    ROOT_DIR1=os.path.join(MP1, 'fsrand')
+    ROOT_DIR2=os.path.join(MP2, 'fsrand')
     EXCLUDE_RULES = ['rebalance_dir', 'rebalance_file']
     # EXCLUDE_RULES = []
     INCLUDE_RULES = ['dump_load_dump', 'mkdir', 'create_file', 'set_xattr']
-    cmd1 = CommandOperation('cmd1', mp=MP1)
-    cmd2 = CommandOperation('cmd2', mp=MP2)
+    cmd1 = CommandOperation('cmd1', MP1, ROOT_DIR1)
+    cmd2 = CommandOperation('cmd2', MP2, ROOT_DIR2)
+    fsop1 = FsOperation('fs1', ROOT_DIR1)
+    fsop2 = FsOperation('fs2', ROOT_DIR2)
+    def __init__(self):
+        super().__init__()
+
+    def get_default_rootdir1(self):
+        return os.path.join(self.MP1, 'fsrand')
+
+    def get_default_rootdir2(self):
+        return os.path.join(self.MP2, 'fsrand')
 
     def equal(self, result1, result2):
         if type(result1) != type(result2):
             return False
         if isinstance(result1, Exception):
             if 'panic:' in str(result1) or 'panic:' in str(result2):
                 return False
-            r1 = str(result1).replace(self.MP1, '')
-            r2 = str(result2).replace(self.MP2, '')
-            return r1 == r2
-        elif isinstance(result1, str):
-            r1 = str(result1).replace(self.MP1, '')
-            r2 = str(result2).replace(self.MP2, '')
-            return r1 == r2
-        elif isinstance(result1, tuple):
-            r1 = [str(item).replace(self.MP1, '') for item in result1]
-            r2 = [str(item).replace(self.MP2, '') for item in result2]
-            return r1 == r2
-        else:
-            return  result1 == result2
+            result1 = str(result1)
+            result2 = str(result2)
+        result1 = common.replace(result1, self.MP1, '***')
+        result2 = common.replace(result2, self.MP2, '***')
+        print(f'result1 is {result1}\nresult2 is {result2}')
+        return result1 == result2
 
     def get_client_version(self, mount):
         output = run_cmd(f'{mount} version')
         return output.split()[2]
 
-    def __init__(self):
-        super().__init__()
-
     def should_run(self, rule):
         if len(self.EXCLUDE_RULES) > 0:
             return rule not in self.EXCLUDE_RULES
@@ -187,6 +190,7 @@ def dump_load_dump(self, folder, fast=False, skip_trash=False, threads=10, keep_
         result1 = self.cmd1.do_dump_load_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key)
         result2 = self.cmd2.do_dump_load_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key)
         print(result1)
+        print(result2)
         result1 = self.clean_dump(result1)
         result2 = self.clean_dump(result2)
         d=self.diff(result1, result2)

.github/scripts/hypo/command_op.py

@@ -22,10 +22,10 @@
 class CommandOperation:
     JFS_CONTROL_FILES=['.accesslog', '.config', '.stats']
     stats = Statistics()
-    def __init__(self, name, mp):
+    def __init__(self, name, mp, root_dir):
         self.logger = common.setup_logger(f'./{name}.log', name, os.environ.get('LOG_LEVEL', 'INFO'))
         self.mp = mp
-        self.root_dir = self.mp+'/fsrand'
+        self.root_dir = root_dir
         self.meta_url = self.get_meta_url(mp)
 
     def get_meta_url(self, mp):
@@ -102,7 +102,7 @@ def parse_info(self, info: str):
                     paths.append(li[i].strip())
                 else:
                     break
-        paths = ','.join(paths)
+        paths = ','.join(sorted(paths))
         return filename, files, dirs, length, size, paths
 
     def do_info(self, entry, strict=True, user='root', raw=True, recuisive=False):
@@ -140,7 +140,7 @@ def do_rmr(self, entry, user='root'):
 
     def do_status(self):
         try:
-            result = self.run_cmd(f'./juicefs status {self.meta_url}', stderr=subprocess.DEVNULL)
+            result = self.run_cmd(f'./juicefs status {self.meta_url} --log-level error', stderr=subprocess.DEVNULL)
             result = json.loads(result)['Setting']
         except subprocess.CalledProcessError as e:
             return self.handleException(e, 'do_status', '')

.github/scripts/hypo/command_test.py

@@ -1,5 +1,5 @@
 import unittest
-from . import cmd 
+from command import JuicefsCommandMachine
 
 class TestCommand(unittest.TestCase):
     def test_dump(self):
@@ -21,5 +21,13 @@ def test_info(self):
         state.info(entry=folders_0, raw=True, recuisive=True, user='user1')
         state.teardown()
 
+    def skip_test_clone(self):
+        state = JuicefsCommandMachine()
+        v1 = state.init_folders()
+        v2 = state.create_file(content=b'\x9bcR\xba', file_name='ygbl', mode='x', parent=v1, umask=466, user='root')
+        state.chmod(entry=v1, mode=715, user='root')
+        state.clone(entry=v2, new_entry_name='drqj', parent=v1, preserve=False, user='user1')
+        state.teardown()
+
 if __name__ == '__main__':
     unittest.main()

.github/scripts/hypo/common.py

@@ -6,8 +6,15 @@
 import pwd
 import subprocess
 import sys
-
-
+def replace(src, old, new):
+    if isinstance(src, str):
+        return src.replace(old, new)
+    elif isinstance(src, list) or isinstance(src, tuple):
+        return [replace(x, old, new) for x in src]
+    elif isinstance(src, dict):
+        return {k: replace(v, old, new) for k, v in src.items()}
+    else:
+        return src
 def run_cmd(command: str) -> str:
     print('run_cmd:'+command)
     if '|' in command or '>' in command:

.github/scripts/hypo/fs.py

@@ -30,42 +30,37 @@ class JuicefsMachine(RuleBasedStateMachine):
     EntryWithACL = Bundle('entry_with_acl')
     FilesWithXattr = Bundle('files_with_xattr')
     start = time.time()
-    ROOT_DIR1=os.environ.get('ROOT_DIR1', '/tmp/fsrand').rstrip('/')
-    ROOT_DIR2=os.environ.get('ROOT_DIR2', '/tmp/jfs/fsrand').rstrip('/')
-
-    fsop1 = FsOperation('fs1', ROOT_DIR1)
-    fsop2 = FsOperation('fs2', ROOT_DIR2)
-
-    ZONES = {ROOT_DIR1:common.get_zones(ROOT_DIR1), ROOT_DIR2:common.get_zones(ROOT_DIR2)}
     SUDO_USERS = ['root']
     USERS=['root', 'user1', 'user2','user3']
     GROUPS = USERS+['group1', 'group2', 'group3', 'group4']
     group_created = False
     INCLUDE_RULES = []
     EXCLUDE_RULES = ['rebalance_dir', 'rebalance_file', \
                         'clone_cp_file', 'clone_cp_dir']
-
+    fsop1 = FsOperation('fs1', os.environ.get('ROOT_DIR1', '/tmp/fsrand'))
+    fsop2 = FsOperation('fs2', os.environ.get('ROOT_DIR2', '/tmp/jfs/fsrand'))
+
     @initialize(target=Folders)
     def init_folders(self):
-        if not os.path.exists(self.ROOT_DIR1):
-            os.makedirs(self.ROOT_DIR1)
-        if not os.path.exists(self.ROOT_DIR2):
-            os.makedirs(self.ROOT_DIR2)
-        if os.environ.get('PROFILE', 'dev') != 'generate':
-            common.clean_dir(self.ROOT_DIR1)
-            common.clean_dir(self.ROOT_DIR2)
+        self.fsop1.init_rootdir()
+        self.fsop2.init_rootdir()
         return ''
 
     def create_users(self, users):
         for user in users:
             if user != 'root':
                 common.create_user(user)
 
+    def get_default_rootdir1(self):
+        return '/tmp/fsrand'
+
+    def get_default_rootdir2(self):
+        return '/tmp/jfs/fsrand'
+
     def __init__(self):
         super(JuicefsMachine, self).__init__()
         print(f'__init__')
-        if os.environ.get('EXCLUDE_RULES') is not None:
-            self.EXCLUDE_RULES = os.environ.get('EXCLUDE_RULES').split(',')
+
         if not self.group_created:
             for group in self.GROUPS:
                 common.create_group(group)
@@ -83,17 +78,13 @@ def equal(self, result1, result2):
         if type(result1) != type(result2):
             return False
         if isinstance(result1, Exception):
-            r1 = str(result1).replace(self.ROOT_DIR1, '')
-            r2 = str(result2).replace(self.ROOT_DIR2, '')
-            return r1 == r2
-        elif isinstance(result1, tuple):
-            return result1 == result2
-        elif isinstance(result1, str):
-            r1 = str(result1).replace(self.ROOT_DIR1, '')
-            r2 = str(result2).replace(self.ROOT_DIR2, '')
-            return  r1 == r2
-        else:
-            return result1 == result2
+            if 'panic:' in str(result1) or 'panic:' in str(result2):
+                return False
+            result1 = str(result1)
+            result2 = str(result2)
+        result1 = common.replace(result1, self.fsop1.root_dir, '***')
+        result2 = common.replace(result2, self.fsop2.root_dir, '***')
+        return result1 == result2
 
     def seteuid(self, user):
         os.seteuid(pwd.getpwnam(user).pw_uid)
@@ -472,49 +463,44 @@ def chown(self, entry, owner, user='root'):
 
     @rule( dir =Folders, vdirs = st.integers(min_value=2, max_value=31) )
     @precondition(lambda self: self.should_run('split_dir') \
-                  and (common.is_jfs(self.ROOT_DIR1) or common.is_jfs(self.ROOT_DIR2))
+                  and (self.fsop1.is_jfs() or self.fsop2.is_jfs())
     )
     def split_dir(self, dir, vdirs):
         self.fsop1.do_split_dir(dir, vdirs)
         self.fsop2.do_split_dir(dir, vdirs)
-
 
     @rule(dir = Folders)
     @precondition(lambda self: self.should_run('merge_dir') \
-                   and (common.is_jfs(self.ROOT_DIR1) or common.is_jfs(self.ROOT_DIR2))
+                 and (self.fsop1.is_jfs() or self.fsop2.is_jfs())
     )
     def merge_dir(self, dir):
         self.fsop1.do_merge_dir(dir)
         self.fsop2.do_merge_dir(dir)
 
-    @rule(dir = Folders,
-          zone1=st.sampled_from(ZONES[ROOT_DIR1]),
-          zone2=st.sampled_from(ZONES[ROOT_DIR2]),
-          is_vdir=st.booleans())
+    # @rule(dir = Folders,
+    #       zone1=st.sampled_from(self.fsop1.get_zones()),
+    #       zone2=st.sampled_from(self.fsop2.get_zones()),
+    #       is_vdir=st.booleans())
     @precondition(lambda self: self.should_run('rebalance_dir') \
-                   and (common.is_jfs(self.ROOT_DIR1) or common.is_jfs(self.ROOT_DIR2))
+                   and (self.fsop1.is_jfs() or self.fsop2.is_jfs())
     )
     def rebalance_dir(self, dir, zone1, zone2, is_vdir):
         self.fsop1.do_rebalance(dir, zone1, is_vdir)
         self.fsop2.do_rebalance(dir, zone2, is_vdir)
 
-    @rule(file = Files, 
-          zone1=st.sampled_from(ZONES[ROOT_DIR1]),
-          zone2=st.sampled_from(ZONES[ROOT_DIR2]),
-          )
+    # @rule(file = Files, 
+    #       zone1=st.sampled_from(self.fsop1.get_zones()),
+    #       zone2=st.sampled_from(self.fsop2.get_zones()),
+    #       )
     @precondition(lambda self: self.should_run('rebalance_file') \
-                   and (common.is_jfs(self.ROOT_DIR1) or common.is_jfs(self.ROOT_DIR2))
+                   and (self.fsop1.is_jfs() or self.fsop2.is_jfs())
     )
     def rebalance_file(self, file, zone1, zone2):
         self.fsop1.do_rebalance(file, zone1, False)
         self.fsop2.do_rebalance(file, zone2, False)
 
     def teardown(self):
         pass
-        # if COMPARE and os.path.exists(ROOT_DIR1):
-        #     common.compare_content(ROOT_DIR1, ROOT_DIR2)
-        #     common.compare_stat(ROOT_DIR1, ROOT_DIR2)
-        #     common.compare_acl(ROOT_DIR1, ROOT_DIR2)
 
 if __name__ == '__main__':
     MAX_EXAMPLE=int(os.environ.get('MAX_EXAMPLE', '100'))

.github/scripts/hypo/fs_op.py

@@ -23,9 +23,9 @@
 class FsOperation:
     JFS_CONTROL_FILES=['.accesslog', '.config', '.stats']
     stats = Statistics()
-    def __init__(self, name, root_dir):
+    def __init__(self, name, root_dir:str):
         self.logger =common.setup_logger(f'./{name}.log', name, os.environ.get('LOG_LEVEL', 'INFO'))
-        self.root_dir = root_dir
+        self.root_dir = root_dir.rstrip('/')
 
     def run_cmd(self, command:str) -> str:
         self.logger.info(f'run_cmd: {command}')
@@ -41,6 +41,18 @@ def run_cmd(self, command:str) -> str:
             raise e
         return output.stdout.decode()
 
+    def get_zones(self):
+        return common.get_zones(self.root_dir)
+
+    def is_jfs(self):
+        return common.is_jfs(self.root_dir)
+
+    def init_rootdir(self):
+        if not os.path.exists(self.root_dir):
+            os.makedirs(self.root_dir)
+        if os.environ.get('PROFILE', 'dev') != 'generate':
+            common.clean_dir(self.root_dir)
+
     def seteuid(self, user):
         os.seteuid(pwd.getpwnam(user).pw_uid)
         os.setegid(pwd.getpwnam(user).pw_gid)