CI: random test on juicefs command. (#4737)

.github/scripts/command/random.sh

@@ -1,5 +1,7 @@
 #!/bin/bash -e
 source .github/scripts/common/common.sh
+[[ -z "$MAX_EXAMPLE" ]] && MAX_EXAMPLE=100
+[[ -z "$STEP_COUNT" ]] && STEP_COUNT=50
 
 [[ -z "$META1" ]] && META1=sqlite3
 source .github/scripts/start_meta_engine.sh
@@ -16,26 +18,28 @@ prepare_test()
     meta_url=$1
     mp=$2
     volume=$3
+    shift 3
+    options=$@
     umount_jfs $mp $meta_url
     python3 .github/scripts/flush_meta.py $meta_url
     rm -rf /var/jfs/$volume || true
     rm -rf /var/jfsCache/$volume || true
-    ./juicefs format $meta_url $volume --enable-acl --trash-days 0
+    ./juicefs format $meta_url $volume $options
     ./juicefs mount -d $meta_url $mp
 }
 
 test_run_examples()
 {
-    prepare_test $META_URL1 /tmp/jfs1 myjfs1
-    prepare_test $META_URL2 /tmp/jfs2 myjfs2
+    prepare_test $META_URL1 /tmp/jfs1 myjfs1 --enable-acl --trash-days 0
+    prepare_test $META_URL2 /tmp/jfs2 myjfs2 --enable-acl --trash-days 0
     python3 .github/scripts/hypo/command_test.py
 }
 
 test_run_all()
 {
     prepare_test $META_URL1 /tmp/jfs1 myjfs1
     prepare_test $META_URL2 /tmp/jfs2 myjfs2
-    python3 .github/scripts/hypo/command.py
+    CHECK_NLINK=false MAX_EXAMPLE=$MAX_EXAMPLE STEP_COUNT=$STEP_COUNT python3 .github/scripts/hypo/command.py
 }
 
 source .github/scripts/common/run_test.sh && run_test $@

.github/scripts/hypo/command.py

@@ -50,7 +50,7 @@ class JuicefsCommandMachine(JuicefsMachine):
     MP2 = '/tmp/jfs2'
     ROOT_DIR1=os.path.join(MP1, 'fsrand')
     ROOT_DIR2=os.path.join(MP2, 'fsrand')
-    EXCLUDE_RULES = ['rebalance_dir', 'rebalance_file']
+    EXCLUDE_RULES = ['rebalance_dir', 'rebalance_file', 'config']
     # EXCLUDE_RULES = []
     INCLUDE_RULES = ['dump_load_dump', 'mkdir', 'create_file', 'set_xattr']
     cmd1 = CommandOperation('cmd1', MP1, ROOT_DIR1)
@@ -76,7 +76,7 @@ def equal(self, result1, result2):
             result2 = str(result2)
         result1 = common.replace(result1, self.MP1, '***')
         result2 = common.replace(result2, self.MP2, '***')
-        print(f'result1 is {result1}\nresult2 is {result2}')
+        # print(f'result1 is {result1}\nresult2 is {result2}')
         return result1 == result2
 
     def get_client_version(self, mount):
@@ -88,7 +88,7 @@ def should_run(self, rule):
             return rule not in self.EXCLUDE_RULES
         else:
             return rule in self.INCLUDE_RULES
-        
+
     @rule(
           entry = Entries.filter(lambda x: x != multiple()),
           raw = st.just(True),
@@ -168,59 +168,85 @@ def clone(self, entry, parent, new_entry_name, preserve=False, user='root'):
         fast = st.booleans(),
         skip_trash = st.booleans(),
         threads = st.integers(min_value=1, max_value=10),
-        keep_secret_key = st.booleans()
+        keep_secret_key = st.booleans(),
+        user = st.just('root')
     )
     @precondition(lambda self: self.should_run('dump'))
-    def dump(self, folder, fast, skip_trash, threads, keep_secret_key):
-        result1 = self.cmd1.do_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key)
-        result2 = self.cmd2.do_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key)
-        result1 = self.clean_dump(result1)
-        result2 = self.clean_dump(result2)
-        d=self.diff(result1, result2)
+    def dump(self, folder, fast, skip_trash, threads, keep_secret_key, user='root'):
+        result1 = self.cmd1.do_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key, user=user)
+        result2 = self.cmd2.do_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key, user=user)
+        d=''
+        if isinstance(result1, str) and isinstance(result2, str):
+            d=self.diff(result1, result2)
         assert self.equal(result1, result2), f'\033[31mdump:\nresult1 is {result1}\nresult2 is {result2}\ndiff is {d}\033[0m'
 
     @rule(folder = st.just(''),
         fast = st.booleans(),
         skip_trash = st.booleans(),
         threads = st.integers(min_value=1, max_value=10),
-        keep_secret_key = st.booleans()
+        keep_secret_key = st.booleans(),
+        user = st.just('root')
     )
     @precondition(lambda self: self.should_run('dump_load_dump'))
-    def dump_load_dump(self, folder, fast=False, skip_trash=False, threads=10, keep_secret_key=False):
-        result1 = self.cmd1.do_dump_load_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key)
-        result2 = self.cmd2.do_dump_load_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key)
-        print(result1)
-        print(result2)
-        result1 = self.clean_dump(result1)
-        result2 = self.clean_dump(result2)
-        d=self.diff(result1, result2)
+    def dump_load_dump(self, folder, fast=False, skip_trash=False, threads=10, keep_secret_key=False, user='root'):
+        result1 = self.cmd1.do_dump_load_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key, user=user)
+        result2 = self.cmd2.do_dump_load_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key, user=user)
+        d=''
+        if isinstance(result1, str) and isinstance(result2, str):
+            d=self.diff(result1, result2)
         assert self.equal(result1, result2), f'\033[31mdump:\nresult1 is {result1}\nresult2 is {result2}\ndiff is {d}\033[0m'
 
     def diff(self, str1:str, str2:str):
         differ = Differ()
         diff = differ.compare(str1.splitlines(), str2.splitlines())
         return '\n'.join([line for line in diff])
-
-    def clean_dump(self, dump):
-        lines = dump.split('\n')
-        new_lines = []
-        exclude_keys = ['Name', 'UUID', 'usedSpace', 'usedInodes', 'nextInodes', 'nextChunk', 'nextTrash', 'nextSession']
-        reset_keys = ['id', 'inode', 'atimensec', 'mtimensec', 'ctimensec', 'atime', 'ctime', 'mtime']
-        for line in lines:
-            should_delete = False
-            for key in exclude_keys:
-                if f'"{key}"' in line:
-                    should_delete = True
-                    break
-            if should_delete:
-                continue
-            for key in reset_keys:
-                if f'"{key}"' in line:
-                    pattern = rf'"{key}":(\d+)'
-                    line = re.sub(pattern, f'"{key}":0', line)
-            new_lines.append(line)
-        return '\n'.join(new_lines)
 
+    @rule(
+        user = st_sudo_user
+    )
+    @precondition(lambda self: self.should_run('trash_list'))
+    def trash_list(self, user='root'):
+        result1 = self.cmd1.do_trash_list(user=user)
+        result2 = self.cmd2.do_trash_list(user=user)
+        assert self.equal(result1, result2), f'\033[31mtrash_list:\nresult1 is {result1}\nresult2 is {result2}\033[0m'
+
+    @rule(
+        put_back = st.booleans(),
+        threads = st.integers(min_value=1, max_value=10),
+        user=st_sudo_user
+    )
+    @precondition(lambda self: self.should_run('restore'))
+    def restore(self, put_back, threads, user='root'):
+        result1 = self.cmd1.do_restore(put_back=put_back, threads=threads, user=user)
+        result2 = self.cmd2.do_restore(put_back=put_back, threads=threads, user=user)
+        assert self.equal(result1, result2), f'\033[31mrestore:\nresult1 is {result1}\nresult2 is {result2}\033[0m'
+
+    @rule(
+        entry = Entries.filter(lambda x: x != multiple()),
+        threads = st.integers(min_value=1, max_value=10),
+        user = st_sudo_user
+    )
+    @precondition(lambda self: self.should_run('compact'))
+    def compact(self, entry, threads, user='root'):
+        result1 = self.cmd1.do_compact(entry=entry, threads=threads, user=user)
+        result2 = self.cmd2.do_compact(entry=entry, threads=threads, user=user)
+        assert self.equal(result1, result2), f'\033[31mcompact:\nresult1 is {result1}\nresult2 is {result2}\033[0m'
+
+    @rule(
+        capacity = st.integers(min_value=1, max_value=2),
+        inodes = st.one_of(st.just(0), st.integers(min_value=50, max_value=100)),
+        trash_days = st.integers(min_value=0, max_value=1),
+        enable_acl = st.booleans(),
+        encrypt_secret = st.booleans(),
+        force = st.booleans(),
+        yes = st.just(True),
+        user = st_sudo_user
+    )
+    @precondition(lambda self: self.should_run('config'))
+    def config(self, capacity, inodes, trash_days, enable_acl, encrypt_secret, force, yes, user='root'):
+        result1 = self.cmd1.do_config(capacity=capacity, inodes=inodes, trash_days=trash_days, enable_acl=enable_acl, encrypt_secret=encrypt_secret, force=force, yes=yes, user=user)
+        result2 = self.cmd2.do_config(capacity=capacity, inodes=inodes, trash_days=trash_days, enable_acl=enable_acl, encrypt_secret=encrypt_secret, force=force, yes=yes, user=user)
+        assert self.equal(result1, result2), f'\033[31mconfig:\nresult1 is {result1}\nresult2 is {result2}\033[0m'
 
     def teardown(self):
         pass

.github/scripts/hypo/command_op.py

@@ -28,15 +28,27 @@ def __init__(self, name, mp, root_dir):
         self.root_dir = root_dir
         self.meta_url = self.get_meta_url(mp)
 
+    def guess_password(self, meta_url):
+        if '****' not in meta_url:
+            return meta_url
+        if meta_url.startswith('postgres://'):
+            return meta_url.replace('****', 'postgres')
+        else:
+            return meta_url.replace('****', 'root')
+
     def get_meta_url(self, mp):
         with open(os.path.join(mp, '.config')) as f:
             config = json.loads(f.read())
             pid = config['Pid']
             process = psutil.Process(pid)
             cmdline = process.cmdline()
             for item in cmdline:
-                if '://' in item:
-                    return item
+                if ' ' in item:
+                    for subitem in item.split(' '):
+                        if '://' in subitem:
+                            return self.guess_password(subitem)
+                elif '://' in item:
+                    return self.guess_password(item)
             raise Exception(f'get_meta_url: {cmdline} does not contain meta url')
 
     def run_cmd(self, command:str, stderr=subprocess.STDOUT) -> str:
@@ -150,45 +162,67 @@ def do_status(self):
             result['EncryptAlgo'], result['TrashDays'], result['MetaVersion'], \
             result['MinClientVersion'], result['DirStats'], result['EnableACL']
 
-    def do_dump(self, folder, fast=False, skip_trash=False, threads=1, keep_secret_key=False):
+    def do_dump(self, folder, fast=False, skip_trash=False, threads=1, keep_secret_key=False, user='root'):
         abspath = os.path.join(self.root_dir, folder)
         subdir = os.path.relpath(abspath, self.mp)
         try:
-            cmd=self.get_dump_cmd(self.meta_url, subdir, fast, skip_trash, keep_secret_key, threads)
+            cmd=self.get_dump_cmd(self.meta_url, subdir, fast, skip_trash, keep_secret_key, threads, user)
             result = self.run_cmd(cmd, stderr=subprocess.DEVNULL)
         except subprocess.CalledProcessError as e:
             return self.handleException(e,  'do_dump', abspath)
         self.stats.success('do_dump')
         self.logger.info(f'do_dump {abspath} succeed')
-        return result
+        return self.clean_dump(result)
 
-    def get_dump_cmd(self, meta_url, subdir, fast, skip_trash, keep_secret_key, threads):
-        cmd = f'./juicefs dump {meta_url} '
+    def get_dump_cmd(self, meta_url, subdir, fast, skip_trash, keep_secret_key, threads, user='root'):
+        cmd = f'sudo -u {user} ./juicefs dump {meta_url} '
         cmd += f' --subdir /{subdir}' if subdir != '' else ''
         cmd += f' --fast' if fast else ''
         cmd += f' --skip-trash' if skip_trash else ''
         cmd += f' --keep-secret-key' if keep_secret_key else ''
         cmd += f' --threads {threads}'
+        cmd += f' --log-level error'
         return cmd
 
-    def do_dump_load_dump(self, folder, fast=False, skip_trash=False, threads=1, keep_secret_key=False):
+    def do_dump_load_dump(self, folder, fast=False, skip_trash=False, threads=1, keep_secret_key=False, user='root'):
         abspath = os.path.join(self.root_dir, folder)
         subdir = os.path.relpath(abspath, self.mp)
         try:
-            cmd = self.get_dump_cmd(self.meta_url, subdir, fast, skip_trash, keep_secret_key, threads)
+            print(f'meta_url is {self.meta_url}')
+            cmd = self.get_dump_cmd(self.meta_url, subdir, fast, skip_trash, keep_secret_key, threads, user)
             result = self.run_cmd(cmd, stderr=subprocess.DEVNULL)
             with open('dump.json', 'w') as f:
                 f.write(result)
             if os.path.exists('load.db'):
                 os.remove('load.db')
-            self.run_cmd(f'./juicefs load sqlite3://load.db dump.json')
-            cmd = self.get_dump_cmd('sqlite3://load.db', '', fast, skip_trash, keep_secret_key, threads)
+            self.run_cmd(f'sudo -u {user} ./juicefs load sqlite3://load.db dump.json')
+            cmd = self.get_dump_cmd('sqlite3://load.db', '', fast, skip_trash, keep_secret_key, threads, user)
             result = self.run_cmd(cmd, stderr=subprocess.DEVNULL)
         except subprocess.CalledProcessError as e:
             return self.handleException(e, 'do_dump', abspath)
         self.stats.success('do_dump')
         self.logger.info(f'do_dump {abspath} succeed')
-        return result
+        return self.clean_dump(result)
+
+    def clean_dump(self, dump):
+        lines = dump.split('\n')
+        new_lines = []
+        exclude_keys = ['Name', 'UUID', 'usedSpace', 'usedInodes', 'nextInodes', 'nextChunk', 'nextTrash', 'nextSession']
+        reset_keys = ['id', 'inode', 'atimensec', 'mtimensec', 'ctimensec', 'atime', 'ctime', 'mtime']
+        for line in lines:
+            should_delete = False
+            for key in exclude_keys:
+                if f'"{key}"' in line:
+                    should_delete = True
+                    break
+            if should_delete:
+                continue
+            for key in reset_keys:
+                if f'"{key}"' in line:
+                    pattern = rf'"{key}":(\d+)'
+                    line = re.sub(pattern, f'"{key}":0', line)
+            new_lines.append(line)
+        return '\n'.join(new_lines)
 
     def do_warmup(self, entry, user='root'):
         abspath = os.path.join(self.root_dir, entry)
@@ -258,6 +292,21 @@ def do_trash_list(self, user='root'):
         self.logger.info(f'do_trash_list succeed')
         return tuple(li)
 
+    def do_restore(self, put_back, threads, user='root'):
+        abspath = os.path.join(self.mp, '.trash')
+        try:
+            li = os.listdir(abspath)
+            for trash_dir in li:
+                cmd = f'sudo -u {user} ./juicefs restore {trash_dir} --threads {threads}'
+                if put_back:
+                    cmd += ' --put-back'
+                self.run_cmd(cmd)
+        except subprocess.CalledProcessError as e:
+            return self.handleException(e, 'do_restore', abspath, user=user)
+        self.stats.success('do_restore')
+        self.logger.info(f'do_restore succeed')
+        return True
+
     def do_trash_restore(self, index, user='root'):
         trash_list = self.do_trash_list()
         if len(trash_list) == 0:
@@ -274,4 +323,27 @@ def do_trash_restore(self, index, user='root'):
         self.stats.success('do_trash_restore')
         self.logger.info(f'do_trash_restore succeed')
         return restored_path
-
+
+    def do_compact(self, entry, threads, user):
+        path = os.path.join(self.root_dir, entry)
+        try:
+            self.run_cmd(f'sudo -u {user} ./juicefs compact {path} --threads {threads}')
+        except subprocess.CalledProcessError as e:
+            return self.handleException(e, 'do_compact', path, user=user)
+        self.stats.success('do_compact')
+        self.logger.info(f'do_compact succeed')
+        return True
+
+    def do_config(self, capacity, inodes, trash_days, enable_acl, encrypt_secret, force, yes, user):
+        try:
+            cmd = f'sudo -u {user} ./juicefs config {self.meta_url} --capacity {capacity} --inodes {inodes} --trash-days {trash_days} --enable-acl {enable_acl} --encrypt-secret {encrypt_secret}'
+            if force:
+                cmd += ' --force'
+            if yes:
+                cmd += ' --yes'
+            self.run_cmd(cmd)
+        except subprocess.CalledProcessError as e:
+            return self.handleException(e, 'do_config', '')
+        self.stats.success('do_config')
+        self.logger.info(f'do_config succeed')
+        return True

.github/scripts/hypo/command_test.py

@@ -14,7 +14,7 @@ def test_dump(self):
         state.dump_load_dump(folders_0)
         state.teardown()
 
-    def test_info(self):
+    def skip_test_info(self):
         state = JuicefsCommandMachine()
         folders_0 = state.init_folders()
         files_2 = state.create_file(content=b'0', file_name='mvvd', mode='a', parent=folders_0, umask=293, user='root')
@@ -29,5 +29,11 @@ def skip_test_clone(self):
         state.clone(entry=v2, new_entry_name='drqj', parent=v1, preserve=False, user='user1')
         state.teardown()
 
+    def test_config(self):
+        state = JuicefsCommandMachine()
+        folders_0 = state.init_folders()
+        state.config(capacity=1, enable_acl=True, encrypt_secret=True, force=False, inodes=81, trash_days=0, user='root', yes=True)
+        state.teardown()
+
 if __name__ == '__main__':
     unittest.main()