From 39312bf681f45cb6f0258a6cc658ffaab5e3a0c3 Mon Sep 17 00:00:00 2001 From: Zhou Cheng Date: Thu, 18 Apr 2024 16:13:07 +0800 Subject: [PATCH] CI: refactor random test (#4726) --- .github/scripts/command/random.sh | 42 +++++++++ .github/scripts/hypo/command.py | 40 +++++---- .github/scripts/hypo/command_op.py | 8 +- .github/scripts/hypo/command_test.py | 10 ++- .github/scripts/hypo/common.py | 11 ++- .github/scripts/hypo/fs.py | 76 +++++++---------- .github/scripts/hypo/fs_op.py | 16 +++- .github/scripts/hypo/s3.py | 17 ++-- .github/workflows/command2.yml | 122 +++++++++++++++++++++++++++ .github/workflows/fsrand.yml | 2 +- 10 files changed, 259 insertions(+), 85 deletions(-) create mode 100755 .github/scripts/command/random.sh create mode 100644 .github/workflows/command2.yml diff --git a/.github/scripts/command/random.sh b/.github/scripts/command/random.sh new file mode 100755 index 000000000000..74ae65a292ac --- /dev/null +++ b/.github/scripts/command/random.sh @@ -0,0 +1,42 @@ +#!/bin/bash -e +source .github/scripts/common/common.sh + +[[ -z "$META1" ]] && META1=sqlite3 +source .github/scripts/start_meta_engine.sh +start_meta_engine $META1 +META_URL1=$(get_meta_url $META1) + +[[ -z "$META2" ]] && META2=redis +source .github/scripts/start_meta_engine.sh +start_meta_engine $META2 +META_URL2=$(get_meta_url $META2) + +prepare_test() +{ + meta_url=$1 + mp=$2 + volume=$3 + umount_jfs $mp $meta_url + python3 .github/scripts/flush_meta.py $meta_url + rm -rf /var/jfs/$volume || true + rm -rf /var/jfsCache/$volume || true + ./juicefs format $meta_url $volume --enable-acl --trash-days 0 + ./juicefs mount -d $meta_url $mp +} + +test_run_examples() +{ + prepare_test $META_URL1 /tmp/jfs1 myjfs1 + prepare_test $META_URL2 /tmp/jfs2 myjfs2 + python3 .github/scripts/hypo/command_test.py +} + +test_run_all() +{ + prepare_test $META_URL1 /tmp/jfs1 myjfs1 + prepare_test $META_URL2 /tmp/jfs2 myjfs2 + python3 .github/scripts/hypo/command.py +} + +source .github/scripts/common/run_test.sh && run_test $@ + diff --git a/.github/scripts/hypo/command.py b/.github/scripts/hypo/command.py index 7421bc128152..0dd5d70114a8 100644 --- a/.github/scripts/hypo/command.py +++ b/.github/scripts/hypo/command.py @@ -34,6 +34,7 @@ from fs_op import FsOperation from command_op import CommandOperation from fs import JuicefsMachine +import common SEED=int(os.environ.get('SEED', random.randint(0, 1000000000))) @@ -47,11 +48,23 @@ class JuicefsCommandMachine(JuicefsMachine): Entries = Files | Folders MP1 = '/tmp/jfs1' MP2 = '/tmp/jfs2' + ROOT_DIR1=os.path.join(MP1, 'fsrand') + ROOT_DIR2=os.path.join(MP2, 'fsrand') EXCLUDE_RULES = ['rebalance_dir', 'rebalance_file'] # EXCLUDE_RULES = [] INCLUDE_RULES = ['dump_load_dump', 'mkdir', 'create_file', 'set_xattr'] - cmd1 = CommandOperation('cmd1', mp=MP1) - cmd2 = CommandOperation('cmd2', mp=MP2) + cmd1 = CommandOperation('cmd1', MP1, ROOT_DIR1) + cmd2 = CommandOperation('cmd2', MP2, ROOT_DIR2) + fsop1 = FsOperation('fs1', ROOT_DIR1) + fsop2 = FsOperation('fs2', ROOT_DIR2) + def __init__(self): + super().__init__() + + def get_default_rootdir1(self): + return os.path.join(self.MP1, 'fsrand') + + def get_default_rootdir2(self): + return os.path.join(self.MP2, 'fsrand') def equal(self, result1, result2): if type(result1) != type(result2): @@ -59,27 +72,17 @@ def equal(self, result1, result2): if isinstance(result1, Exception): if 'panic:' in str(result1) or 'panic:' in str(result2): return False - r1 = str(result1).replace(self.MP1, '') - r2 = str(result2).replace(self.MP2, '') - return r1 == r2 - elif isinstance(result1, str): - r1 = str(result1).replace(self.MP1, '') - r2 = str(result2).replace(self.MP2, '') - return r1 == r2 - elif isinstance(result1, tuple): - r1 = [str(item).replace(self.MP1, '') for item in result1] - r2 = [str(item).replace(self.MP2, '') for item in result2] - return r1 == r2 - else: - return result1 == result2 + result1 = str(result1) + result2 = str(result2) + result1 = common.replace(result1, self.MP1, '***') + result2 = common.replace(result2, self.MP2, '***') + print(f'result1 is {result1}\nresult2 is {result2}') + return result1 == result2 def get_client_version(self, mount): output = run_cmd(f'{mount} version') return output.split()[2] - def __init__(self): - super().__init__() - def should_run(self, rule): if len(self.EXCLUDE_RULES) > 0: return rule not in self.EXCLUDE_RULES @@ -187,6 +190,7 @@ def dump_load_dump(self, folder, fast=False, skip_trash=False, threads=10, keep_ result1 = self.cmd1.do_dump_load_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key) result2 = self.cmd2.do_dump_load_dump(folder=folder, fast=fast, skip_trash=skip_trash, threads=threads, keep_secret_key=keep_secret_key) print(result1) + print(result2) result1 = self.clean_dump(result1) result2 = self.clean_dump(result2) d=self.diff(result1, result2) diff --git a/.github/scripts/hypo/command_op.py b/.github/scripts/hypo/command_op.py index c4456732dc3b..3e31b99a23b6 100644 --- a/.github/scripts/hypo/command_op.py +++ b/.github/scripts/hypo/command_op.py @@ -22,10 +22,10 @@ class CommandOperation: JFS_CONTROL_FILES=['.accesslog', '.config', '.stats'] stats = Statistics() - def __init__(self, name, mp): + def __init__(self, name, mp, root_dir): self.logger = common.setup_logger(f'./{name}.log', name, os.environ.get('LOG_LEVEL', 'INFO')) self.mp = mp - self.root_dir = self.mp+'/fsrand' + self.root_dir = root_dir self.meta_url = self.get_meta_url(mp) def get_meta_url(self, mp): @@ -102,7 +102,7 @@ def parse_info(self, info: str): paths.append(li[i].strip()) else: break - paths = ','.join(paths) + paths = ','.join(sorted(paths)) return filename, files, dirs, length, size, paths def do_info(self, entry, strict=True, user='root', raw=True, recuisive=False): @@ -140,7 +140,7 @@ def do_rmr(self, entry, user='root'): def do_status(self): try: - result = self.run_cmd(f'./juicefs status {self.meta_url}', stderr=subprocess.DEVNULL) + result = self.run_cmd(f'./juicefs status {self.meta_url} --log-level error', stderr=subprocess.DEVNULL) result = json.loads(result)['Setting'] except subprocess.CalledProcessError as e: return self.handleException(e, 'do_status', '') diff --git a/.github/scripts/hypo/command_test.py b/.github/scripts/hypo/command_test.py index ace37a38c78b..184a62115d37 100644 --- a/.github/scripts/hypo/command_test.py +++ b/.github/scripts/hypo/command_test.py @@ -1,5 +1,5 @@ import unittest -from . import cmd +from command import JuicefsCommandMachine class TestCommand(unittest.TestCase): def test_dump(self): @@ -21,5 +21,13 @@ def test_info(self): state.info(entry=folders_0, raw=True, recuisive=True, user='user1') state.teardown() + def skip_test_clone(self): + state = JuicefsCommandMachine() + v1 = state.init_folders() + v2 = state.create_file(content=b'\x9bcR\xba', file_name='ygbl', mode='x', parent=v1, umask=466, user='root') + state.chmod(entry=v1, mode=715, user='root') + state.clone(entry=v2, new_entry_name='drqj', parent=v1, preserve=False, user='user1') + state.teardown() + if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/.github/scripts/hypo/common.py b/.github/scripts/hypo/common.py index 0c77db184d8f..69349c336301 100644 --- a/.github/scripts/hypo/common.py +++ b/.github/scripts/hypo/common.py @@ -6,8 +6,15 @@ import pwd import subprocess import sys - - +def replace(src, old, new): + if isinstance(src, str): + return src.replace(old, new) + elif isinstance(src, list) or isinstance(src, tuple): + return [replace(x, old, new) for x in src] + elif isinstance(src, dict): + return {k: replace(v, old, new) for k, v in src.items()} + else: + return src def run_cmd(command: str) -> str: print('run_cmd:'+command) if '|' in command or '>' in command: diff --git a/.github/scripts/hypo/fs.py b/.github/scripts/hypo/fs.py index 8f89f0fea44e..d378105dd34f 100644 --- a/.github/scripts/hypo/fs.py +++ b/.github/scripts/hypo/fs.py @@ -30,13 +30,6 @@ class JuicefsMachine(RuleBasedStateMachine): EntryWithACL = Bundle('entry_with_acl') FilesWithXattr = Bundle('files_with_xattr') start = time.time() - ROOT_DIR1=os.environ.get('ROOT_DIR1', '/tmp/fsrand').rstrip('/') - ROOT_DIR2=os.environ.get('ROOT_DIR2', '/tmp/jfs/fsrand').rstrip('/') - - fsop1 = FsOperation('fs1', ROOT_DIR1) - fsop2 = FsOperation('fs2', ROOT_DIR2) - - ZONES = {ROOT_DIR1:common.get_zones(ROOT_DIR1), ROOT_DIR2:common.get_zones(ROOT_DIR2)} SUDO_USERS = ['root'] USERS=['root', 'user1', 'user2','user3'] GROUPS = USERS+['group1', 'group2', 'group3', 'group4'] @@ -44,16 +37,13 @@ class JuicefsMachine(RuleBasedStateMachine): INCLUDE_RULES = [] EXCLUDE_RULES = ['rebalance_dir', 'rebalance_file', \ 'clone_cp_file', 'clone_cp_dir'] - + fsop1 = FsOperation('fs1', os.environ.get('ROOT_DIR1', '/tmp/fsrand')) + fsop2 = FsOperation('fs2', os.environ.get('ROOT_DIR2', '/tmp/jfs/fsrand')) + @initialize(target=Folders) def init_folders(self): - if not os.path.exists(self.ROOT_DIR1): - os.makedirs(self.ROOT_DIR1) - if not os.path.exists(self.ROOT_DIR2): - os.makedirs(self.ROOT_DIR2) - if os.environ.get('PROFILE', 'dev') != 'generate': - common.clean_dir(self.ROOT_DIR1) - common.clean_dir(self.ROOT_DIR2) + self.fsop1.init_rootdir() + self.fsop2.init_rootdir() return '' def create_users(self, users): @@ -61,11 +51,16 @@ def create_users(self, users): if user != 'root': common.create_user(user) + def get_default_rootdir1(self): + return '/tmp/fsrand' + + def get_default_rootdir2(self): + return '/tmp/jfs/fsrand' + def __init__(self): super(JuicefsMachine, self).__init__() print(f'__init__') - if os.environ.get('EXCLUDE_RULES') is not None: - self.EXCLUDE_RULES = os.environ.get('EXCLUDE_RULES').split(',') + if not self.group_created: for group in self.GROUPS: common.create_group(group) @@ -83,17 +78,13 @@ def equal(self, result1, result2): if type(result1) != type(result2): return False if isinstance(result1, Exception): - r1 = str(result1).replace(self.ROOT_DIR1, '') - r2 = str(result2).replace(self.ROOT_DIR2, '') - return r1 == r2 - elif isinstance(result1, tuple): - return result1 == result2 - elif isinstance(result1, str): - r1 = str(result1).replace(self.ROOT_DIR1, '') - r2 = str(result2).replace(self.ROOT_DIR2, '') - return r1 == r2 - else: - return result1 == result2 + if 'panic:' in str(result1) or 'panic:' in str(result2): + return False + result1 = str(result1) + result2 = str(result2) + result1 = common.replace(result1, self.fsop1.root_dir, '***') + result2 = common.replace(result2, self.fsop2.root_dir, '***') + return result1 == result2 def seteuid(self, user): os.seteuid(pwd.getpwnam(user).pw_uid) @@ -472,38 +463,37 @@ def chown(self, entry, owner, user='root'): @rule( dir =Folders, vdirs = st.integers(min_value=2, max_value=31) ) @precondition(lambda self: self.should_run('split_dir') \ - and (common.is_jfs(self.ROOT_DIR1) or common.is_jfs(self.ROOT_DIR2)) + and (self.fsop1.is_jfs() or self.fsop2.is_jfs()) ) def split_dir(self, dir, vdirs): self.fsop1.do_split_dir(dir, vdirs) self.fsop2.do_split_dir(dir, vdirs) - @rule(dir = Folders) @precondition(lambda self: self.should_run('merge_dir') \ - and (common.is_jfs(self.ROOT_DIR1) or common.is_jfs(self.ROOT_DIR2)) + and (self.fsop1.is_jfs() or self.fsop2.is_jfs()) ) def merge_dir(self, dir): self.fsop1.do_merge_dir(dir) self.fsop2.do_merge_dir(dir) - @rule(dir = Folders, - zone1=st.sampled_from(ZONES[ROOT_DIR1]), - zone2=st.sampled_from(ZONES[ROOT_DIR2]), - is_vdir=st.booleans()) + # @rule(dir = Folders, + # zone1=st.sampled_from(self.fsop1.get_zones()), + # zone2=st.sampled_from(self.fsop2.get_zones()), + # is_vdir=st.booleans()) @precondition(lambda self: self.should_run('rebalance_dir') \ - and (common.is_jfs(self.ROOT_DIR1) or common.is_jfs(self.ROOT_DIR2)) + and (self.fsop1.is_jfs() or self.fsop2.is_jfs()) ) def rebalance_dir(self, dir, zone1, zone2, is_vdir): self.fsop1.do_rebalance(dir, zone1, is_vdir) self.fsop2.do_rebalance(dir, zone2, is_vdir) - @rule(file = Files, - zone1=st.sampled_from(ZONES[ROOT_DIR1]), - zone2=st.sampled_from(ZONES[ROOT_DIR2]), - ) + # @rule(file = Files, + # zone1=st.sampled_from(self.fsop1.get_zones()), + # zone2=st.sampled_from(self.fsop2.get_zones()), + # ) @precondition(lambda self: self.should_run('rebalance_file') \ - and (common.is_jfs(self.ROOT_DIR1) or common.is_jfs(self.ROOT_DIR2)) + and (self.fsop1.is_jfs() or self.fsop2.is_jfs()) ) def rebalance_file(self, file, zone1, zone2): self.fsop1.do_rebalance(file, zone1, False) @@ -511,10 +501,6 @@ def rebalance_file(self, file, zone1, zone2): def teardown(self): pass - # if COMPARE and os.path.exists(ROOT_DIR1): - # common.compare_content(ROOT_DIR1, ROOT_DIR2) - # common.compare_stat(ROOT_DIR1, ROOT_DIR2) - # common.compare_acl(ROOT_DIR1, ROOT_DIR2) if __name__ == '__main__': MAX_EXAMPLE=int(os.environ.get('MAX_EXAMPLE', '100')) diff --git a/.github/scripts/hypo/fs_op.py b/.github/scripts/hypo/fs_op.py index 41bf775f8f68..b43fe994aa02 100644 --- a/.github/scripts/hypo/fs_op.py +++ b/.github/scripts/hypo/fs_op.py @@ -23,9 +23,9 @@ class FsOperation: JFS_CONTROL_FILES=['.accesslog', '.config', '.stats'] stats = Statistics() - def __init__(self, name, root_dir): + def __init__(self, name, root_dir:str): self.logger =common.setup_logger(f'./{name}.log', name, os.environ.get('LOG_LEVEL', 'INFO')) - self.root_dir = root_dir + self.root_dir = root_dir.rstrip('/') def run_cmd(self, command:str) -> str: self.logger.info(f'run_cmd: {command}') @@ -41,6 +41,18 @@ def run_cmd(self, command:str) -> str: raise e return output.stdout.decode() + def get_zones(self): + return common.get_zones(self.root_dir) + + def is_jfs(self): + return common.is_jfs(self.root_dir) + + def init_rootdir(self): + if not os.path.exists(self.root_dir): + os.makedirs(self.root_dir) + if os.environ.get('PROFILE', 'dev') != 'generate': + common.clean_dir(self.root_dir) + def seteuid(self, user): os.seteuid(pwd.getpwnam(user).pw_uid) os.setegid(pwd.getpwnam(user).pw_gid) diff --git a/.github/scripts/hypo/s3.py b/.github/scripts/hypo/s3.py index f0195ee73c00..c074d2d6915c 100644 --- a/.github/scripts/hypo/s3.py +++ b/.github/scripts/hypo/s3.py @@ -14,6 +14,7 @@ from s3_op import S3Client from s3_strategy import * from s3_contant import * +import common # minio client: https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-04-22T17-40-00Z # minio server: minio/minio:RELEASE.2021-04-22T15-44-28Z # docker run -d -p 9000:9000 --name minio -e "MINIO_ACCESS_KEY=minioadmin" -e "MINIO_SECRET_KEY=minioadmin" minio/minio:RELEASE.2021-04-22T15-44-28Z server /data @@ -62,16 +63,6 @@ def init_aliases(self): @initialize(target=policies) def init_policies(self): return multiple(*BUILD_IN_POLICIES) - - def replace(self, result, prefix, url): - if isinstance(result, str): - return result.replace(prefix, '***').replace(url, '***') - elif isinstance(result, list): - return [self.replace(x, prefix, url) for x in result] - elif isinstance(result, dict): - return {k: self.replace(v, prefix, url) for k, v in result.items()} - else: - return result def equal(self, result1, result2): if os.getenv('PROFILE', 'dev') == 'generate': @@ -81,8 +72,10 @@ def equal(self, result1, result2): if isinstance(result1, Exception): result1 = str(result1) result2 = str(result2) - result1 = self.replace(result1, self.PREFIX1, self.URL1) - result2 = self.replace(result2, self.PREFIX2, self.URL2) + result1 = common.replace(result1, self.PREFIX1, '***') + result1 = common.replace(result1, self.URL1, '***') + result2 = common.replace(result2, self.PREFIX2, '***') + result2 = common.replace(result2, self.URL2, '***') # print(f'result1 is {result1}\nresult2 is {result2}') return result1 == result2 diff --git a/.github/workflows/command2.yml b/.github/workflows/command2.yml new file mode 100644 index 000000000000..2a56618cd52b --- /dev/null +++ b/.github/workflows/command2.yml @@ -0,0 +1,122 @@ +name: "command-random-test" + +on: + push: + branches: + - 'main' + - 'release-**' + paths: + - '.github/scripts/command/random.sh' + - '.github/scripts/hypo/command*.py' + - '**/command2.yml' + pull_request: + branches: + - 'main' + - 'release-**' + paths: + - '.github/scripts/command/random.sh' + - '.github/scripts/hypo/command*.py' + - '**/command2.yml' + schedule: + - cron: '30 20 * * *' + + workflow_dispatch: + inputs: + debug: + type: boolean + description: "Run the build with tmate debugging enabled" + required: false + default: false + +jobs: + build-matrix: + runs-on: ubuntu-20.04 + steps: + - id: set-matrix + run: | + echo "github.event_name is ${{github.event_name}}" + echo "GITHUB_REF_NAME is ${GITHUB_REF_NAME}" + if [ "${{github.event_name}}" == "schedule" ]; then + echo 'meta_matrix=["sqlite3", "mysql", "tikv", "tidb", "postgres", "mariadb", "fdb"]' >> $GITHUB_OUTPUT + elif [ "${{github.event_name}}" == "pull_request" ]; then + echo 'meta_matrix=["sqlite3", "tikv"]' >> $GITHUB_OUTPUT + elif [ "${{github.event_name}}" == "workflow_dispatch" ]; then + echo 'meta_matrix=["sqlite3", "tikv"]' >> $GITHUB_OUTPUT + else + echo 'meta_matrix=["sqlite3", "tikv"]' >> $GITHUB_OUTPUT + fi + outputs: + meta_matrix: ${{ steps.set-matrix.outputs.meta_matrix }} + + test: + needs: [build-matrix] + strategy: + fail-fast: false + matrix: + meta: ${{ fromJson(needs.build-matrix.outputs.meta_matrix) }} + runs-on: ubuntu-20.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 1 + + - name: Set Variable + id: vars + run: | + if [ "${{matrix.meta}}" == "fdb" ]; then + echo "target=juicefs.fdb" >> $GITHUB_OUTPUT + else + echo "target=juicefs" >> $GITHUB_OUTPUT + fi + + - name: Build + uses: ./.github/actions/build + with: + target: ${{steps.vars.outputs.target}} + + - name: Run Example + timeout-minutes: 60 + run: | + sudo META1=redis META2=${{matrix.meta}} .github/scripts/command/random.sh test_run_examples + + - name: Run All + timeout-minutes: 60 + run: | + sudo META1=redis META2=${{matrix.meta}} .github/scripts/command/random.sh test_run_all + + - name: Log + if: always() + run: | + echo "juicefs log" + sudo tail -n 1000 /var/log/juicefs.log + grep ":" /var/log/juicefs.log && exit 1 || true + + - name: Setup upterm session + if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) + # if: failure() + timeout-minutes: 60 + uses: lhotari/action-upterm@v1 + + success-all-test: + runs-on: ubuntu-latest + needs: [test] + if: always() + steps: + - uses: technote-space/workflow-conclusion-action@v3 + - uses: actions/checkout@v3 + + - name: Check Failure + if: env.WORKFLOW_CONCLUSION == 'failure' + run: exit 1 + + - name: Send Slack Notification + if: failure() && github.event_name != 'workflow_dispatch' + uses: juicedata/slack-notify-action@main + with: + channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" + slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" + + - name: Success + if: success() + run: echo "All Done" diff --git a/.github/workflows/fsrand.yml b/.github/workflows/fsrand.yml index e693e7dc4970..a9f7b5762a8b 100644 --- a/.github/workflows/fsrand.yml +++ b/.github/workflows/fsrand.yml @@ -103,7 +103,7 @@ jobs: else PROFILE='pull_request' fi - sudo -E PROFILE=${PROFILE} python3 .github/scripts/hypo/fs.py 2>&1 | tee fsrand.log + sudo -E PROFILE=${PROFILE} LOG_LEVEL=WARNING python3 .github/scripts/hypo/fs.py 2>&1 | tee fsrand.log exit ${PIPESTATUS[0]} - name: check fsrand.log