From 9c00467191d459436c4c1a9fec44526226252ad9 Mon Sep 17 00:00:00 2001 From: zhoucheng Date: Thu, 19 Dec 2024 15:17:52 +0800 Subject: [PATCH 1/6] test load dump in binary format --- .github/scripts/command/load_dump.sh | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/scripts/command/load_dump.sh b/.github/scripts/command/load_dump.sh index 556fd3d25787..d83b387cad58 100755 --- a/.github/scripts/command/load_dump.sh +++ b/.github/scripts/command/load_dump.sh @@ -86,6 +86,16 @@ do_dump_load_with_fsrand(){ do_dump_load_and_compare --fast --skip-trash } +do_dump_load_with_fsrand_binary(){ + trash_days=$1 + prepare_test + ./juicefs format $META_URL myjfs --trash-days $trash_days --enable-acl + ./juicefs mount -d $META_URL /jfs --enable-xattr + SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=30 STEP_COUNT=20 PROFILE=generate ROOT_DIR1=/jfs/fsrand ROOT_DIR2=/tmp/fsrand python3 .github/scripts/hypo/fs.py || true + # find /jfs/fsrand -mindepth 1 -maxdepth 1 ! -name "syly" -exec rm -rf {} \; + do_dump_load_and_compare_binary +} + do_dump_load_and_compare() { option=$@ @@ -101,6 +111,24 @@ do_dump_load_and_compare() umount /jfs2 } +do_dump_load_and_compare_in_binary() +{ + option=$@ + echo option is $option + ./juicefs dump $META_URL dump.bin $option --binary + rm -rf test2.db + ./juicefs load sqlite3://test2.db dump.bin --binary --stat | tee dump.stat + ./juicefs load sqlite3://test2.db dump.bin --binary + ./juicefs dump sqlite3://test2.db dump2.bin $option + ./juicefs load sqlite3://test2.db dump2.bin --binary --stat | tee dump2.stat + diff dump.stat dump2.stat + # compare_dump_json + ./juicefs mount -d sqlite3://test2.db /jfs2 + diff -ur /jfs/fsrand /jfs2/fsrand --no-dereference + compare_stat_acl_xattr /jfs/fsrand /jfs2/fsrand + umount /jfs2 +} + compare_dump_json(){ cp dump.json dump.json.bak cp dump2.json dump2.json.bak From e238c35379ba53bedd672ef8e73adca65fb8ef97 Mon Sep 17 00:00:00 2001 From: zhoucheng Date: Thu, 19 Dec 2024 15:26:12 +0800 Subject: [PATCH 2/6] wip --- .github/scripts/command/load_dump.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/scripts/command/load_dump.sh b/.github/scripts/command/load_dump.sh index d83b387cad58..15b1ac55ae29 100755 --- a/.github/scripts/command/load_dump.sh +++ b/.github/scripts/command/load_dump.sh @@ -86,13 +86,11 @@ do_dump_load_with_fsrand(){ do_dump_load_and_compare --fast --skip-trash } -do_dump_load_with_fsrand_binary(){ - trash_days=$1 +test_dump_load_with_fsrand_binary(){ prepare_test ./juicefs format $META_URL myjfs --trash-days $trash_days --enable-acl ./juicefs mount -d $META_URL /jfs --enable-xattr SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=30 STEP_COUNT=20 PROFILE=generate ROOT_DIR1=/jfs/fsrand ROOT_DIR2=/tmp/fsrand python3 .github/scripts/hypo/fs.py || true - # find /jfs/fsrand -mindepth 1 -maxdepth 1 ! -name "syly" -exec rm -rf {} \; do_dump_load_and_compare_binary } From 8f44cf07e36334da630941af6e5986ae2e89d2ed Mon Sep 17 00:00:00 2001 From: zhoucheng Date: Thu, 19 Dec 2024 15:26:45 +0800 Subject: [PATCH 3/6] wip --- .github/scripts/command/load_dump.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/command/load_dump.sh b/.github/scripts/command/load_dump.sh index 15b1ac55ae29..0fe61ed3fa83 100755 --- a/.github/scripts/command/load_dump.sh +++ b/.github/scripts/command/load_dump.sh @@ -88,7 +88,7 @@ do_dump_load_with_fsrand(){ test_dump_load_with_fsrand_binary(){ prepare_test - ./juicefs format $META_URL myjfs --trash-days $trash_days --enable-acl + ./juicefs format $META_URL myjfs --trash-days 1 --enable-acl ./juicefs mount -d $META_URL /jfs --enable-xattr SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=30 STEP_COUNT=20 PROFILE=generate ROOT_DIR1=/jfs/fsrand ROOT_DIR2=/tmp/fsrand python3 .github/scripts/hypo/fs.py || true do_dump_load_and_compare_binary From e4803685d78af13fe73f2b7653dc47d9b1658810 Mon Sep 17 00:00:00 2001 From: zhoucheng Date: Thu, 19 Dec 2024 15:41:59 +0800 Subject: [PATCH 4/6] fix typo --- .github/scripts/command/load_dump.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/command/load_dump.sh b/.github/scripts/command/load_dump.sh index 0fe61ed3fa83..3b8069cacf00 100755 --- a/.github/scripts/command/load_dump.sh +++ b/.github/scripts/command/load_dump.sh @@ -91,7 +91,7 @@ test_dump_load_with_fsrand_binary(){ ./juicefs format $META_URL myjfs --trash-days 1 --enable-acl ./juicefs mount -d $META_URL /jfs --enable-xattr SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=30 STEP_COUNT=20 PROFILE=generate ROOT_DIR1=/jfs/fsrand ROOT_DIR2=/tmp/fsrand python3 .github/scripts/hypo/fs.py || true - do_dump_load_and_compare_binary + do_dump_load_and_compare_in_binary } do_dump_load_and_compare() From e1bdcb337d4248fc4ae4b96a3631827d12994077 Mon Sep 17 00:00:00 2001 From: zhoucheng Date: Sat, 4 Jan 2025 09:22:42 +1000 Subject: [PATCH 5/6] wip --- .github/scripts/command/load_dump.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/command/load_dump.sh b/.github/scripts/command/load_dump.sh index 3b8069cacf00..f6966eb3f31b 100755 --- a/.github/scripts/command/load_dump.sh +++ b/.github/scripts/command/load_dump.sh @@ -45,6 +45,7 @@ test_dump_with_keep_secret() prepare_test ./juicefs format $META_URL myjfs --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin ./juicefs dump --keep-secret-key $META_URL dump.json --fast + cat dump.json python3 .github/scripts/flush_meta.py $META_URL ./juicefs load $META_URL dump.json ./juicefs mount -d $META_URL /jfs @@ -57,6 +58,7 @@ test_dump_without_keep_secret() prepare_test ./juicefs format $META_URL myjfs --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin ./juicefs dump $META_URL dump.json --fast + cat dump.json python3 .github/scripts/flush_meta.py $META_URL ./juicefs load $META_URL dump.json ./juicefs mount -d $META_URL /jfs && echo "mount should fail" && exit 1 || true From e8188c99fa9c9c1d14d741124b5de0c7c429c145 Mon Sep 17 00:00:00 2001 From: zhoucheng Date: Sat, 4 Jan 2025 19:28:49 +1000 Subject: [PATCH 6/6] wip --- .github/scripts/command/load_dump.sh | 79 +++++++++++++--------------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/.github/scripts/command/load_dump.sh b/.github/scripts/command/load_dump.sh index f6966eb3f31b..3ea9cc5f4543 100755 --- a/.github/scripts/command/load_dump.sh +++ b/.github/scripts/command/load_dump.sh @@ -24,15 +24,24 @@ mc alias set myminio http://localhost:9000 minioadmin minioadmin python3 -c "import xattr" || sudo pip install xattr test_dump_load_with_iflag(){ + do_dump_load_with_iflag +} + +test_dump_load_with_iflag_binary(){ + do_dump_load_with_iflag --binary +} + +do_dump_load_with_iflag(){ + option=$@ prepare_test ./juicefs format $META_URL myjfs ./juicefs mount -d $META_URL /jfs --enable-ioctl echo "hello" > /jfs/hello.txt chattr +i /jfs/hello.txt - ./juicefs dump $META_URL dump.json --fast + ./juicefs dump $META_URL dump.json $option umount_jfs /jfs $META_URL python3 .github/scripts/flush_meta.py $META_URL - ./juicefs load $META_URL dump.json + ./juicefs load $META_URL dump.json $option ./juicefs mount -d $META_URL /jfs --enable-ioctl echo "hello" > /jfs/hello.txt && echo "write should fail" && exit 1 || true chattr -i /jfs/hello.txt @@ -40,27 +49,30 @@ test_dump_load_with_iflag(){ cat /jfs/hello.txt | grep world } -test_dump_with_keep_secret() +test_dump_load_with_keep_secret_key(){ + do_dump_load_with_keep_secret_key +} + +test_dump_load_with_keep_secret_key_in_binary(){ + do_dump_load_with_keep_secret_key --binary +} + +do_dump_load_with_keep_secret_key() { + option=$@ prepare_test ./juicefs format $META_URL myjfs --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin - ./juicefs dump --keep-secret-key $META_URL dump.json --fast - cat dump.json + ./juicefs dump --keep-secret-key $META_URL dump.json $option python3 .github/scripts/flush_meta.py $META_URL - ./juicefs load $META_URL dump.json + ./juicefs load $META_URL dump.json $option ./juicefs mount -d $META_URL /jfs echo "hello" > /jfs/hello.txt cat /jfs/hello.txt | grep hello -} -test_dump_without_keep_secret() -{ - prepare_test - ./juicefs format $META_URL myjfs --storage minio --bucket http://localhost:9000/test --access-key minioadmin --secret-key minioadmin - ./juicefs dump $META_URL dump.json --fast - cat dump.json + umount_jfs /jfs $META_URL + ./juicefs dump $META_URL dump.json $option python3 .github/scripts/flush_meta.py $META_URL - ./juicefs load $META_URL dump.json + ./juicefs load $META_URL dump.json $option ./juicefs mount -d $META_URL /jfs && echo "mount should fail" && exit 1 || true ./juicefs config --secret-key minioadmin $META_URL ./juicefs mount -d $META_URL /jfs @@ -68,6 +80,7 @@ test_dump_without_keep_secret() cat /jfs/hello.txt | grep hello } + test_dump_load_with_trash_enable(){ do_dump_load_with_fsrand 1 } @@ -81,48 +94,28 @@ do_dump_load_with_fsrand(){ ./juicefs format $META_URL myjfs --trash-days $trash_days --enable-acl ./juicefs mount -d $META_URL /jfs --enable-xattr SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=30 STEP_COUNT=20 PROFILE=generate ROOT_DIR1=/jfs/fsrand ROOT_DIR2=/tmp/fsrand python3 .github/scripts/hypo/fs.py || true - # find /jfs/fsrand -mindepth 1 -maxdepth 1 ! -name "syly" -exec rm -rf {} \; do_dump_load_and_compare + do_dump_load_and_compare --binary do_dump_load_and_compare --fast do_dump_load_and_compare --skip-trash do_dump_load_and_compare --fast --skip-trash } -test_dump_load_with_fsrand_binary(){ - prepare_test - ./juicefs format $META_URL myjfs --trash-days 1 --enable-acl - ./juicefs mount -d $META_URL /jfs --enable-xattr - SEED=$SEED LOG_LEVEL=WARNING MAX_EXAMPLE=30 STEP_COUNT=20 PROFILE=generate ROOT_DIR1=/jfs/fsrand ROOT_DIR2=/tmp/fsrand python3 .github/scripts/hypo/fs.py || true - do_dump_load_and_compare_in_binary -} - do_dump_load_and_compare() { option=$@ echo option is $option ./juicefs dump $META_URL dump.json $option rm -rf test2.db - ./juicefs load sqlite3://test2.db dump.json + if [[ "$option" == *"--binary"* ]]; then + ./juicefs load sqlite3://test2.db dump.json $option + else + ./juicefs load sqlite3://test2.db dump.json + fi ./juicefs dump sqlite3://test2.db dump2.json $option - # compare_dump_json - ./juicefs mount -d sqlite3://test2.db /jfs2 - diff -ur /jfs/fsrand /jfs2/fsrand --no-dereference - compare_stat_acl_xattr /jfs/fsrand /jfs2/fsrand - umount /jfs2 -} - -do_dump_load_and_compare_in_binary() -{ - option=$@ - echo option is $option - ./juicefs dump $META_URL dump.bin $option --binary - rm -rf test2.db - ./juicefs load sqlite3://test2.db dump.bin --binary --stat | tee dump.stat - ./juicefs load sqlite3://test2.db dump.bin --binary - ./juicefs dump sqlite3://test2.db dump2.bin $option - ./juicefs load sqlite3://test2.db dump2.bin --binary --stat | tee dump2.stat - diff dump.stat dump2.stat - # compare_dump_json + # if [[ "$option" != *"--binary"* ]]; then + # compare_dump_json + # fi ./juicefs mount -d sqlite3://test2.db /jfs2 diff -ur /jfs/fsrand /jfs2/fsrand --no-dereference compare_stat_acl_xattr /jfs/fsrand /jfs2/fsrand