Merge branch 'main' into support-put-inplace

# Conflicts:
#	pkg/object/hdfs.go

.github/scripts/command/quota.sh

@@ -62,6 +62,68 @@ test_total_inodes(){
     echo a | tee /jfs/test2001 2>error.log && echo "write should fail on out of inodes" && exit 1 || true
 }
 
+test_nested_dir(){
+    prepare_test
+    ./juicefs format $META_URL myjfs
+    ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
+    file_count=1000
+    mkdir -p /jfs/d1/{d1,d2,d3,d4,d5,d6}/{d1,d2,d3,d4,d5,d6}/{d1,d2,d3,d4,d5,d6}
+    dir_count=$(find /jfs/d1 -type d | wc -l)
+    echo "dir_count: $dir_count"
+    ./juicefs quota set $META_URL --path /d1 --inodes $((file_count+dir_count-1))
+    sleep $HEARTBEAT_INTERVAL
+    for i in $(seq 1 $file_count); do
+        subdir=$(find /jfs/d1/ -type d | shuf -n 1)
+        echo "touch $subdir/test$i" && touch $subdir/test$i
+    done
+    sleep $VOLUME_QUOTA_FLUSH_INTERVAL
+    subdir=$(find /jfs/d1/ -type d | shuf -n 1)
+    touch $subdir/test 2>error.log && echo "write should fail on out of inodes" && exit 1 || true
+    grep -i "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
+
+    ./juicefs quota set $META_URL --path /d1 --inodes $((file_count+dir_count))
+    sleep $HEARTBEAT_INTERVAL
+    subdir=$(find /jfs/d1/ -type d | shuf -n 1)
+    touch $subdir/test
+}
+
+test_remove_and_restore(){
+    prepare_test
+    ./juicefs format $META_URL myjfs
+    ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
+    mkdir -p /jfs/d
+    ./juicefs quota set $META_URL --path /d --capacity 1
+    sleep $((HEARTBEAT_INTERVAL+1))
+    dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1
+    sleep $DIR_QUOTA_FLUSH_INTERVAL
+    ./juicefs quota get $META_URL --path /d 2>&1 | tee quota.log
+    used=$(cat quota.log | grep "/d" | awk -F'|' '{print $5}'  | tr -d '[:space:]')
+    [[ $used != "100%" ]] && echo "used should be 100%" && exit 1 || true
+    echo a | tee -a /jfs/d/test1 2>error.log && echo "write should fail on out of space" && exit 1 || true
+    grep -i "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
+
+    echo "remove test1" && rm /jfs/d/test1 -rf
+    sleep $DIR_QUOTA_FLUSH_INTERVAL
+    ./juicefs quota get $META_URL --path /d 2>&1 | tee quota.log
+    used=$(cat quota.log | grep "/d" | awk -F'|' '{print $5}'  | tr -d '[:space:]')
+    [[ $used != "0%" ]] && echo "used should be 0%" && exit 1 || true
+
+    trash_dir=$(ls /jfs/.trash)
+    ./juicefs restore $META_URL $trash_dir --put-back
+    ./juicefs quota get $META_URL --path /d 2>&1 | tee quota.log
+    used=$(cat quota.log | grep "/d" | awk -F'|' '{print $5}'  | tr -d '[:space:]')
+    [[ $used != "100%" ]] && echo "used should be 100%" && exit 1 || true
+    sleep $HEARTBEAT_INTERVAL
+    echo a | tee -a /jfs/d/test1 2>error.log && echo "write should fail on out of space" && exit 1 || true
+    grep -i "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
+
+    echo "remove test1" && rm /jfs/d/test1 -rf
+    dd if=/dev/zero of=/jfs/d/test2 bs=1M count=1
+    trash_dir=$(ls /jfs/.trash)
+    ./juicefs restore $META_URL $trash_dir --put-back 2>&1 | tee restore.log
+    grep "disk quota exceeded" restore.log || (echo "check restore log failed" && exit 1)
+}
+
 test_dir_capacity(){
     prepare_test
     ./juicefs format $META_URL myjfs

.github/workflows/aitests.yml

@@ -126,13 +126,13 @@ jobs:
           df -lh
 
       - name: log
-        if: ${{ always() }}
+        if: always()
         run: | 
           tail -300 /var/log/juicefs.log
           grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"

.github/workflows/bash/rm_fs

@@ -30,8 +30,10 @@ gf29 growfiles -W gf29 -b -D 0 -r 1-4096 -R 0-33554432 -i 0 -L 60 -C 1 -u -f gfs
 gf30 growfiles -W gf30 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -o O_RDWR,O_CREAT,O_SYNC -g 20480 -T 10 -t 20480 -f gf-sync-$$ -d $TMPDIR
 rwtest01 export LTPROOT; rwtest -N rwtest01 -c -q -i 60s  -f sync 10%25000:$TMPDIR/rw-sync-$$
 rwtest02 export LTPROOT; rwtest -N rwtest02 -c -q -i 60s  -f buffered 10%25000:$TMPDIR/rw-buffered-$$
+rwtest03 export LTPROOT; rwtest -N rwtest03 -c -q -i 60s -n 2  -f buffered -s mmread,mmwrite -m random -Dv 10%25000:$TMPDIR/mm-buff-$$
 rwtest04 export LTPROOT; rwtest -N rwtest04 -c -q -i 60s -n 2  -f sync -s mmread,mmwrite -m random -Dv 10%25000:$TMPDIR/mm-sync-$$
 rwtest05 export LTPROOT; rwtest -N rwtest05 -c -q -i 50 -T 64b 500b:$TMPDIR/rwtest01%f
 iogen01 export LTPROOT; rwtest -N iogen01 -i 120s -s read,write -Da -Dv -n 2 500b:$TMPDIR/doio.f1.$$ 1000b:$TMPDIR/doio.f2.$$
 quota_remount_test01 quota_remount_test01.sh
 isofs isofs.sh
+fs_fill fs_fill

.github/workflows/bash/rm_syscalls

@@ -1,23 +1,78 @@
+alarm02 alarm02
+alarm03 alarm03
+alarm05 alarm05
+alarm06 alarm06
+alarm07 alarm07
+bind01 bind01
+bind02 bind02
+bind03 bind03
+bind04 bind04
+bind05 bind05
+bind06 bind06
 bpf_prog05 bpf_prog05
 cacheflush01 cacheflush01
 chown01_16 chown01_16
 chown02_16 chown02_16
 chown03_16 chown03_16
 chown04_16 chown04_16
 chown05_16 chown05_16
+clock_adjtime01 clock_adjtime01
+clock_adjtime02 clock_adjtime02
+clock_getres01 clock_getres01
+clock_nanosleep01 clock_nanosleep01
+clock_nanosleep02 clock_nanosleep02
 clock_nanosleep03 clock_nanosleep03
+clock_nanosleep04 clock_nanosleep04
+clock_gettime01 clock_gettime01
+clock_gettime02 clock_gettime02
 clock_gettime03 clock_gettime03
+clock_gettime04 clock_gettime04
 leapsec01 leapsec01
+clock_settime01 clock_settime01
+clock_settime02 clock_settime02
+clock_settime03 clock_settime03
 close_range01 close_range01
 close_range02 close_range02
 fallocate06 fallocate06
+fanotify01 fanotify01
+fanotify02 fanotify02
+fanotify03 fanotify03
+fanotify04 fanotify04
+fanotify05 fanotify05
+fanotify06 fanotify06
+fanotify07 fanotify07
+fanotify08 fanotify08
+fanotify09 fanotify09
+fanotify10 fanotify10
+fanotify11 fanotify11
+fanotify12 fanotify12
+fanotify13 fanotify13
+fanotify14 fanotify14
+fanotify15 fanotify15
+fanotify16 fanotify16
+fanotify17 fanotify17
+fanotify18 fanotify18
+fanotify19 fanotify19
 fchown01_16 fchown01_16
 fchown02_16 fchown02_16
 fchown03_16 fchown03_16
 fchown04_16 fchown04_16
 fchown05_16 fchown05_16
 fcntl06 fcntl06
 fcntl06_64 fcntl06_64
+fork01 fork01
+fork02 fork02
+fork03 fork03
+fork04 fork04
+fork05 fork05
+fork06 fork06
+fork07 fork07
+fork08 fork08
+fork09 fork09
+fork10 fork10
+fork11 fork11
+fork13 fork13 -i 1000000
+fork14 fork14
 getegid01_16 getegid01_16
 getegid02_16 getegid02_16
 geteuid01_16 geteuid01_16
@@ -33,12 +88,26 @@ getresuid01_16 getresuid01_16
 getresuid02_16 getresuid02_16
 getresuid03_16 getresuid03_16
 getrusage04 getrusage04
+gettimeofday01 gettimeofday01
+gettimeofday02 gettimeofday02
 getuid01_16 getuid01_16
 getuid03_16 getuid03_16
+ioctl03      ioctl03
 ioctl_sg01 ioctl_sg01
 fanotify16 fanotify16
 fanotify18 fanotify18
 fanotify19 fanotify19
+kill02 kill02
+kill03 kill03
+kill05 kill05
+kill06 kill06
+kill07 kill07
+kill08 kill08
+kill09 kill09
+kill10 kill10
+kill11 kill11
+kill12 kill12
+kill13 kill13
 lchown01_16 lchown01_16
 lchown02_16 lchown02_16
 lchown03_16 lchown03_16
@@ -63,20 +132,36 @@ move_pages11 move_pages11
 move_pages12 move_pages12
 msgctl05 msgctl05
 msgstress04 msgstress04
+nanosleep01 nanosleep01
+nanosleep02 nanosleep02
+nanosleep04 nanosleep04
 openat201 openat201
 openat202 openat202
 openat203 openat203
 madvise06 madvise06
 madvise09 madvise09
+pselect01 pselect01
+pselect01_64 pselect01_64
 ptrace04 ptrace04
 quotactl01 quotactl01
 quotactl04 quotactl04
 quotactl06 quotactl06
 readdir21 readdir21
 recvmsg03 recvmsg03
+rt_sigaction01 rt_sigaction01
+rt_sigaction02 rt_sigaction02
+rt_sigaction03 rt_sigaction03
+rt_sigprocmask01 rt_sigprocmask01
+rt_sigprocmask02 rt_sigprocmask02
+rt_sigqueueinfo01 rt_sigqueueinfo01
+rt_sigsuspend01 rt_sigsuspend01
+rt_sigtimedwait01 rt_sigtimedwait01
+rt_tgsigqueueinfo01 rt_tgsigqueueinfo01
 sbrk03 sbrk03
+select02 select02
 semctl08 semctl08
 semctl09 semctl09
+sendfile09_64 sendfile09_64
 set_mempolicy01 set_mempolicy01
 set_mempolicy02 set_mempolicy02
 set_mempolicy03 set_mempolicy03
@@ -117,9 +202,12 @@ setreuid04_16 setreuid04_16
 setreuid05_16 setreuid05_16
 setreuid06_16 setreuid06_16
 setreuid07_16 setreuid07_16
+setsockopt06 setsockopt06
+setsockopt07 setsockopt07
 setuid01_16 setuid01_16
 setuid03_16 setuid03_16
 setuid04_16 setuid04_16
+shmctl05 shmctl05
 shmctl06 shmctl06
 socketcall01 socketcall01
 socketcall02 socketcall02
@@ -132,10 +220,88 @@ swapon02 swapon02
 swapon03 swapon03
 switch01 endian_switch01
 sysinfo03 sysinfo03
+syslog01 syslog01
+syslog02 syslog02
+syslog03 syslog03
+syslog04 syslog04
+syslog05 syslog05
+syslog06 syslog06
+syslog07 syslog07
+syslog08 syslog08
+syslog09 syslog09
+syslog10 syslog10
+syslog11 syslog11
+syslog12 syslog12
+times03 times03
 timerfd04 timerfd04
+timerfd_settime02 timerfd_settime02
 perf_event_open02 perf_event_open02
 statx07 statx07
 io_uring02 io_uring02
+ioctl_loop05 ioctl_loop05
+# all local filesystems
+chdir01 chdir01
+copy_file_range01 copy_file_range01
+fallocate04 fallocate04
+fallocate05 fallocate05
+fdatasync03 fdatasync03
+fgetxattr01 fgetxattr01
+fremovexattr01 fremovexattr01
+fremovexattr02 fremovexattr02
+fsconfig01 fsconfig01
+fsetxattr01 fsetxattr01
+fsmount01 fsmount01
+fsmount02 fsmount02
+fsopen01 fsopen01
+fspick01 fspick01
+fspick02 fspick02
+fsync01 fsync01
+fsync04 fsync04
+lremovexattr01 lremovexattr01
+move_mount01 move_mount01
+move_mount02 move_mount02
+msync04 msync04
+open_tree01 open_tree01
+open_tree02 open_tree02
+preadv03 preadv03
+preadv03_64 preadv03_64
+preadv203 preadv203
+preadv203_64 preadv203_64
+pwritev03 pwritev03
+pwritev03_64 pwritev03_64
+setxattr01 setxattr01
+statx04 statx04
+sync01 sync01
+sync_file_range02 sync_file_range02
+syncfs01 syncfs01
+utime03 utime03
+writev03 writev03
+# cross mount (may fail on multi-zones meta)
+inotify03 inotify03
+inotify07 inotify07
+inotify08 inotify08
+lchown03  lchown03
+linkat02 linkat02
+madvise01 madvise01
+mknod07 mknod07
+mknodat02 mknodat02
+mmap16 mmap16
+mount03 mount03
+mount05 mount05
+mount06 mount06
+open12 open12
+pivot_root01 pivot_root01
+readahead02 readahead02
+rename11 rename11
+renameat01 renameat01
+statx05 statx05
+umount01 umount01
+umount02 umount02
+umount03 umount03
+umount2_01 umount2_01
+umount2_02 umount2_02
+umount2_03 umount2_03
+utime06 utime06
 # not supported
 ioctl_loop05 ioctl_loop05
 fcntl17 fcntl17
@@ -145,4 +311,4 @@ getxattr05 getxattr05
 # not stable
 finit_module02 finit_module02
 msgstress03 msgstress03
-kill11 kill11
+kill11 kill11