diff --git a/.github/scripts/command/quota.sh b/.github/scripts/command/quota.sh
index 7913e66a9700..c227bfdfa4e0 100755
--- a/.github/scripts/command/quota.sh
+++ b/.github/scripts/command/quota.sh
@@ -62,6 +62,68 @@ test_total_inodes(){
     echo a | tee /jfs/test2001 2>error.log && echo "write should fail on out of inodes" && exit 1 || true
 }
 
+test_nested_dir(){
+    prepare_test
+    ./juicefs format $META_URL myjfs
+    ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
+    file_count=1000
+    mkdir -p /jfs/d1/{d1,d2,d3,d4,d5,d6}/{d1,d2,d3,d4,d5,d6}/{d1,d2,d3,d4,d5,d6}
+    dir_count=$(find /jfs/d1 -type d | wc -l)
+    echo "dir_count: $dir_count"
+    ./juicefs quota set $META_URL --path /d1 --inodes $((file_count+dir_count-1))
+    sleep $HEARTBEAT_INTERVAL
+    for i in $(seq 1 $file_count); do
+        subdir=$(find /jfs/d1/ -type d | shuf -n 1)
+        echo "touch $subdir/test$i" && touch $subdir/test$i
+    done
+    sleep $VOLUME_QUOTA_FLUSH_INTERVAL
+    subdir=$(find /jfs/d1/ -type d | shuf -n 1)
+    touch $subdir/test 2>error.log && echo "write should fail on out of inodes" && exit 1 || true
+    grep -i "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
+
+    ./juicefs quota set $META_URL --path /d1 --inodes $((file_count+dir_count))
+    sleep $HEARTBEAT_INTERVAL
+    subdir=$(find /jfs/d1/ -type d | shuf -n 1)
+    touch $subdir/test
+}
+
+test_remove_and_restore(){
+    prepare_test
+    ./juicefs format $META_URL myjfs
+    ./juicefs mount -d $META_URL /jfs --heartbeat $HEARTBEAT_INTERVAL
+    mkdir -p /jfs/d
+    ./juicefs quota set $META_URL --path /d --capacity 1
+    sleep $((HEARTBEAT_INTERVAL+1))
+    dd if=/dev/zero of=/jfs/d/test1 bs=1G count=1
+    sleep $DIR_QUOTA_FLUSH_INTERVAL
+    ./juicefs quota get $META_URL --path /d 2>&1 | tee quota.log
+    used=$(cat quota.log | grep "/d" | awk -F'|' '{print $5}'  | tr -d '[:space:]')
+    [[ $used != "100%" ]] && echo "used should be 100%" && exit 1 || true
+    echo a | tee -a /jfs/d/test1 2>error.log && echo "write should fail on out of space" && exit 1 || true
+    grep -i "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
+
+    echo "remove test1" && rm /jfs/d/test1 -rf
+    sleep $DIR_QUOTA_FLUSH_INTERVAL
+    ./juicefs quota get $META_URL --path /d 2>&1 | tee quota.log
+    used=$(cat quota.log | grep "/d" | awk -F'|' '{print $5}'  | tr -d '[:space:]')
+    [[ $used != "0%" ]] && echo "used should be 0%" && exit 1 || true
+
+    trash_dir=$(ls /jfs/.trash)
+    ./juicefs restore $META_URL $trash_dir --put-back
+    ./juicefs quota get $META_URL --path /d 2>&1 | tee quota.log
+    used=$(cat quota.log | grep "/d" | awk -F'|' '{print $5}'  | tr -d '[:space:]')
+    [[ $used != "100%" ]] && echo "used should be 100%" && exit 1 || true
+    sleep $HEARTBEAT_INTERVAL
+    echo a | tee -a /jfs/d/test1 2>error.log && echo "write should fail on out of space" && exit 1 || true
+    grep -i "Disk quota exceeded" error.log || (echo "grep failed" && exit 1)
+
+    echo "remove test1" && rm /jfs/d/test1 -rf
+    dd if=/dev/zero of=/jfs/d/test2 bs=1M count=1
+    trash_dir=$(ls /jfs/.trash)
+    ./juicefs restore $META_URL $trash_dir --put-back 2>&1 | tee restore.log
+    grep "disk quota exceeded" restore.log || (echo "check restore log failed" && exit 1)
+}
+
 test_dir_capacity(){
     prepare_test
     ./juicefs format $META_URL myjfs
diff --git a/.github/workflows/aitests.yml b/.github/workflows/aitests.yml
index 3f9f0ca8757f..b895a4ec225f 100644
--- a/.github/workflows/aitests.yml
+++ b/.github/workflows/aitests.yml
@@ -126,13 +126,13 @@ jobs:
           df -lh
 
       - name: log
-        if: ${{ always() }}
+        if: always()
         run: | 
           tail -300 /var/log/juicefs.log
           grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
diff --git a/.github/workflows/bash/rm_fs b/.github/workflows/bash/rm_fs
index bdda0c20386d..9ec5ee4d9511 100644
--- a/.github/workflows/bash/rm_fs
+++ b/.github/workflows/bash/rm_fs
@@ -30,8 +30,10 @@ gf29 growfiles -W gf29 -b -D 0 -r 1-4096 -R 0-33554432 -i 0 -L 60 -C 1 -u -f gfs
 gf30 growfiles -W gf30 -D 0 -b -i 0 -L 60 -u -B 1000b -e 1 -o O_RDWR,O_CREAT,O_SYNC -g 20480 -T 10 -t 20480 -f gf-sync-$$ -d $TMPDIR
 rwtest01 export LTPROOT; rwtest -N rwtest01 -c -q -i 60s  -f sync 10%25000:$TMPDIR/rw-sync-$$
 rwtest02 export LTPROOT; rwtest -N rwtest02 -c -q -i 60s  -f buffered 10%25000:$TMPDIR/rw-buffered-$$
+rwtest03 export LTPROOT; rwtest -N rwtest03 -c -q -i 60s -n 2  -f buffered -s mmread,mmwrite -m random -Dv 10%25000:$TMPDIR/mm-buff-$$
 rwtest04 export LTPROOT; rwtest -N rwtest04 -c -q -i 60s -n 2  -f sync -s mmread,mmwrite -m random -Dv 10%25000:$TMPDIR/mm-sync-$$
 rwtest05 export LTPROOT; rwtest -N rwtest05 -c -q -i 50 -T 64b 500b:$TMPDIR/rwtest01%f
 iogen01 export LTPROOT; rwtest -N iogen01 -i 120s -s read,write -Da -Dv -n 2 500b:$TMPDIR/doio.f1.$$ 1000b:$TMPDIR/doio.f2.$$
 quota_remount_test01 quota_remount_test01.sh
 isofs isofs.sh
+fs_fill fs_fill
\ No newline at end of file
diff --git a/.github/workflows/bash/rm_syscalls b/.github/workflows/bash/rm_syscalls
index 6723b32f5315..214784451c6f 100644
--- a/.github/workflows/bash/rm_syscalls
+++ b/.github/workflows/bash/rm_syscalls
@@ -1,3 +1,14 @@
+alarm02 alarm02
+alarm03 alarm03
+alarm05 alarm05
+alarm06 alarm06
+alarm07 alarm07
+bind01 bind01
+bind02 bind02
+bind03 bind03
+bind04 bind04
+bind05 bind05
+bind06 bind06
 bpf_prog05 bpf_prog05
 cacheflush01 cacheflush01
 chown01_16 chown01_16
@@ -5,12 +16,43 @@ chown02_16 chown02_16
 chown03_16 chown03_16
 chown04_16 chown04_16
 chown05_16 chown05_16
+clock_adjtime01 clock_adjtime01
+clock_adjtime02 clock_adjtime02
+clock_getres01 clock_getres01
+clock_nanosleep01 clock_nanosleep01
+clock_nanosleep02 clock_nanosleep02
 clock_nanosleep03 clock_nanosleep03
+clock_nanosleep04 clock_nanosleep04
+clock_gettime01 clock_gettime01
+clock_gettime02 clock_gettime02
 clock_gettime03 clock_gettime03
+clock_gettime04 clock_gettime04
 leapsec01 leapsec01
+clock_settime01 clock_settime01
+clock_settime02 clock_settime02
+clock_settime03 clock_settime03
 close_range01 close_range01
 close_range02 close_range02
 fallocate06 fallocate06
+fanotify01 fanotify01
+fanotify02 fanotify02
+fanotify03 fanotify03
+fanotify04 fanotify04
+fanotify05 fanotify05
+fanotify06 fanotify06
+fanotify07 fanotify07
+fanotify08 fanotify08
+fanotify09 fanotify09
+fanotify10 fanotify10
+fanotify11 fanotify11
+fanotify12 fanotify12
+fanotify13 fanotify13
+fanotify14 fanotify14
+fanotify15 fanotify15
+fanotify16 fanotify16
+fanotify17 fanotify17
+fanotify18 fanotify18
+fanotify19 fanotify19
 fchown01_16 fchown01_16
 fchown02_16 fchown02_16
 fchown03_16 fchown03_16
@@ -18,6 +60,19 @@ fchown04_16 fchown04_16
 fchown05_16 fchown05_16
 fcntl06 fcntl06
 fcntl06_64 fcntl06_64
+fork01 fork01
+fork02 fork02
+fork03 fork03
+fork04 fork04
+fork05 fork05
+fork06 fork06
+fork07 fork07
+fork08 fork08
+fork09 fork09
+fork10 fork10
+fork11 fork11
+fork13 fork13 -i 1000000
+fork14 fork14
 getegid01_16 getegid01_16
 getegid02_16 getegid02_16
 geteuid01_16 geteuid01_16
@@ -33,12 +88,26 @@ getresuid01_16 getresuid01_16
 getresuid02_16 getresuid02_16
 getresuid03_16 getresuid03_16
 getrusage04 getrusage04
+gettimeofday01 gettimeofday01
+gettimeofday02 gettimeofday02
 getuid01_16 getuid01_16
 getuid03_16 getuid03_16
+ioctl03      ioctl03
 ioctl_sg01 ioctl_sg01
 fanotify16 fanotify16
 fanotify18 fanotify18
 fanotify19 fanotify19
+kill02 kill02
+kill03 kill03
+kill05 kill05
+kill06 kill06
+kill07 kill07
+kill08 kill08
+kill09 kill09
+kill10 kill10
+kill11 kill11
+kill12 kill12
+kill13 kill13
 lchown01_16 lchown01_16
 lchown02_16 lchown02_16
 lchown03_16 lchown03_16
@@ -63,20 +132,36 @@ move_pages11 move_pages11
 move_pages12 move_pages12
 msgctl05 msgctl05
 msgstress04 msgstress04
+nanosleep01 nanosleep01
+nanosleep02 nanosleep02
+nanosleep04 nanosleep04
 openat201 openat201
 openat202 openat202
 openat203 openat203
 madvise06 madvise06
 madvise09 madvise09
+pselect01 pselect01
+pselect01_64 pselect01_64
 ptrace04 ptrace04
 quotactl01 quotactl01
 quotactl04 quotactl04
 quotactl06 quotactl06
 readdir21 readdir21
 recvmsg03 recvmsg03
+rt_sigaction01 rt_sigaction01
+rt_sigaction02 rt_sigaction02
+rt_sigaction03 rt_sigaction03
+rt_sigprocmask01 rt_sigprocmask01
+rt_sigprocmask02 rt_sigprocmask02
+rt_sigqueueinfo01 rt_sigqueueinfo01
+rt_sigsuspend01 rt_sigsuspend01
+rt_sigtimedwait01 rt_sigtimedwait01
+rt_tgsigqueueinfo01 rt_tgsigqueueinfo01
 sbrk03 sbrk03
+select02 select02
 semctl08 semctl08
 semctl09 semctl09
+sendfile09_64 sendfile09_64
 set_mempolicy01 set_mempolicy01
 set_mempolicy02 set_mempolicy02
 set_mempolicy03 set_mempolicy03
@@ -117,9 +202,12 @@ setreuid04_16 setreuid04_16
 setreuid05_16 setreuid05_16
 setreuid06_16 setreuid06_16
 setreuid07_16 setreuid07_16
+setsockopt06 setsockopt06
+setsockopt07 setsockopt07
 setuid01_16 setuid01_16
 setuid03_16 setuid03_16
 setuid04_16 setuid04_16
+shmctl05 shmctl05
 shmctl06 shmctl06
 socketcall01 socketcall01
 socketcall02 socketcall02
@@ -132,10 +220,88 @@ swapon02 swapon02
 swapon03 swapon03
 switch01 endian_switch01
 sysinfo03 sysinfo03
+syslog01 syslog01
+syslog02 syslog02
+syslog03 syslog03
+syslog04 syslog04
+syslog05 syslog05
+syslog06 syslog06
+syslog07 syslog07
+syslog08 syslog08
+syslog09 syslog09
+syslog10 syslog10
+syslog11 syslog11
+syslog12 syslog12
+times03 times03
 timerfd04 timerfd04
+timerfd_settime02 timerfd_settime02
 perf_event_open02 perf_event_open02
 statx07 statx07
 io_uring02 io_uring02
+ioctl_loop05 ioctl_loop05
+# all local filesystems
+chdir01 chdir01
+copy_file_range01 copy_file_range01
+fallocate04 fallocate04
+fallocate05 fallocate05
+fdatasync03 fdatasync03
+fgetxattr01 fgetxattr01
+fremovexattr01 fremovexattr01
+fremovexattr02 fremovexattr02
+fsconfig01 fsconfig01
+fsetxattr01 fsetxattr01
+fsmount01 fsmount01
+fsmount02 fsmount02
+fsopen01 fsopen01
+fspick01 fspick01
+fspick02 fspick02
+fsync01 fsync01
+fsync04 fsync04
+lremovexattr01 lremovexattr01
+move_mount01 move_mount01
+move_mount02 move_mount02
+msync04 msync04
+open_tree01 open_tree01
+open_tree02 open_tree02
+preadv03 preadv03
+preadv03_64 preadv03_64
+preadv203 preadv203
+preadv203_64 preadv203_64
+pwritev03 pwritev03
+pwritev03_64 pwritev03_64
+setxattr01 setxattr01
+statx04 statx04
+sync01 sync01
+sync_file_range02 sync_file_range02
+syncfs01 syncfs01
+utime03 utime03
+writev03 writev03
+# cross mount (may fail on multi-zones meta)
+inotify03 inotify03
+inotify07 inotify07
+inotify08 inotify08
+lchown03  lchown03
+linkat02 linkat02
+madvise01 madvise01
+mknod07 mknod07
+mknodat02 mknodat02
+mmap16 mmap16
+mount03 mount03
+mount05 mount05
+mount06 mount06
+open12 open12
+pivot_root01 pivot_root01
+readahead02 readahead02
+rename11 rename11
+renameat01 renameat01
+statx05 statx05
+umount01 umount01
+umount02 umount02
+umount03 umount03
+umount2_01 umount2_01
+umount2_02 umount2_02
+umount2_03 umount2_03
+utime06 utime06
 # not supported
 ioctl_loop05 ioctl_loop05
 fcntl17 fcntl17
@@ -145,4 +311,4 @@ getxattr05 getxattr05
 # not stable
 finit_module02 finit_module02
 msgstress03 msgstress03
-kill11 kill11
+kill11 kill11
\ No newline at end of file
diff --git a/.github/workflows/chaos.yml b/.github/workflows/chaos.yml
index 6cd4db13cf7c..4b97f4697649 100644
--- a/.github/workflows/chaos.yml
+++ b/.github/workflows/chaos.yml
@@ -153,14 +153,14 @@ jobs:
           done
 
       - name: Check csi controller log
-        if: ${{ always() }}
+        if: always()
         run: |
           kubectl describe pvc dynamic-ce
           kubectl -n kube-system get po -l app=juicefs-csi-controller
           kubectl -n kube-system logs juicefs-csi-controller-0 juicefs-plugin
 
       - name: Check csi node log
-        if: ${{ always() }}
+        if: always()
         run: |
           POD_NAME=$(kubectl get pods -n kube-system -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}'  | grep juicefs-csi-node)
           echo POD_NAME is $POD_NAME
@@ -170,7 +170,7 @@ jobs:
           # grep -i "error" csi_node.log && exit 1 || true
 
       - name: Check mount point pod
-        if: ${{ always() }}
+        if: always()
         run: | 
           POD_NAME=$(kubectl get pods -n kube-system -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}'  | grep juicefs-chart-testing-control-plane-pvc)
           echo POD_NAME is $POD_NAME
@@ -180,7 +180,7 @@ jobs:
           grep "<FATAL>:" juicefs.log && exit 1 || true
 
       - name: Check vdbench log
-        if: ${{ always() }}
+        if: always()
         run: | 
           POD_NAME=$(kubectl get pods -n default -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}'  | grep dynamic-ce )
           echo POD_NAME is $POD_NAME
@@ -190,7 +190,7 @@ jobs:
           # grep -i "error" vdbench.log && exit 1 || true
 
       - name: Check Redis log
-        if: ${{ always() }}
+        if: always()
         run: | 
           POD_NAME=$(kubectl get pods -n kube-system -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}'  | grep redis )
           echo POD_NAME is $POD_NAME
@@ -200,7 +200,7 @@ jobs:
           # grep -i "error" redis.log && exit 1 || true
 
       - name: Check Minio log
-        if: ${{ always() }}
+        if: always()
         run: | 
           POD_NAME=$(kubectl get pods -n kube-system -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}'  | grep minio )
           echo POD_NAME is $POD_NAME
@@ -210,7 +210,7 @@ jobs:
           # grep -i "error" minio.log && exit 1 || true
 
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         # if: failure()
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
@@ -228,12 +228,12 @@ jobs:
         run: exit 1
 
       - name: Send Slack Notification
-        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        if: failure() && github.event_name != 'workflow_dispatch' 
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
 
       - name: Success
-        if: ${{ success() }}
+        if: success() 
         run: echo "All Done"
\ No newline at end of file
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 66f54dc2e3f6..b033518d660e 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -88,7 +88,7 @@ jobs:
       uses: github/codeql-action/analyze@v2
 
     - name: Send Slack Notification
-      if: ${{ failure() }}
+      if: failure()
       uses: juicedata/slack-notify-action@main
       with:
         channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
diff --git a/.github/workflows/command.yml b/.github/workflows/command.yml
index 216f7cf0dc47..dd4b9b0b96e6 100644
--- a/.github/workflows/command.yml
+++ b/.github/workflows/command.yml
@@ -103,14 +103,14 @@ jobs:
           sudo META=${{matrix.meta}} .github/scripts/command/info.sh
 
       - name: Log
-        if: ${{ always() }}
+        if: always()
         run: |
           echo "juicefs log"
           sudo tail -n 1000 /var/log/juicefs.log
           grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
           
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         # if: failure()
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
@@ -137,14 +137,14 @@ jobs:
           sudo META=${{matrix.meta}} .github/scripts/command/quota.sh 
 
       - name: Log
-        if: ${{ always() }}
+        if: always()
         run: |
           echo "juicefs log"
           sudo tail -n 1000 /var/log/juicefs.log
           grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
           
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         # if: failure()
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
@@ -190,14 +190,14 @@ jobs:
           sudo CI=true META=${{matrix.meta}} KEY_TYPE=$random_type .github/scripts/command/sync_cluster.sh 
 
       - name: Log
-        if: ${{ always() }}
+        if: always()
         run: |
           echo "juicefs log"
           sudo tail -n 1000 /var/log/juicefs.log
           grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
           
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         # if: failure()
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
@@ -216,12 +216,12 @@ jobs:
         run: exit 1
 
       - name: Send Slack Notification
-        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        if: failure() && github.event_name != 'workflow_dispatch'
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
 
       - name: Success
-        if: ${{ success() }}
+        if: success()
         run: echo "All Done"
diff --git a/.github/workflows/elastictest.yml b/.github/workflows/elastictest.yml
index f765e1edd387..dddf6cb21573 100644
--- a/.github/workflows/elastictest.yml
+++ b/.github/workflows/elastictest.yml
@@ -83,20 +83,20 @@ jobs:
           cat /tmp/report.md
 
       - name: Log
-        if: ${{ failure() }}
+        if: failure()
         run: |
           echo "juicefs log"
           sudo tail -n 1000 /var/log/juicefs.log
           grep "<FATAL>:" /var/log/juicefs.log && exit 1
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"  
 
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug }}
+        if: failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
\ No newline at end of file
diff --git a/.github/workflows/encoding_mysql.yml b/.github/workflows/encoding_mysql.yml
index fc757969a814..b321d04e0137 100644
--- a/.github/workflows/encoding_mysql.yml
+++ b/.github/workflows/encoding_mysql.yml
@@ -284,13 +284,13 @@ jobs:
           ./juicefs umount ${{steps.vars.outputs.MOUNT_POINT2}} || echo $?
           
       - name: log
-        if: ${{ always() }}
+        if: always()
         run: | 
           tail -300 ~/.juicefs/juicefs.log
           grep "<FATAL>:" ~/.juicefs/juicefs.log && exit 1 || true
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure() 
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
diff --git a/.github/workflows/filebench.yml b/.github/workflows/filebench.yml
index 0ea5ec178060..76a5bcb7b51c 100644
--- a/.github/workflows/filebench.yml
+++ b/.github/workflows/filebench.yml
@@ -96,7 +96,7 @@ jobs:
           fi
 
       - name: Log
-        if: ${{ always() }}
+        if: always()
         run: |
           df -lh
           echo "juicefs log"
diff --git a/.github/workflows/fio_benchmark.yml b/.github/workflows/fio_benchmark.yml
index 727fae21bb1f..8d215429aa7c 100644
--- a/.github/workflows/fio_benchmark.yml
+++ b/.github/workflows/fio_benchmark.yml
@@ -184,7 +184,7 @@ jobs:
         shell: bash
 
       - name: log
-        if: ${{ always() }}
+        if: always()
         shell: bash
         run: | 
           if [ -f ~/.juicefs/juicefs.log ]; then
@@ -205,12 +205,12 @@ jobs:
         run: exit 1
 
       - name: Send Slack Notification
-        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        if: failure() && github.event_name != 'workflow_dispatch'
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
 
       - name: Success
-        if: ${{ success() }}
+        if: success()
         run: echo "All Done"
\ No newline at end of file
diff --git a/.github/workflows/fsrand.yml b/.github/workflows/fsrand.yml
index 7b2402c96768..87fd460d1ff8 100644
--- a/.github/workflows/fsrand.yml
+++ b/.github/workflows/fsrand.yml
@@ -117,7 +117,7 @@ jobs:
           ls -l /tmp/jfs/fsrand && exit 1 || true
 
       - name: log
-        if: ${{ always() }}
+        if: always()
         shell: bash
         run: | 
           if [ -f ~/.juicefs/juicefs.log ]; then
@@ -130,13 +130,13 @@ jobs:
           fi
 
       - name: Send Slack Notification
-        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        if: failure() && github.event_name != 'workflow_dispatch'
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"  
           
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
diff --git a/.github/workflows/gateway.yml b/.github/workflows/gateway.yml
index b439e550ce85..7e09e85c1d9f 100644
--- a/.github/workflows/gateway.yml
+++ b/.github/workflows/gateway.yml
@@ -184,7 +184,7 @@ jobs:
         shell: bash
 
       - name: log
-        if: ${{ always() }}
+        if: always()
         shell: bash
         run: | 
           if [ -f ~/.juicefs/juicefs.log ]; then
@@ -193,7 +193,7 @@ jobs:
           fi
 
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         timeout-minutes: 1
         uses: lhotari/action-upterm@v1
 
@@ -210,12 +210,12 @@ jobs:
         run: exit 1
 
       - name: Send Slack Notification
-        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        if: failure() && github.event_name != 'workflow_dispatch'
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
 
       - name: Success
-        if: ${{ success() }}
+        if: success()
         run: echo "All Done"
diff --git a/.github/workflows/integrationtests.yml b/.github/workflows/integrationtests.yml
index 6bf8afbf0ad9..ac516a2f0526 100644
--- a/.github/workflows/integrationtests.yml
+++ b/.github/workflows/integrationtests.yml
@@ -95,13 +95,13 @@ jobs:
 
 
       - name: log
-        if: ${{ always() }}
+        if: always()
         run: |
           tail -300 /var/log/juicefs.log
           grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
@@ -109,6 +109,6 @@ jobs:
 
 
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
diff --git a/.github/workflows/load.yml b/.github/workflows/load.yml
index ab78304e3801..872f1aa3ad8b 100644
--- a/.github/workflows/load.yml
+++ b/.github/workflows/load.yml
@@ -105,7 +105,7 @@ jobs:
           sudo MYSQL_PASSWORD=${{secrets.MYSQL_PASSWORD_FOR_JUICEDATA}} META=${{matrix.meta}} START_META=false .github/scripts/command/load_dump_bench.sh test_list_with_big_dir
 
       - name: log
-        if: ${{ always() }}
+        if: always()
         shell: bash
         run: | 
           if [ -f ~/.juicefs/juicefs.log ]; then
@@ -114,7 +114,7 @@ jobs:
           fi
           
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         # if: failure()
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
@@ -132,13 +132,13 @@ jobs:
         run: exit 1
 
       - name: Send Slack Notification
-        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        if: failure() && github.event_name != 'workflow_dispatch' 
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
 
       - name: Success
-        if: ${{ success() }}
+        if: success()
         run: echo "All Done"
         
\ No newline at end of file
diff --git a/.github/workflows/ltpfs.yml b/.github/workflows/ltpfs.yml
index 0a6032e8b8bd..93dfcc76fd3a 100644
--- a/.github/workflows/ltpfs.yml
+++ b/.github/workflows/ltpfs.yml
@@ -63,20 +63,20 @@ jobs:
           sudo chmod +x /tmp/rm_list.sh
           sudo chmod 777 runtest/fs
           sudo /tmp/rm_list.sh /tmp/rm_fs /opt/ltp/runtest/fs
-          sudo ./runltp -d /tmp/jfs -f fs,fs_bind,fs_perms_simple,fsx,io,smoketest,fcntl-locktests -C result.log.failed -T result.log.tconf -l result.log  
+          sudo ./runltp -d /tmp/jfs -f fs,fs_perms_simple,fsx,io,fcntl-locktests -C result.log.failed -T result.log.tconf -l result.log
 
       - name: tconf Log
-        if: ${{ always() }}
+        if: always()
         run: |
           cat /opt/ltp/output/result.log.tconf
 
       - name: check ltpsyscall failed log
-        if: ${{ always() }}
+        if: always()
         run: |
           cat /opt/ltp/output/result.log.failed
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
diff --git a/.github/workflows/ltpsyscalls.yml b/.github/workflows/ltpsyscalls.yml
index 6ecedfb06385..836debe460b2 100644
--- a/.github/workflows/ltpsyscalls.yml
+++ b/.github/workflows/ltpsyscalls.yml
@@ -112,23 +112,23 @@ jobs:
           fi
 
       - name: tconf Log
-        if: ${{ always() }}
+        if: always()
         run: |
           cat /opt/ltp/output/result.log.tconf
 
       - name: check ltpsyscall failed log
-        if: ${{ always() }}
+        if: always()
         run: |
           cat /opt/ltp/output/result.log.failed
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"  
 
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
diff --git a/.github/workflows/misc.yml b/.github/workflows/misc.yml
index 8603cb5446ae..69db67430a16 100644
--- a/.github/workflows/misc.yml
+++ b/.github/workflows/misc.yml
@@ -120,13 +120,13 @@ jobs:
           $juicefs fsck $meta
 
       - name: log
-        if: ${{ always() }}
+        if: always()
         run: | 
           tail -300 ~/.juicefs/juicefs.log
           grep -v "UUID changed"  ~/.juicefs/juicefs.log | grep "<FATAL>:"  && exit 1 || true
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
diff --git a/.github/workflows/mongodb_mmap.yml b/.github/workflows/mongodb_mmap.yml
index 1f38f402c6fd..77eaf38f4b39 100644
--- a/.github/workflows/mongodb_mmap.yml
+++ b/.github/workflows/mongodb_mmap.yml
@@ -78,21 +78,21 @@ jobs:
           time ./bin/ycsb run mongodb -s -P workloads/workloadf -threads 10 > outputRun.txt
 
       - name: Log
-        if: ${{ always() }}
+        if: always()
         run: |
           echo "juicefs log"
           sudo tail -n 1000 /var/log/juicefs.log
           grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"  
 
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         # if: failure()
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
\ No newline at end of file
diff --git a/.github/workflows/mutate-test-sdk.yml b/.github/workflows/mutate-test-sdk.yml
index 2dee08d423c2..ff16ed3d2771 100644
--- a/.github/workflows/mutate-test-sdk.yml
+++ b/.github/workflows/mutate-test-sdk.yml
@@ -98,13 +98,13 @@ jobs:
           path: sdk/java/target/pit-reports
 
       - name: Send Slack Notification
-        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        if: failure() && github.event_name != 'workflow_dispatch'
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
 
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
\ No newline at end of file
diff --git a/.github/workflows/mutate-test.yml b/.github/workflows/mutate-test.yml
index 0492efc3f9ce..be52170d4148 100644
--- a/.github/workflows/mutate-test.yml
+++ b/.github/workflows/mutate-test.yml
@@ -197,7 +197,7 @@ jobs:
           JOB_NAME=${{matrix.test_file}} JOB_URL=${{steps.jobs.outputs.html_url}}  STAT_RESULT_FILE=stat_result.log python3 .github/scripts/mutate/save_report.py
 
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
 
@@ -242,13 +242,13 @@ jobs:
             Usage: https://github.com/juicedata/juicefs/blob/main/.github/scripts/mutate/how_to_use_mutate_test.md  
 
       - name: Send Slack Notification
-        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        if: failure() && github.event_name != 'workflow_dispatch'
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
 
       - name: Success
-        if: ${{ success() }}
+        if: success()
         run: echo "All Done"
 
diff --git a/.github/workflows/mysqltest_bigtable.yml b/.github/workflows/mysqltest_bigtable.yml
index a7c15998de40..e9ec0971cb7f 100644
--- a/.github/workflows/mysqltest_bigtable.yml
+++ b/.github/workflows/mysqltest_bigtable.yml
@@ -101,14 +101,14 @@ jobs:
           done
 
       - name: Log
-        if: ${{ always() }}
+        if: always()
         run: |
           echo "juicefs log"
           sudo tail -n 1000 /var/log/juicefs.log
           grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
diff --git a/.github/workflows/permission-check.yaml b/.github/workflows/permission-check.yaml
index 92f063bf8870..29acb9233c54 100644
--- a/.github/workflows/permission-check.yaml
+++ b/.github/workflows/permission-check.yaml
@@ -92,7 +92,7 @@ jobs:
           sudo prove -rv tests/
 
       - name: log
-        if: ${{ always() }}
+        if: always()
         shell: bash
         run: | 
           if [ -f ~/.juicefs/juicefs.log ]; then
@@ -105,14 +105,14 @@ jobs:
           fi
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"  
 
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
 
@@ -129,12 +129,12 @@ jobs:
         run: exit 1
 
       - name: Send Slack Notification
-        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        if: failure() && github.event_name != 'workflow_dispatch'
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
 
       - name: Success
-        if: ${{ success() }}
+        if: success()
         run: echo "All Done"
\ No newline at end of file
diff --git a/.github/workflows/pjdfstest.yml b/.github/workflows/pjdfstest.yml
index 71a1387ae058..627c8889a24a 100644
--- a/.github/workflows/pjdfstest.yml
+++ b/.github/workflows/pjdfstest.yml
@@ -119,7 +119,7 @@ jobs:
           sudo prove -rv tests/
 
       - name: log
-        if: ${{ always() }}
+        if: always()
         shell: bash
         run: | 
           if [ -f ~/.juicefs/juicefs.log ]; then
@@ -132,14 +132,14 @@ jobs:
           fi
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"  
 
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
 
@@ -156,12 +156,12 @@ jobs:
         run: exit 1
 
       - name: Send Slack Notification
-        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        if: failure() && github.event_name != 'workflow_dispatch'
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
 
       - name: Success
-        if: ${{ success() }}
+        if: success() 
         run: echo "All Done"
\ No newline at end of file
diff --git a/.github/workflows/redis_compile.yml b/.github/workflows/redis_compile.yml
index c9c7a87d639d..eb058ac49858 100644
--- a/.github/workflows/redis_compile.yml
+++ b/.github/workflows/redis_compile.yml
@@ -64,14 +64,14 @@ jobs:
 
 
       - name: Log
-        if: ${{ always() }}
+        if: always()
         run: |
           echo "juicefs log"
           sudo tail -n 1000 /var/log/juicefs.log
           grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
diff --git a/.github/workflows/rmfiles.yml b/.github/workflows/rmfiles.yml
index 695014c04bfa..5e21dee7dd8b 100644
--- a/.github/workflows/rmfiles.yml
+++ b/.github/workflows/rmfiles.yml
@@ -104,11 +104,16 @@ jobs:
           ls -l $mp/test && exit 1 || true
 
       - name: log
-        if: ${{ always() }}
+        if: always()
         shell: bash
         run: | 
           tail -300 ~/.juicefs/juicefs.log
           grep "<FATAL>:" ~/.juicefs/juicefs.log && exit 1 || true
+          
+      - name: Setup upterm session
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
+        timeout-minutes: 60
+        uses: lhotari/action-upterm@v1
 
   success-all-test:
     runs-on: ubuntu-latest
@@ -123,12 +128,12 @@ jobs:
         run: exit 1
 
       - name: Send Slack Notification
-        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        if: failure() && github.event_name != 'workflow_dispatch'
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
 
       - name: Success
-        if: ${{ success() }}
+        if: success()
         run: echo "All Done"
\ No newline at end of file
diff --git a/.github/workflows/sdktest.yml b/.github/workflows/sdktest.yml
index a1d36054f717..8886f75e62e0 100644
--- a/.github/workflows/sdktest.yml
+++ b/.github/workflows/sdktest.yml
@@ -72,13 +72,13 @@ jobs:
           sudo mvn package -B -Dmaven.test.skip=true --quiet -Dmaven.javadoc.skip=true
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"  
 
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
diff --git a/.github/workflows/storage.yml b/.github/workflows/storage.yml
index 2b9a61152fa3..7f362847a08b 100644
--- a/.github/workflows/storage.yml
+++ b/.github/workflows/storage.yml
@@ -66,7 +66,7 @@ jobs:
           sudo STORAGE=${{matrix.storage}} .github/scripts/storage.sh
 
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         # if: failure()
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
@@ -84,12 +84,12 @@ jobs:
         run: exit 1
 
       - name: Send Slack Notification
-        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        if: failure() && github.event_name != 'workflow_dispatch'
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
 
       - name: Success
-        if: ${{ success() }}
+        if: success()
         run: echo "All Done"
\ No newline at end of file
diff --git a/.github/workflows/sysbench.yml b/.github/workflows/sysbench.yml
index 42c67d5c6c45..d45bbaeed257 100644
--- a/.github/workflows/sysbench.yml
+++ b/.github/workflows/sysbench.yml
@@ -154,7 +154,7 @@ jobs:
           compress: ${{matrix.compress}}
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
diff --git a/.github/workflows/tpcds.yml b/.github/workflows/tpcds.yml
index f7d5b20192f1..f5b2dcf0b3c7 100644
--- a/.github/workflows/tpcds.yml
+++ b/.github/workflows/tpcds.yml
@@ -159,7 +159,7 @@ jobs:
           -i ~/work/juicefs/juicefs/.github/workflows/resources/tpcds_run.scala
           
       - name: Log
-        if: ${{ always() }}
+        if: always()
         run: |
           if [ -f /var/log/juicefs.log ]; then
             echo "juicefs log"
@@ -168,13 +168,13 @@ jobs:
           fi
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure() 
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"  
 
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
diff --git a/.github/workflows/tpch_clickhouse.yml b/.github/workflows/tpch_clickhouse.yml
index 12a3948fb76d..c85d6f755987 100644
--- a/.github/workflows/tpch_clickhouse.yml
+++ b/.github/workflows/tpch_clickhouse.yml
@@ -287,7 +287,7 @@ jobs:
 
       - name: Log
         shell: bash
-        if: ${{ always() }}
+        if: always()
         run: |
           if [ -f /var/log/juicefs.log ]; then 
             echo "juicefs log"
@@ -296,7 +296,7 @@ jobs:
           fi
 
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml
index 845f22acddc5..9bf63d016fa1 100644
--- a/.github/workflows/unittests.yml
+++ b/.github/workflows/unittests.yml
@@ -93,7 +93,7 @@ jobs:
       #     files: ./cov.out
 
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         timeout-minutes: 120
         uses: lhotari/action-upterm@v1
 
@@ -111,13 +111,13 @@ jobs:
         run: exit 1
 
       - name: Send Slack Notification
-        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        if: failure() && github.event_name != 'workflow_dispatch' 
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
 
       - name: Success
-        if: ${{ success() }}
+        if: success()
         run: echo "All Done"
 
diff --git a/.github/workflows/vdbench.yml b/.github/workflows/vdbench.yml
index f309207b3724..21e576424d84 100644
--- a/.github/workflows/vdbench.yml
+++ b/.github/workflows/vdbench.yml
@@ -99,14 +99,14 @@ jobs:
           path: output
 
       - name: log
-        if: ${{ always() }}
+        if: always()
         shell: bash
         run: | 
           tail -300 ~/.juicefs/juicefs.log
           grep "<FATAL>:" ~/.juicefs/juicefs.log && exit 1 || true
           
       - name: Send Slack Notification
-        if: ${{ failure() }}
+        if: failure()
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml
index 342de031cb3f..f83418cd0e53 100644
--- a/.github/workflows/verify.yml
+++ b/.github/workflows/verify.yml
@@ -91,7 +91,7 @@ jobs:
         ./juicefs.fdb version
 
     - name: Setup upterm session
-      if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
+      if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
       # if: failure()
       timeout-minutes: 60
       uses: lhotari/action-upterm@v1
\ No newline at end of file
diff --git a/.github/workflows/version_compatible_hypo.yml b/.github/workflows/version_compatible_hypo.yml
index 9307278b9e28..97d81c4cb2d4 100644
--- a/.github/workflows/version_compatible_hypo.yml
+++ b/.github/workflows/version_compatible_hypo.yml
@@ -117,14 +117,14 @@ jobs:
           timeout 3600 python3 .github/scripts/testVersionCompatible.py || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then echo test failed; exit $code; fi
 
       - name: Upload command log
-        if: ${{ always() }}
+        if: always()
         uses: actions/upload-artifact@v3
         with:
           name: ${{matrix.meta}}-${{matrix.old_juicefs_version}}.command.log
           path: ~/command.log
 
       - name: Display log
-        if: ${{ always() }}
+        if: always()
         shell: bash
         run: | 
           if [ -f "/home/runner/.juicefs/juicefs.log" ]; then
@@ -133,7 +133,7 @@ jobs:
           fi
 
       - name: Display command
-        if: ${{ always() }}
+        if: always()
         shell: bash
         run: | 
           if [ -f "$HOME/command.log" ]; then
@@ -141,7 +141,7 @@ jobs:
           fi
 
       - name: Setup upterm session
-        if: ${{ failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug == 'true' }}
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
         timeout-minutes: 60
         uses: lhotari/action-upterm@v1
 
@@ -159,12 +159,12 @@ jobs:
         run: exit 1
 
       - name: Send Slack Notification
-        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        if: failure() && github.event_name != 'workflow_dispatch'
         uses: juicedata/slack-notify-action@main
         with:
           channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
           slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"
 
       - name: Success
-        if: ${{ success() }}
+        if: success()
         run: echo "All Done"
\ No newline at end of file
diff --git a/docs/en/reference/how_to_set_up_metadata_engine.md b/docs/en/reference/how_to_set_up_metadata_engine.md
index c01a0f56add4..575bf0d56b70 100644
--- a/docs/en/reference/how_to_set_up_metadata_engine.md
+++ b/docs/en/reference/how_to_set_up_metadata_engine.md
@@ -125,9 +125,10 @@ juicefs format --storage s3 \
 
 In the code mentioned above, we use the `rediss://` protocol header to enable mTLS functionality, and then use the following options to specify the path of the client certificate:
 
-- `tls-cert-file`: The path of the client certificate.
-- `tls-key-file`: The path of the private key.
-- `tls-ca-cert-file`: The path of the CA certificate. It is optional. If it is not specified, the system CA certificate will be used.
+- `tls-cert-file=<path>`: The path of the client certificate.
+- `tls-key-file=<path>`: The path of the private key.
+- `tls-ca-cert-file=<path>`: The path of the CA certificate. It is optional. If it is not specified, the system CA certificate will be used.
+- `insecure-skip-verify=true` It can skip verifying the server certificate.
 
 When specifying options in a URL, start with the `?` symbol and use the `&` symbol to separate multiple options, for example: `?tls-cert-file=client.crt&tls-key-file=client.key`.
 
diff --git a/docs/en/security/trash.md b/docs/en/security/trash.md
index 36a4f7534bc7..b842c692520c 100644
--- a/docs/en/security/trash.md
+++ b/docs/en/security/trash.md
@@ -24,6 +24,8 @@ juicefs config META-URL --trash-days=7
 juicefs config META-URL --trash-days=0
 ```
 
+In addition, the automatic cleaning of the trash relies on the background job of the JuiceFS client. To ensure that the background job can be executed properly, at least one online mount point is required, and the [`--no-bgjob`](../reference/command_reference.md#mount) parameter should not be used when mounting the file system.
+
 ## Recover files {#recover}
 
 When files are deleted, they will be moved to a directory that takes up the format of `.trash/YYYY-MM-DD-HH/[parent inode]-[file inode]-[file name]`, where `YYYY-MM-DD-HH` is the UTC time of the deletion. You can locate the deleted files and recover them if you remember when they are deleted.
@@ -103,9 +105,17 @@ juicefs restore $META_URL 2023-08-14-05 --put-back
 
 ## Permanently delete files {#purge}
 
-The Trash directory behaves the same as normal directories, in the sense that recovering files is to simply run `mv` commands, hence to permanently delete files, run `rm` with root. But notice that **even if Trash files reach their expiration, they are not necessarily immediately deleted from Trash**, because expired files are deleted within the client background jobs, which by default runs every hour.
+When files in the trash directory reach their expiration time, they will be automatically cleaned up. It is important to note that the file cleaning is performed by the background job of the JuiceFS client, which is scheduled to run every hour by default. Therefore, when there are a large number of expired files, the cleaning speed of the object storage may not be as fast as expected, and it may take some time to see the change in storage capacity.
+
+If you want to permanently delete files before their expiration time, you need to have `root` privileges and use [`juicefs rmr`](../reference/command_reference.md#rmr) or the system's built-in `rm` command to delete the files in the `.trash` directory, so that storage space can be immediately released.
+
+For example, to permanently delete a directory in the trash:
+
+```shell
+juicefs rmr .trash/2022-11-30-10/
+```
 
-In order for expiration to work properly, at least one mount point is required, and it must be able to run background jobs (not using [`--no-bgjob`](../reference/command_reference.md#mount)). When files are permanently deleted from Trash, metadata as well as object storage data are deleted synchronously, on top of that, scans for expiration are executed every hour, that's why when expiring or deleting a large amount of files, object storage deletion may not be as rapid as expected. If you would like to speed up this process, you can create multiple mount points to overcome the speed limit of a single client, and then under their `.trash` directories, manually run `juicefs rmr` to purge files in batches.
+If you want to delete expired files more quickly, you can mount multiple mount points to exceed the deletion speed limit of a single client.
 
 ## Trash and slices {#gc}
 
diff --git a/docs/zh_cn/reference/how_to_set_up_metadata_engine.md b/docs/zh_cn/reference/how_to_set_up_metadata_engine.md
index 6c13b68fa48c..01054af1a27f 100644
--- a/docs/zh_cn/reference/how_to_set_up_metadata_engine.md
+++ b/docs/zh_cn/reference/how_to_set_up_metadata_engine.md
@@ -125,9 +125,10 @@ juicefs format --storage s3 \
 
 上面的示例代码使用 `rediss://` 协议头来开启 mTLS 功能，然后使用以下选项来指定客户端证书的路径：
 
-- `tls-cert-file` 指定客户端证书的路径
-- `tls-key-file` 指定客户端密钥的路径
-- `tls-ca-cert-file` 指定签发客户端证书的 CA 证书路径，它是可选的，如果不指定，客户端会使用系统默认的 CA 证书进行验证。
+- `tls-cert-file=<path>` 指定客户端证书的路径
+- `tls-key-file=<path>` 指定客户端密钥的路径
+- `tls-ca-cert-file=<path>` 指定签发客户端证书的 CA 证书路径，它是可选的，如果不指定，客户端会使用系统默认的 CA 证书进行验证。
+- `insecure-skip-verify=true` 可以用来跳过对服务端证书的验证
 
 在 URL 指定选项时，以 `?` 符号开头，使用 `&` 符号来分隔多个选项，例如：`?tls-cert-file=client.crt&tls-key-file=client.key`。
 
diff --git a/docs/zh_cn/security/trash.md b/docs/zh_cn/security/trash.md
index 197f951afd72..f63b4be9519c 100644
--- a/docs/zh_cn/security/trash.md
+++ b/docs/zh_cn/security/trash.md
@@ -23,6 +23,8 @@ juicefs config META-URL --trash-days=7
 juicefs config META-URL --trash-days=0
 ```
 
+另外，回收站自动清理依赖 JuiceFS 客户端的后台任务，为了保证后台任务能够正常执行，需要至少 1 个在线的挂载点，并且在挂载文件系统时不可以使用 [`--no-bgjob`](../reference/command_reference.md#mount) 参数。
+
 ## 恢复文件 {#recover}
 
 文件被删除时，会根据删除时间，被保存在格式为 `.trash/YYYY-MM-DD-HH/[parent inode]-[file inode]-[file name]` 的目录，其中 `YYYY-MM-DD-HH` 就是删除操作的 UTC 时间。因此只需要确定文件的删除时间，就能在对应的目录中找到他们，来进行恢复操作。
@@ -102,9 +104,17 @@ juicefs restore $META_URL 2023-08-14-05 --put-back
 
 ## 彻底删除文件 {#purge}
 
-回收站目录各方面表现都和普通目录一致——恢复文件就是 `mv` 出来，如果想彻底删除文件，那么 `rm` 就能做到（需要 root 权限）。但要注意，**就算回收站内的文件已经到达了过期时间，也未必会立刻从回收站消失**，这是因为过期文件的清理由 JuiceFS 客户端定期在后台任务（background job，也称 bgjob）执行，默认每小时清理一次。
+当回收站中的文件到了过期时间，会被自动清理。需要注意的是，文件清理由 JuiceFS 客户端的后台任务（background job，也称 bgjob）执行，默认每小时清理一次，因此面对大量文件过期时，对象存储的清理速度未必和你期望的一样快，可能需要一些时间才能看到存储容量变化。
+
+如果你希望在过期时间到来之前彻底删除文件，需要使用 root 用户身份，用 [`juicefs rmr`](../reference/command_reference.md#rmr) 或系统自带的 `rm` 命令来删除回收站目录 `.trash` 中的文件，这样就能立刻释放存储空间。
+
+例如，彻底删除回收站中某个目录：
+
+```shell
+juicefs rmr .trash/2022-11-30-10/
+```
 
-因此，为了让过期文件能够正常清理，需要至少 1 个在线的挂载点，并且能够正常执行后台任务（不能开启 [`--no-bgjob`](../reference/command_reference.md#mount)）。从回收站彻底删除文件时，会同步删除元数据和对象存储，再加上清理任务由客户端每小时定期执行，因此面对大量文件过期（或者强制删除）时，对象存储的清理速度未必和你期望的一样快。如果希望更快速删除过期文件，可以挂载多个挂载点来突破单个客户端的删除速度上限，然后在这些挂载点的 `.trash` 目录下，对需要删除的文件按批次手动运行 `juicefs rmr` 来强行删除。
+如果希望更快速删除过期文件，可以挂载多个挂载点来突破单个客户端的删除速度上限。
 
 ## 回收站和文件碎片 {#gc}
 
diff --git a/go.mod b/go.mod
index 370eff6ced57..daade1ce3a36 100644
--- a/go.mod
+++ b/go.mod
@@ -231,7 +231,6 @@ require (
 	github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a // indirect
 	github.com/willf/bitset v1.1.11 // indirect
 	github.com/willf/bloom v2.0.3+incompatible // indirect
-	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.9 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
 	go.opencensus.io v0.23.0 // indirect
@@ -265,4 +264,4 @@ replace github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible => g
 
 replace github.com/urfave/cli/v2 v2.19.3 => github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83
 
-replace github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b => github.com/juicedata/go-nfs-client v0.0.0-20230619072909-36eec939432b
+replace github.com/vmware/go-nfs-client v0.0.0-20190605212624-d43b92724c1b => github.com/juicedata/go-nfs-client v0.0.0-20231018052507-dbca444fa7e8
diff --git a/go.sum b/go.sum
index 5ed9e0bb5c0b..199e8d6355e6 100644
--- a/go.sum
+++ b/go.sum
@@ -622,10 +622,10 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83 h1:RyHTka3jCnTaUqfRYjlwcQlr53aasmkvHEbYLXthqr8=
 github.com/juicedata/cli/v2 v2.19.4-0.20230605075551-9c9c5c0dce83/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI=
-github.com/juicedata/go-nfs-client v0.0.0-20230619072909-36eec939432b h1:tZixumON7boPNeTPBkaD5ObOUJ8DaES4VL17XMrY0BU=
-github.com/juicedata/go-nfs-client v0.0.0-20230619072909-36eec939432b/go.mod h1:xOMqi3lOrcGe9uZLnSzgaq94Vc3oz6VPCNDLJUnXpKs=
 github.com/juicedata/go-fuse/v2 v2.1.1-0.20230726081302-124dbfa991d7 h1:4evzoVz1/AZfk9tqxWdzVYTMl2dC7VjEJHfaSFDrKS8=
 github.com/juicedata/go-fuse/v2 v2.1.1-0.20230726081302-124dbfa991d7/go.mod h1:B1nGE/6RBFyBRC1RRnf23UpwCdyJ31eukw34oAKukAc=
+github.com/juicedata/go-nfs-client v0.0.0-20231018052507-dbca444fa7e8 h1:mVVipCbohnzKZPiHGzFncLKEJZpypqjpGr4End2PP48=
+github.com/juicedata/go-nfs-client v0.0.0-20231018052507-dbca444fa7e8/go.mod h1:xOMqi3lOrcGe9uZLnSzgaq94Vc3oz6VPCNDLJUnXpKs=
 github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d h1:kpQMvNZJKGY3PTt7OSoahYc4nM0HY67SvK0YyS0GLwA=
 github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d/go.mod h1:dlxKkLh3qAIPtgr2U/RVzsZJDuXA1ffg+Njikfmhvgw=
 github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825 h1:7KrwI4HPqvNLKVcfkfDMLQQmT0GrnCs8T9EX+XCdZnM=
@@ -889,8 +889,6 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf
 github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
 github.com/pyroscope-io/client v0.7.0 h1:LWuuqPQ1oa6x7BnmUOuo/aGwdX85QGhWZUBYWWW3zdk=
 github.com/pyroscope-io/client v0.7.0/go.mod h1:4h21iOU4pUOq0prKyDlvYRL+SCKsBc5wKiEtV+rJGqU=
-github.com/pyroscope-io/godeltaprof v0.1.0 h1:UBqtjt0yZi4jTxqZmLAs34XG6ycS3vUTlhEUSq4NHLE=
-github.com/pyroscope-io/godeltaprof v0.1.0/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE=
 github.com/pyroscope-io/godeltaprof v0.1.2 h1:MdlEmYELd5w+lvIzmZvXGNMVzW2Qc9jDMuJaPOR75g4=
 github.com/pyroscope-io/godeltaprof v0.1.2/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE=
 github.com/qingstor/qingstor-sdk-go/v4 v4.4.0 h1:tbItWtGB1TDfYzqK8dtm6tV+xWU5iYMwL37C6AL5dDs=
diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go
index 4a56da504eed..ccd720f3b51a 100644
--- a/pkg/object/nfs.go
+++ b/pkg/object/nfs.go
@@ -275,7 +275,7 @@ func (n *nfsStore) List(prefix, marker, delimiter string, limit int64, followLin
 		return nil, err
 	}
 	for _, e := range entries {
-		p := filepath.Join(prefix, e.Name())
+		p := filepath.Join(dir, e.Name())
 		if e.IsDir() && !e.isSymlink {
 			p = filepath.ToSlash(p + "/")
 		}
diff --git a/pkg/object/object_storage_test.go b/pkg/object/object_storage_test.go
index b4522ddad59e..3fb56c4feed1 100644
--- a/pkg/object/object_storage_test.go
+++ b/pkg/object/object_storage_test.go
@@ -309,7 +309,7 @@ func testStorage(t *testing.T, s ObjectStorage) {
 		}
 	} else {
 		if len(obs) != 2 {
-			t.Fatalf("list with delimiter should return three results but got %d", len(obs))
+			t.Fatalf("list with delimiter should return two results but got %d", len(obs))
 		}
 		keys := []string{"a/", "a1"}
 		for i, o := range obs {
@@ -752,6 +752,17 @@ func TestOBS(t *testing.T) { //skip mutate
 	testStorage(t, b)
 }
 
+func TestNFS(t *testing.T) { //skip mutate
+	if os.Getenv("NFS_ADDR") == "" {
+		t.SkipNow()
+	}
+	b, err := newNFSStore(os.Getenv("NFS_ADDR"), os.Getenv("NFS_ACCESS_KEY"), os.Getenv("NFS_SECRET_KEY"), "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	testStorage(t, b)
+}
+
 func TestHDFS(t *testing.T) { //skip mutate
 	conf := make(hadoopconf.HadoopConf)
 	conf["dfs.namenode.rpc-address.ns.namenode1"] = "hadoop01:8020"