CI: fix flaky test (#4838)

juicedata · May 11, 2024 · 3c5dd37 · 3c5dd37
1 parent 19279bf
commit 3c5dd37
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 10 deletions.
diff --git a/.github/scripts/command/cache.sh b/.github/scripts/command/cache.sh
@@ -110,23 +110,23 @@ test_disk_failover()
     dd if=/dev/urandom of=/tmp/test_failover bs=1M count=$TEST_FILE_SIZE
     cp /tmp/test_failover /tmp/jfs/test_failover
     ./juicefs warmup /tmp/jfs/test_failover
-    du -sh /var/jfsCache?
+    du -sh /var/jfsCache? || true
     ./juicefs warmup --check /tmp/jfs 2>&1 | tee check.log
     check_warmup_log check.log
     check_cache_distribute $TEST_FILE_SIZE /var/jfsCache1 /var/jfsCache2 /var/jfsCache3
     ./juicefs warmup --evict /tmp/jfs/test_failover
-    du -sh /var/jfsCache?
+    du -sh /var/jfsCache? || true
     ./juicefs warmup --check /tmp/jfs 2>&1 | tee check.log
     check_evict_log check.log
     mv cache.db cache.db.bak
     # /etc/init.d/redis-server stop
     # sleep 10
     ./juicefs warmup /tmp/jfs/test_failover
-    du -sh /var/jfsCache2 /var/jfsCache3
+    du -sh /var/jfsCache2 /var/jfsCache3 || true
     ./juicefs warmup --check /tmp/jfs 2>&1 | tee check.log
     sleep 10
     ./juicefs warmup /tmp/jfs/test_failover
-    du -sh /var/jfsCache2 /var/jfsCache3
+    du -sh /var/jfsCache2 /var/jfsCache3 || true
     ./juicefs warmup --check /tmp/jfs 2>&1 | tee check.log
     check_warmup_log check.log
     check_cache_distribute $TEST_FILE_SIZE /var/jfsCache2 /var/jfsCache3
@@ -149,7 +149,7 @@ test_mount_same_disk_after_failure()
     dd if=/dev/urandom of=/tmp/test_failover bs=1M count=$TEST_FILE_SIZE
     cp /tmp/test_failover /tmp/jfs/test_failover
     ./juicefs warmup /tmp/jfs/test_failover
-    du -sh /var/jfsCache?
+    du -sh /var/jfsCache? || true
     ./juicefs warmup --check /tmp/jfs 2>&1 | tee check.log
     check_warmup_log check.log
     check_cache_distribute $TEST_FILE_SIZE /var/jfsCache1 /var/jfsCache2 /var/jfsCache3
@@ -163,7 +163,7 @@ test_mount_same_disk_after_failure()
     # timeout 30s bash -c 'until nc -zv localhost 6379; do sleep 1; done'
     ./juicefs mount $META_URL /tmp/jfs -d --cache-dir=/var/jfsCache2:/var/jfsCache3:/var/jfsCache1
     echo "sleep 3s to wait to build cache in memory " && sleep 3
-    du -sh /var/jfsCache1 /var/jfsCache2 /var/jfsCache3
+    du -sh /var/jfsCache1 /var/jfsCache2 /var/jfsCache3 || true
     ./juicefs warmup --check /tmp/jfs 2>&1 | tee check.log
     check_warmup_log check.log
     echo stop minio && docker stop minio
@@ -183,7 +183,7 @@ skip_test_rebalance_after_disk_failure_and_replace()
     dd if=/dev/urandom of=/tmp/test_failover bs=1M count=$TEST_FILE_SIZE
     cp /tmp/test_failover /tmp/jfs/test_failover
     ./juicefs warmup /tmp/jfs/test_failover
-    du -sh /var/jfsCache?
+    du -sh /var/jfsCache? || true
     ./juicefs warmup --check /tmp/jfs 2>&1 | tee check.log
     check_warmup_log check.log
     check_cache_distribute $TEST_FILE_SIZE /var/jfsCache1 /var/jfsCache2 /var/jfsCache3

diff --git a/.github/scripts/command/gateway.sh b/.github/scripts/command/gateway.sh
@@ -14,13 +14,21 @@ export MINIO_REFRESH_IAM_INTERVAL=10s
 prepare_test()
 {
     umount_jfs /tmp/jfs $META_URL
-    lsof -i :9001 | awk 'NR!=1 {print $2}' | xargs -r kill -9 || true
-    lsof -i :9002 | awk 'NR!=1 {print $2}' | xargs -r kill -9 || true
+    kill_gateway 9001
+    kill_gateway 9002
     python3 .github/scripts/flush_meta.py $META_URL
     rm -rf /var/jfs/myjfs || true
     rm -rf /var/jfsCache/myjfs || true
 }
 
+kill_gateway() {
+    port=$1
+    lsof -i:$port || true
+    lsof -t -i :$port | xargs -r kill -9 || true
+}
+
+trap 'kill_gateway 9001; kill_gateway 9002' EXIT
+
 start_two_gateway()
 {
     prepare_test

diff --git a/.github/scripts/command/quota.sh b/.github/scripts/command/quota.sh
@@ -55,7 +55,7 @@ test_total_inodes(){
     sleep $((HEARTBEAT_INTERVAL+1))
     set +x
     for i in {1001..2000}; do
-        echo $i | tee /jfs/test$i > /dev/null || (df -i /jfs && ls /jfs/ -l | wc-l  && exit 1)
+        echo $i | tee /jfs/test$i > /dev/null || (df -i /jfs && ls /jfs/ -l | wc -l  && exit 1)
     done
     set -x
     sleep $VOLUME_QUOTA_FLUSH_INTERVAL