fix: skip dirs with no permissions (#4121)

juicedata · Nov 1, 2023 · c270084 · c270084
1 parent 7f73821
commit c270084
Show file tree

Hide file tree

Showing 11 changed files with 70 additions and 7 deletions.
diff --git a/.github/scripts/setup-hdfs.sh b/.github/scripts/setup-hdfs.sh
@@ -90,5 +90,6 @@ echo "hello world" > /tmp/testfile
 cd ~/app/hadoop-${HADOOP_VERSION}/bin
 ./hdfs dfs -put /tmp/testfile /
 ./hdfs dfs -rm /testfile
+./hdfs dfs -chmod 777 /
 
 echo "hdfs started successfully"
diff --git a/.github/workflows/dockerfile-sftp b/.github/workflows/dockerfile-sftp
@@ -1,5 +1,4 @@
-FROM debian:stretch-slim
-RUN sed -i s@/archive.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list
+FROM debian:stable-slim
 RUN apt-get clean
 RUN apt-get update
 
@@ -9,5 +8,7 @@ RUN mkdir /run/sshd
 RUN sed -i 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config
 RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config
 RUN echo "root:password"|chpasswd
+RUN useradd -m testUser1
+RUN echo "testUser1:password"|chpasswd
 EXPOSE 22
 CMD    ["/usr/sbin/sshd", "-D"]
diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml
@@ -41,8 +41,8 @@ jobs:
       GLUSTER_VOLUME: jfstest/gv0
       DISPLAY_PROGRESSBAR: false
       HDFS_ADDR: localhost:8020
-      SFTP_HOST: localhost:2222:/upload/
-      SFTP_USER: root
+      SFTP_HOST: localhost:2222:/home/testUser1/upload/
+      SFTP_USER: testUser1
       SFTP_PASS: password
       WEBDAV_TEST_BUCKET: 127.0.0.1:9007
       TIKV_ADDR: 127.0.0.1

diff --git a/pkg/object/file.go b/pkg/object/file.go
@@ -291,6 +291,10 @@ func (d *filestore) List(prefix, marker, delimiter string, limit int64, followLi
 	}
 	entries, err := readDirSorted(dir, followLink)
 	if err != nil {
+		if os.IsPermission(err) {
+			logger.Warnf("skip %s: %s", dir, err)
+			return nil, nil
+		}
 		if os.IsNotExist(err) {
 			return nil, nil
 		}

diff --git a/pkg/object/filesystem_test.go b/pkg/object/filesystem_test.go
@@ -64,7 +64,7 @@ func TestHDFS2(t *testing.T) { //skip mutate
 	if os.Getenv("HDFS_ADDR") == "" {
 		t.Skip()
 	}
-	dfs, _ := newHDFS(os.Getenv("HDFS_ADDR"), "", "", "")
+	dfs, _ := newHDFS(os.Getenv("HDFS_ADDR"), "testUser1", "", "")
 	testFileSystem(t, dfs)
 }
 
@@ -78,7 +78,7 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
 	}
 	// initialize directory tree
 	for _, key := range keys {
-		if err := s.Put(key, bytes.NewReader([]byte{})); err != nil {
+		if err := s.Put(key, bytes.NewReader([]byte{'a', 'b'})); err != nil {
 			t.Fatalf("PUT object `%s` failed: %q", key, err)
 		}
 	}
@@ -125,6 +125,34 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
 		t.Fatalf("testKeysEqual fail: %s", err)
 	}
 
+	if ss, ok := s.(FileSystem); ok {
+		for _, mode := range []uint32{0022, 0122, 0422} {
+			t.Logf("test mode %o", os.FileMode(mode))
+			err := ss.Chmod("x/", os.FileMode(mode))
+			if err != nil {
+				t.Fatalf("chmod %ofailed: %s", mode, err)
+			}
+
+			objs, err = listAll(s, "x", "", 100, true)
+			if err != nil {
+				t.Fatalf("list failed: %s mode %o", err, mode)
+			}
+			expectedKeys = []string{"x/", "xy.txt", "xyz/", "xyz/xyz.txt"}
+			if mode == 0422 {
+				if _, ok := ss.(*gluster); ok {
+					expectedKeys = []string{"x/", "x/x.txt", "xy.txt", "xyz/", "xyz/xyz.txt"}
+				}
+			}
+			if err = testKeysEqual(objs, expectedKeys); err != nil {
+				t.Fatalf("testKeysEqual fail: %s mode %o", err, mode)
+			}
+			err = ss.Chmod("x/", os.FileMode(0777))
+			if err != nil {
+				t.Fatalf("chmod %o failed: %s", mode, err)
+			}
+		}
+	}
+
 	objs, err = listAll(s, "xy", "", 100, true)
 	if err != nil {
 		t.Fatalf("list failed: %s", err)

diff --git a/pkg/object/gluster.go b/pkg/object/gluster.go
@@ -221,6 +221,10 @@ func (d *gluster) List(prefix, marker, delimiter string, limit int64, followLink
 	}
 	entries, err := d.readDirSorted(dir, followLink)
 	if err != nil {
+		if os.IsPermission(err) {
+			logger.Warnf("skip %s: %s", dir, err)
+			return nil, nil
+		}
 		if os.IsNotExist(err) {
 			return nil, nil
 		}

diff --git a/pkg/object/gluster_test.go b/pkg/object/gluster_test.go
@@ -30,4 +30,13 @@ func TestGluster(t *testing.T) {
 	}
 	b, _ := newGluster(os.Getenv("GLUSTER_VOLUME"), "", "", "")
 	testStorage(t, b)
+
+}
+
+func TestGluster2(t *testing.T) {
+	if os.Getenv("GLUSTER_VOLUME") == "" {
+		t.SkipNow()
+	}
+	b, _ := newGluster(os.Getenv("GLUSTER_VOLUME"), "", "", "")
+	testFileSystem(t, b)
 }
diff --git a/pkg/object/hdfs.go b/pkg/object/hdfs.go
@@ -208,6 +208,10 @@ func (h *hdfsclient) List(prefix, marker, delimiter string, limit int64, followL
 		entries, err = file.Readdir(0)
 	}
 	if err != nil {
+		if os.IsPermission(err) {
+			logger.Warnf("skip %s: %s", dir, err)
+			return nil, nil
+		}
 		if os.IsNotExist(err) {
 			return nil, nil
 		}

diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go
@@ -269,6 +269,10 @@ func (n *nfsStore) List(prefix, marker, delimiter string, limit int64, followLin
 	}
 	entries, err := n.readDirSorted(dir, followLink)
 	if err != nil {
+		if os.IsPermission(err) {
+			logger.Warnf("skip %s: %s", dir, err)
+			return nil, nil
+		}
 		if os.IsNotExist(err) {
 			return nil, nil
 		}

diff --git a/pkg/object/sftp.go b/pkg/object/sftp.go
@@ -385,6 +385,10 @@ func (f *sftpStore) List(prefix, marker, delimiter string, limit int64, followLi
 	}
 	infos, err := c.sftpClient.ReadDir(dir)
 	if err != nil {
+		if os.IsPermission(err) {
+			logger.Warnf("skip %s: %s", dir, err)
+			return nil, nil
+		}
 		if os.IsNotExist(err) {
 			return nil, nil
 		}

diff --git a/pkg/object/webdav.go b/pkg/object/webdav.go
@@ -192,7 +192,11 @@ func (w *webdav) List(prefix, marker, delimiter string, limit int64, followLink
 
 	infos, err := w.c.ReadDir(root)
 	if err != nil {
-		if os.IsNotExist(err) {
+		if gowebdav.IsErrCode(err, http.StatusForbidden) {
+			logger.Warnf("skip %s: %s", root, err)
+			return nil, nil
+		}
+		if gowebdav.IsErrNotFound(err) {
 			return nil, nil
 		}
 		return nil, err