diff --git a/.github/scripts/setup-hdfs.sh b/.github/scripts/setup-hdfs.sh index 189976ffd279..3c3ea96decf4 100755 --- a/.github/scripts/setup-hdfs.sh +++ b/.github/scripts/setup-hdfs.sh @@ -90,5 +90,6 @@ echo "hello world" > /tmp/testfile cd ~/app/hadoop-${HADOOP_VERSION}/bin ./hdfs dfs -put /tmp/testfile / ./hdfs dfs -rm /testfile +./hdfs dfs -chmod 777 / echo "hdfs started successfully" diff --git a/.github/workflows/dockerfile-sftp b/.github/workflows/dockerfile-sftp index 5e662d87d527..77bcf244ce0a 100644 --- a/.github/workflows/dockerfile-sftp +++ b/.github/workflows/dockerfile-sftp @@ -1,5 +1,4 @@ -FROM debian:stretch-slim -RUN sed -i s@/archive.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list +FROM debian:stable-slim RUN apt-get clean RUN apt-get update @@ -9,5 +8,7 @@ RUN mkdir /run/sshd RUN sed -i 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config RUN echo "root:password"|chpasswd +RUN useradd -m testUser1 +RUN echo "testUser1:password"|chpasswd EXPOSE 22 CMD ["/usr/sbin/sshd", "-D"] diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 9bf63d016fa1..dc57816bd49b 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -41,8 +41,8 @@ jobs: GLUSTER_VOLUME: jfstest/gv0 DISPLAY_PROGRESSBAR: false HDFS_ADDR: localhost:8020 - SFTP_HOST: localhost:2222:/upload/ - SFTP_USER: root + SFTP_HOST: localhost:2222:/home/testUser1/upload/ + SFTP_USER: testUser1 SFTP_PASS: password WEBDAV_TEST_BUCKET: 127.0.0.1:9007 TIKV_ADDR: 127.0.0.1 diff --git a/pkg/object/file.go b/pkg/object/file.go index 1769c969d134..fb66f018bab0 100644 --- a/pkg/object/file.go +++ b/pkg/object/file.go @@ -291,6 +291,10 @@ func (d *filestore) List(prefix, marker, delimiter string, limit int64, followLi } entries, err := readDirSorted(dir, followLink) if err != nil { + if os.IsPermission(err) { + logger.Warnf("skip %s: %s", dir, err) + return nil, nil + } if os.IsNotExist(err) { return nil, nil } diff --git a/pkg/object/filesystem_test.go b/pkg/object/filesystem_test.go index 642fc7c3cd2c..82d00b34f9fb 100644 --- a/pkg/object/filesystem_test.go +++ b/pkg/object/filesystem_test.go @@ -64,7 +64,7 @@ func TestHDFS2(t *testing.T) { //skip mutate if os.Getenv("HDFS_ADDR") == "" { t.Skip() } - dfs, _ := newHDFS(os.Getenv("HDFS_ADDR"), "", "", "") + dfs, _ := newHDFS(os.Getenv("HDFS_ADDR"), "testUser1", "", "") testFileSystem(t, dfs) } @@ -78,7 +78,7 @@ func testFileSystem(t *testing.T, s ObjectStorage) { } // initialize directory tree for _, key := range keys { - if err := s.Put(key, bytes.NewReader([]byte{})); err != nil { + if err := s.Put(key, bytes.NewReader([]byte{'a', 'b'})); err != nil { t.Fatalf("PUT object `%s` failed: %q", key, err) } } @@ -125,6 +125,34 @@ func testFileSystem(t *testing.T, s ObjectStorage) { t.Fatalf("testKeysEqual fail: %s", err) } + if ss, ok := s.(FileSystem); ok { + for _, mode := range []uint32{0022, 0122, 0422} { + t.Logf("test mode %o", os.FileMode(mode)) + err := ss.Chmod("x/", os.FileMode(mode)) + if err != nil { + t.Fatalf("chmod %ofailed: %s", mode, err) + } + + objs, err = listAll(s, "x", "", 100, true) + if err != nil { + t.Fatalf("list failed: %s mode %o", err, mode) + } + expectedKeys = []string{"x/", "xy.txt", "xyz/", "xyz/xyz.txt"} + if mode == 0422 { + if _, ok := ss.(*gluster); ok { + expectedKeys = []string{"x/", "x/x.txt", "xy.txt", "xyz/", "xyz/xyz.txt"} + } + } + if err = testKeysEqual(objs, expectedKeys); err != nil { + t.Fatalf("testKeysEqual fail: %s mode %o", err, mode) + } + err = ss.Chmod("x/", os.FileMode(0777)) + if err != nil { + t.Fatalf("chmod %o failed: %s", mode, err) + } + } + } + objs, err = listAll(s, "xy", "", 100, true) if err != nil { t.Fatalf("list failed: %s", err) diff --git a/pkg/object/gluster.go b/pkg/object/gluster.go index 3ea1cb539144..927a4ee5db27 100644 --- a/pkg/object/gluster.go +++ b/pkg/object/gluster.go @@ -221,6 +221,10 @@ func (d *gluster) List(prefix, marker, delimiter string, limit int64, followLink } entries, err := d.readDirSorted(dir, followLink) if err != nil { + if os.IsPermission(err) { + logger.Warnf("skip %s: %s", dir, err) + return nil, nil + } if os.IsNotExist(err) { return nil, nil } diff --git a/pkg/object/gluster_test.go b/pkg/object/gluster_test.go index e155cb7e5a9d..f7eaa0554752 100644 --- a/pkg/object/gluster_test.go +++ b/pkg/object/gluster_test.go @@ -30,4 +30,13 @@ func TestGluster(t *testing.T) { } b, _ := newGluster(os.Getenv("GLUSTER_VOLUME"), "", "", "") testStorage(t, b) + +} + +func TestGluster2(t *testing.T) { + if os.Getenv("GLUSTER_VOLUME") == "" { + t.SkipNow() + } + b, _ := newGluster(os.Getenv("GLUSTER_VOLUME"), "", "", "") + testFileSystem(t, b) } diff --git a/pkg/object/hdfs.go b/pkg/object/hdfs.go index b06d3be4a615..e297d8eb3c6a 100644 --- a/pkg/object/hdfs.go +++ b/pkg/object/hdfs.go @@ -208,6 +208,10 @@ func (h *hdfsclient) List(prefix, marker, delimiter string, limit int64, followL entries, err = file.Readdir(0) } if err != nil { + if os.IsPermission(err) { + logger.Warnf("skip %s: %s", dir, err) + return nil, nil + } if os.IsNotExist(err) { return nil, nil } diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go index ccd720f3b51a..0076e6de62ea 100644 --- a/pkg/object/nfs.go +++ b/pkg/object/nfs.go @@ -269,6 +269,10 @@ func (n *nfsStore) List(prefix, marker, delimiter string, limit int64, followLin } entries, err := n.readDirSorted(dir, followLink) if err != nil { + if os.IsPermission(err) { + logger.Warnf("skip %s: %s", dir, err) + return nil, nil + } if os.IsNotExist(err) { return nil, nil } diff --git a/pkg/object/sftp.go b/pkg/object/sftp.go index f91e0f798481..714b3a979bf0 100644 --- a/pkg/object/sftp.go +++ b/pkg/object/sftp.go @@ -385,6 +385,10 @@ func (f *sftpStore) List(prefix, marker, delimiter string, limit int64, followLi } infos, err := c.sftpClient.ReadDir(dir) if err != nil { + if os.IsPermission(err) { + logger.Warnf("skip %s: %s", dir, err) + return nil, nil + } if os.IsNotExist(err) { return nil, nil } diff --git a/pkg/object/webdav.go b/pkg/object/webdav.go index b6898afe0e50..8636cfdde862 100644 --- a/pkg/object/webdav.go +++ b/pkg/object/webdav.go @@ -192,7 +192,11 @@ func (w *webdav) List(prefix, marker, delimiter string, limit int64, followLink infos, err := w.c.ReadDir(root) if err != nil { - if os.IsNotExist(err) { + if gowebdav.IsErrCode(err, http.StatusForbidden) { + logger.Warnf("skip %s: %s", root, err) + return nil, nil + } + if gowebdav.IsErrNotFound(err) { return nil, nil } return nil, err