fix fs close return cache and control symbolic in listStatus (#105)

Co-authored-by: alantong(佟明达) <[email protected]>
tencentyun · Feb 3, 2023 · c798aac · c798aac
1 parent c083016
commit c798aac
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 7 deletions.
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>com.qcloud.cos</groupId>
     <artifactId>hadoop-cos</artifactId>
-    <version>8.2.3</version>
+    <version>8.2.4</version>
     <packaging>jar</packaging>
 
     <name>Apache Hadoop Tencent Cloud COS Support</name>

diff --git a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java
@@ -24,6 +24,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.apache.hadoop.fs.CosNUtils.propagateBucketOptions;
 
@@ -54,6 +55,7 @@ public class CosFileSystem extends FileSystem {
     private boolean isPosixUseOFSRanger;
     private boolean isPosixImpl = false;
     private FileSystem actualImplFS = null;
+    private final AtomicBoolean closed = new AtomicBoolean(false);
 
     private URI uri;
     private Path workingDir;
@@ -126,8 +128,9 @@ public void initialize(URI uri, Configuration originalConf) throws IOException {
             this.actualImplFS = getActualFileSystemByClassName(posixBucketFSImpl);
 
             // judge normal impl first, skip the class nodef error when only use normal bucket
+            // outside can use native store to tell whether is posix bucket, not need head bucket twice. can be used by flink cos.
+            this.nativeStore.setPosixBucket(true);
             if (this.actualImplFS instanceof CosNFileSystem) {
-                this.nativeStore.setPosixBucket(true);
                 ((CosNFileSystem) this.actualImplFS).withStore(this.nativeStore).withBucket(bucket)
                         .withPosixBucket(isPosixFSStore).withRangerCredentialsClient(rangerCredentialsClient);
             } else if (this.actualImplFS instanceof CHDFSHadoopFileSystemAdapter) {
@@ -656,11 +659,19 @@ private void checkInitialized() throws IOException {
     @Override
     public void close() throws IOException {
         LOG.info("begin to close cos file system");
-        this.actualImplFS.close();
-        if (null != this.nativeStore && this.isDefaultNativeStore) {
-            // close range client later, inner native store
-            this.nativeStore.close();
+        if (this.closed.getAndSet(true)) {
+            // already closed
+            return;
         }
         this.initialized = false;
+        try {
+            super.close();
+        } finally {
+            this.actualImplFS.close();
+            if (null != this.nativeStore && this.isDefaultNativeStore) {
+                // close range client later, inner native store
+                this.nativeStore.close();
+            }
+        }
     }
 }
diff --git a/src/main/java/org/apache/hadoop/fs/CosNFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosNFileSystem.java
@@ -662,7 +662,7 @@ public FileStatus[] listStatus(Path f) throws IOException {
                     LOG.debug("This is just the directory we have been asked to list. cos key: {}.",
                             fileMetadata.getKey());
                 } else {
-                    if (fileMetadata.getLength() < this.symbolicLinkSizeThreshold) {
+                    if (this.supportsSymlinks() && fileMetadata.getLength() < this.symbolicLinkSizeThreshold) {
                         CosNSymlinkMetadata cosNSymlinkMetadata = this.nativeStore.retrieveSymlinkMetadata(
                                 fileMetadata.getKey());
                         if (null != cosNSymlinkMetadata) {