diff --git a/pom.xml b/pom.xml index 8a4ade17..70de1177 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.qcloud.cos hadoop-cos - 8.2.3 + 8.2.4 jar Apache Hadoop Tencent Cloud COS Support diff --git a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java index 370f6edc..33165711 100644 --- a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java +++ b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.hadoop.fs.CosNUtils.propagateBucketOptions; @@ -54,6 +55,7 @@ public class CosFileSystem extends FileSystem { private boolean isPosixUseOFSRanger; private boolean isPosixImpl = false; private FileSystem actualImplFS = null; + private final AtomicBoolean closed = new AtomicBoolean(false); private URI uri; private Path workingDir; @@ -126,8 +128,9 @@ public void initialize(URI uri, Configuration originalConf) throws IOException { this.actualImplFS = getActualFileSystemByClassName(posixBucketFSImpl); // judge normal impl first, skip the class nodef error when only use normal bucket + // outside can use native store to tell whether is posix bucket, not need head bucket twice. can be used by flink cos. + this.nativeStore.setPosixBucket(true); if (this.actualImplFS instanceof CosNFileSystem) { - this.nativeStore.setPosixBucket(true); ((CosNFileSystem) this.actualImplFS).withStore(this.nativeStore).withBucket(bucket) .withPosixBucket(isPosixFSStore).withRangerCredentialsClient(rangerCredentialsClient); } else if (this.actualImplFS instanceof CHDFSHadoopFileSystemAdapter) { @@ -656,11 +659,19 @@ private void checkInitialized() throws IOException { @Override public void close() throws IOException { LOG.info("begin to close cos file system"); - this.actualImplFS.close(); - if (null != this.nativeStore && this.isDefaultNativeStore) { - // close range client later, inner native store - this.nativeStore.close(); + if (this.closed.getAndSet(true)) { + // already closed + return; } this.initialized = false; + try { + super.close(); + } finally { + this.actualImplFS.close(); + if (null != this.nativeStore && this.isDefaultNativeStore) { + // close range client later, inner native store + this.nativeStore.close(); + } + } } } diff --git a/src/main/java/org/apache/hadoop/fs/CosNFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosNFileSystem.java index d73e039b..347a3fce 100644 --- a/src/main/java/org/apache/hadoop/fs/CosNFileSystem.java +++ b/src/main/java/org/apache/hadoop/fs/CosNFileSystem.java @@ -662,7 +662,7 @@ public FileStatus[] listStatus(Path f) throws IOException { LOG.debug("This is just the directory we have been asked to list. cos key: {}.", fileMetadata.getKey()); } else { - if (fileMetadata.getLength() < this.symbolicLinkSizeThreshold) { + if (this.supportsSymlinks() && fileMetadata.getLength() < this.symbolicLinkSizeThreshold) { CosNSymlinkMetadata cosNSymlinkMetadata = this.nativeStore.retrieveSymlinkMetadata( fileMetadata.getKey()); if (null != cosNSymlinkMetadata) {