From 1fcb761b203ffd4b30445d5387c9737f4f7b50b7 Mon Sep 17 00:00:00 2001 From: vintmd <61688729+vintmd@users.noreply.github.com> Date: Tue, 14 Jun 2022 11:08:46 +0800 Subject: [PATCH] fix gateway mode to process posix bucket (#62) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: alantong(佟明达) --- pom.xml | 2 +- src/main/java/org/apache/hadoop/fs/CosFileSystem.java | 2 ++ src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java | 2 +- .../org/apache/hadoop/fs/CosNativeFileSystemStore.java | 5 +++++ .../java/org/apache/hadoop/fs/NativeFileSystemStore.java | 7 +++++++ 5 files changed, 16 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d8555266..7fe2f6af 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.qcloud.cos hadoop-cos - 8.1.2 + 8.1.3 jar Apache Hadoop Tencent Cloud COS Support diff --git a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java index 4967a016..f85547ee 100644 --- a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java +++ b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java @@ -122,6 +122,7 @@ public void initialize(URI uri, Configuration conf) throws IOException { this.nativeStore.close(); this.nativeStore = null; } else if (this.actualImplFS instanceof CosNFileSystem) { + this.nativeStore.isPosixBucket(true); ((CosNFileSystem) this.actualImplFS).withStore(this.nativeStore).withBucket(bucket) .withPosixBucket(isPosixFSStore).withRangerCredentialsClient(rangerCredentialsClient); } else { @@ -132,6 +133,7 @@ public void initialize(URI uri, Configuration conf) throws IOException { } } else { // normal cos hadoop file system implements this.actualImplFS = getActualFileSystemByClassName("org.apache.hadoop.fs.CosNFileSystem"); + this.nativeStore.isPosixBucket(false); ((CosNFileSystem) this.actualImplFS).withStore(this.nativeStore).withBucket(bucket) .withPosixBucket(this.isPosixFSStore).withRangerCredentialsClient(rangerCredentialsClient); } diff --git a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java index 82a5099f..127f40e6 100644 --- a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java +++ b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java @@ -12,7 +12,7 @@ @InterfaceStability.Unstable public class CosNConfigKeys extends CommonConfigurationKeys { public static final String USER_AGENT = "fs.cosn.user.agent"; - public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v8.1.2"; + public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v8.1.3"; public static final String TENCENT_EMR_VERSION_KEY = "fs.emr.version"; diff --git a/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java b/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java index a3826d3d..428eade2 100644 --- a/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java +++ b/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java @@ -1223,6 +1223,11 @@ public void close() { this.cosClient = null; } + @Override + public void isPosixBucket(boolean isPosixBucket) { + this.isPosixBucket = isPosixBucket; + } + // process Exception and print detail private void handleException(Exception e, String key) throws IOException { String cosPath = "cosn://" + bucketName + key; diff --git a/src/main/java/org/apache/hadoop/fs/NativeFileSystemStore.java b/src/main/java/org/apache/hadoop/fs/NativeFileSystemStore.java index 071af6d1..1a5df8d9 100644 --- a/src/main/java/org/apache/hadoop/fs/NativeFileSystemStore.java +++ b/src/main/java/org/apache/hadoop/fs/NativeFileSystemStore.java @@ -112,5 +112,12 @@ CosNPartialListing list(String prefix, int maxListingLength, */ void dump() throws IOException; + /** + * Used for outer to decide inner process. + * if you use the CosNFileSystem gateway mode, + * must set native store to posix process + */ + void isPosixBucket(boolean isPosixBucket); + void close(); }