diff --git a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java index f89d299f..86909f0d 100644 --- a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java +++ b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java @@ -484,7 +484,6 @@ private boolean useOFSRanger() { */ private void checkCustomAuth(Configuration conf) throws IOException { // todo: need get token first - checkInitialized(); this.rangerCredentialsClient.doCheckCustomAuth(conf); } diff --git a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java index 06b52744..b597c013 100644 --- a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java +++ b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java @@ -160,4 +160,8 @@ public class CosNConfigKeys extends CommonConfigurationKeys { public static final String COSN_POSIX_BUCKET_USE_OFS_RANGER_ENABLED = "fs.cosn.posix.bucket.use_ofs_ranger.enabled"; public static final boolean DEFAULT_COSN_POSIX_BUCKET_USE_OFS_RANGER_ENABLED = false; + // create() recursive check dst dir which increase the getFileStatus call which increase head and list qps. + // please notice when set to false may lose data, so only change to false when you know what are you doing. + public static final String COSN_CREATE_RECURSIVE_CHECK_DST_DIR_ENABLED = "fs.cosn.create.recursive.check_dst_dir.enabled"; + public static final boolean DEFAULT_COSN_CREATE_RECURSIVE_CHECK_DST_DIR_ENABLED = true; } diff --git a/src/main/java/org/apache/hadoop/fs/CosNFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosNFileSystem.java index d4e6663f..308acd8f 100644 --- a/src/main/java/org/apache/hadoop/fs/CosNFileSystem.java +++ b/src/main/java/org/apache/hadoop/fs/CosNFileSystem.java @@ -43,6 +43,7 @@ public class CosNFileSystem extends FileSystem { private boolean isPosixBucket; private NativeFileSystemStore nativeStore; private boolean isDefaultNativeStore; + private boolean isCreateRecursiveCheckDstDir; private Path workingDir; private String owner = "Unknown"; private String group = "Unknown"; @@ -120,6 +121,11 @@ public void initialize(URI uri, Configuration conf) throws IOException { CosNConfigKeys.READ_AHEAD_QUEUE_SIZE, CosNConfigKeys.DEFAULT_READ_AHEAD_QUEUE_SIZE ); + + this.isCreateRecursiveCheckDstDir = this.getConf().getBoolean( + CosNConfigKeys.COSN_CREATE_RECURSIVE_CHECK_DST_DIR_ENABLED, + CosNConfigKeys.DEFAULT_COSN_CREATE_RECURSIVE_CHECK_DST_DIR_ENABLED + ); Preconditions.checkArgument(uploadThreadPoolSize > 0, String.format("The uploadThreadPoolSize[%d] should be positive.", uploadThreadPoolSize)); Preconditions.checkArgument(readAheadPoolSize > 0, @@ -296,7 +302,9 @@ public FSDataOutputStream create(Path f, FsPermission permission, throw new FileAlreadyExistsException("Directory already exists: " + f); } } catch (FileNotFoundException e) { - validatePath(f); + if (this.isCreateRecursiveCheckDstDir) { + validatePath(f); + } } Path absolutePath = makeAbsolute(f);