add the dst create dir recursive check (#79)

Co-authored-by: alantong(佟明达) <[email protected]>
tencentyun · Sep 14, 2022 · 3b33083 · 3b33083
1 parent 6bf09d6
commit 3b33083
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 2 deletions.
diff --git a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java
@@ -484,7 +484,6 @@ private boolean useOFSRanger() {
      */
     private void checkCustomAuth(Configuration conf) throws IOException {
         // todo: need get token first
-        checkInitialized();
         this.rangerCredentialsClient.doCheckCustomAuth(conf);
     }
 

diff --git a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java
@@ -160,4 +160,8 @@ public class CosNConfigKeys extends CommonConfigurationKeys {
     public static final String COSN_POSIX_BUCKET_USE_OFS_RANGER_ENABLED = "fs.cosn.posix.bucket.use_ofs_ranger.enabled";
     public static final boolean DEFAULT_COSN_POSIX_BUCKET_USE_OFS_RANGER_ENABLED = false;
 
+    // create() recursive check dst dir which increase the getFileStatus call which increase head and list qps.
+    // please notice when set to false may lose data, so only change to false when you know what are you doing.
+    public static final String COSN_CREATE_RECURSIVE_CHECK_DST_DIR_ENABLED = "fs.cosn.create.recursive.check_dst_dir.enabled";
+    public static final boolean DEFAULT_COSN_CREATE_RECURSIVE_CHECK_DST_DIR_ENABLED = true;
 }
diff --git a/src/main/java/org/apache/hadoop/fs/CosNFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosNFileSystem.java
@@ -43,6 +43,7 @@ public class CosNFileSystem extends FileSystem {
     private boolean isPosixBucket;
     private NativeFileSystemStore nativeStore;
     private boolean isDefaultNativeStore;
+    private boolean isCreateRecursiveCheckDstDir;
     private Path workingDir;
     private String owner = "Unknown";
     private String group = "Unknown";
@@ -120,6 +121,11 @@ public void initialize(URI uri, Configuration conf) throws IOException {
                 CosNConfigKeys.READ_AHEAD_QUEUE_SIZE,
                 CosNConfigKeys.DEFAULT_READ_AHEAD_QUEUE_SIZE
         );
+
+        this.isCreateRecursiveCheckDstDir = this.getConf().getBoolean(
+                CosNConfigKeys.COSN_CREATE_RECURSIVE_CHECK_DST_DIR_ENABLED,
+                CosNConfigKeys.DEFAULT_COSN_CREATE_RECURSIVE_CHECK_DST_DIR_ENABLED
+        );
         Preconditions.checkArgument(uploadThreadPoolSize > 0,
                 String.format("The uploadThreadPoolSize[%d] should be positive.", uploadThreadPoolSize));
         Preconditions.checkArgument(readAheadPoolSize > 0,
@@ -296,7 +302,9 @@ public FSDataOutputStream create(Path f, FsPermission permission,
                 throw new FileAlreadyExistsException("Directory already exists: " + f);
             }
         } catch (FileNotFoundException e) {
-            validatePath(f);
+            if (this.isCreateRecursiveCheckDstDir) {
+                validatePath(f);
+            }
         }
 
         Path absolutePath = makeAbsolute(f);