perf: improve the create operation performance.

tencentyun · Feb 23, 2023 · 877db61 · 877db61
1 parent ab4c584
commit 877db61
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 13 deletions.
diff --git a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java
@@ -222,11 +222,6 @@ public class CosNConfigKeys extends CommonConfigurationKeys {
     public static final String COSN_SUPPORT_SYMLINK_ENABLED = "fs.cosn.support_symlink.enabled";
     public static final boolean DEFAULT_COSN_SUPPORT_SYMLINK_ENABLED = false;
 
-    // create() recursive check dst dir which increase the getFileStatus call which increase head and list qps.
-    // please notice when set to false may lose data, so only change to false when you know what are you doing.
-    public static final String COSN_CREATE_RECURSIVE_CHECK_DST_DIR_ENABLED = "fs.cosn.create.recursive.check_dst_dir.enabled";
-    public static final boolean DEFAULT_COSN_CREATE_RECURSIVE_CHECK_DST_DIR_ENABLED = true;
-
     public static final String COSN_FILESTATUS_LIST_OP_ENABLED = "fs.cosn.filestatus.list.op.enabled";
     public static final boolean DEFAULT_FILESTATUS_LIST_OP_ENABLED = true;
 }
diff --git a/src/main/java/org/apache/hadoop/fs/CosNFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosNFileSystem.java
@@ -147,10 +147,6 @@ public void initialize(URI uri, Configuration conf) throws IOException {
                 CosNConfigKeys.DEFAULT_READ_AHEAD_QUEUE_SIZE
         );
 
-        this.isCreateRecursiveCheckDstDir = this.getConf().getBoolean(
-                CosNConfigKeys.COSN_CREATE_RECURSIVE_CHECK_DST_DIR_ENABLED,
-                CosNConfigKeys.DEFAULT_COSN_CREATE_RECURSIVE_CHECK_DST_DIR_ENABLED
-        );
         Preconditions.checkArgument(uploadThreadPoolSize > 0,
                 String.format("The uploadThreadPoolSize[%d] should be positive.", uploadThreadPoolSize));
         Preconditions.checkArgument(readAheadPoolSize > 0,
@@ -419,10 +415,13 @@ public FSDataOutputStream create(Path f, FsPermission permission,
             if (targetFileStatus.isDirectory()) {
                 throw new FileAlreadyExistsException("Directory already exists: " + f);
             }
-        } catch (FileNotFoundException e) {
-            if (this.isCreateRecursiveCheckDstDir) {
-                validatePath(f);
-            }
+        } catch (FileNotFoundException ignore) {
+            // NOTE: 这里之前认为可能会出现从 COS 的 SDK 或者 API 上传了一个 / 结尾的有内容对象
+            // 那么再在这个文件前缀下面成功创建新的对象而不报错的话，其实是不符合文件系统语义规范的。
+            // 同时，也是为了保证一个完整的目录结构，但是确实会带来元数据查询请求的明显放大。
+            // 不过这里，因为一般不会出现 / 结尾的内容对象，即使出现也不会覆盖丢失（因为这里相当于它的一个commonPrefix，原始对象还在COS里面）
+            // 所以决定去掉这个检查，来改善优化性能。
+            // validatePath(f)
         }
 
         Path absolutePath = makeAbsolute(f);