From d4cb4656f6333c4ac031fafe3b69cdda728c59b8 Mon Sep 17 00:00:00 2001 From: iabetor Date: Wed, 15 Feb 2023 14:42:38 +0800 Subject: [PATCH] s3 rename verify delete and write; posix rename verify double write (#108) Co-authored-by: vinsonruan --- pom.xml | 2 +- .../org/apache/hadoop/fs/CosFileSystem.java | 23 ++++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 70de1177..3ed25909 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.qcloud.cos hadoop-cos - 8.2.4 + 8.2.5 jar Apache Hadoop Tencent Cloud COS Support diff --git a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java index 33165711..dc47365f 100644 --- a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java +++ b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java @@ -104,7 +104,7 @@ public void initialize(URI uri, Configuration originalConf) throws IOException { this.rangerCredentialsClient = this.nativeStore.getRangerCredentialsClient(); this.isPosixUseOFSRanger = this.getConf(). getBoolean(CosNConfigKeys.COSN_POSIX_BUCKET_USE_OFS_RANGER_ENABLED, - CosNConfigKeys.DEFAULT_COSN_POSIX_BUCKET_USE_OFS_RANGER_ENABLED); + CosNConfigKeys.DEFAULT_COSN_POSIX_BUCKET_USE_OFS_RANGER_ENABLED); // required checkCustomAuth if ranger is enabled and custom authentication is enabled checkCustomAuth(conf); @@ -121,7 +121,7 @@ public void initialize(URI uri, Configuration originalConf) throws IOException { } LOG.info("The posix bucket [{}] use the class [{}] as the filesystem implementation, " + - "use each ranger [{}]", bucket, posixBucketFSImpl, this.isPosixUseOFSRanger); + "use each ranger [{}]", bucket, posixBucketFSImpl, this.isPosixUseOFSRanger); // if ofs impl. // network version start from the 2.7. // sdk version start from the 1.0.4. @@ -272,11 +272,22 @@ public FSDataInputStream open(Path f, int bufferSize) throws IOException { public boolean rename(Path src, Path dst) throws IOException { LOG.debug("Rename the source path [{}] to the dest path [{}].", src, dst); checkInitialized(); - checkPermission(src, RangerAccessType.DELETE); + renameCheckPermission(src); checkPermission(dst, RangerAccessType.WRITE); return this.actualImplFS.rename(src, dst); } + private void renameCheckPermission(Path src) throws IOException { + if (useOFSRanger()) { + return; + } + if (this.isPosixImpl) { + checkPermission(src, RangerAccessType.WRITE); + } else { + checkPermission(src, RangerAccessType.DELETE); + } + } + @Override public long getDefaultBlockSize() { return this.actualImplFS.getDefaultBlockSize(); @@ -440,7 +451,7 @@ public ContentSummary getContentSummary(Path f) throws IOException { @Override public void setOwner(Path p, String userName, String groupName) throws IOException { - LOG.debug("set owner, path: {}, userName: {}, groupName: {}",p, userName, groupName); + LOG.debug("set owner, path: {}, userName: {}, groupName: {}", p, userName, groupName); checkInitialized(); checkPermission(p, RangerAccessType.WRITE); this.actualImplFS.setOwner(p, userName, groupName); @@ -497,7 +508,7 @@ private void passThroughRangerConfig() { } } - private HashMap getPOSIXBucketConfigMap() { + private HashMap getPOSIXBucketConfigMap() { HashMap configMap = new HashMap<>(); configMap.put(CosNConfigKeys.COSN_APPID_KEY, Constants.COSN_POSIX_BUCKET_APPID_CONFIG); configMap.put(CosNConfigKeys.COSN_REGION_KEY, Constants.COSN_POSIX_BUCKET_REGION_CONFIG); @@ -510,7 +521,7 @@ private HashMap getPOSIXBucketConfigMap() { // exclude the ofs original config, filter the ofs config with COSN_CONFIG_TRANSFER_PREFIX private void transferOfsConfig() { // cosn config -> ofs config -> trsf ofs config - HashMap configMap = getPOSIXBucketConfigMap(); + HashMap configMap = getPOSIXBucketConfigMap(); for (String org : configMap.keySet()) { String content = this.getConf().get(org); if (null != content && !content.isEmpty()) {