Merge pull request #44 from vintmd/mpu-retry

add the retry when complete mpu error code in body but status code is…
tencentyun · Mar 2, 2022 · ff5dcee · ff5dcee
2 parents 920f07b + 2434ad5
commit ff5dcee
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 3 deletions.
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>com.qcloud.cos</groupId>
     <artifactId>hadoop-cos</artifactId>
-    <version>8.0.4</version>
+    <version>8.0.5</version>
     <packaging>jar</packaging>
 
     <name>Apache Hadoop Tencent Qcloud COS Support</name>
@@ -41,7 +41,7 @@
         <maven.compiler.source>1.7</maven.compiler.source>
         <maven.compiler.target>1.7</maven.compiler.target>
         <hadoop.version>3.3.0</hadoop.version>
-        <cos_api.version>5.6.68</cos_api.version>
+        <cos_api.version>5.6.69</cos_api.version>
         <google.guava.version>24.1.1-jre</google.guava.version>
         <commons_lang3.version>3.1</commons_lang3.version>
         <junit.version>4.8</junit.version>

diff --git a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java
@@ -12,7 +12,7 @@
 @InterfaceStability.Unstable
 public class CosNConfigKeys extends CommonConfigurationKeys {
     public static final String USER_AGENT = "fs.cosn.user.agent";
-    public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v8.0.4";
+    public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v8.0.5";
 
     public static final String TENCENT_EMR_VERSION_KEY = "fs.emr.version";
 

diff --git a/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java b/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java
@@ -1374,6 +1374,26 @@ private <X> Object callCOSClientWithRetry(X request) throws CosServiceException,
                         LOG.error(errMsg, cse);
                         throw new IOException(errMsg);
                     }
+                } else if (request instanceof  CompleteMultipartUploadRequest && statusCode / 100 ==2
+                        && errorCode != null && !errorCode.isEmpty()) {
+                    // complete mpu error code might be in body when status code is 200
+                    // double check to head object only works in big data job case which key is not same.
+                    String key = ((CompleteMultipartUploadRequest) request).getKey();
+                    FileMetadata fileMetadata = this.queryObjectMetadata(key);
+                    if (null != fileMetadata) {
+                        // if file exist direct return.
+                        LOG.info("complete mpu error in body, error code {}, but key {} already exist, length {}",
+                                errorCode, key, fileMetadata.getLength());
+                        return new CompleteMultipartUploadResult();
+                    }
+                    // here same like the copy request not setting the interval sleep for now
+                    if (retryIndex <= this.maxRetryTimes)  {
+                        LOG.info(errMsg, cse);
+                        ++retryIndex;
+                    } else {
+                        LOG.error(errMsg, cse);
+                        throw new IOException(errMsg);
+                    }
                 } else if (statusCode / 100 == 5) {
                     if (retryIndex <= this.maxRetryTimes) {
                         LOG.info(errMsg, cse);