add double check to complete mpu 503

tencentyun · May 11, 2022 · b662209 · b662209
1 parent 0cc1eb4
commit b662209
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 6 deletions.
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>com.qcloud.cos</groupId>
     <artifactId>hadoop-cos</artifactId>
-    <version>8.0.6</version>
+    <version>8.0.7</version>
     <packaging>jar</packaging>
 
     <name>Apache Hadoop Tencent Cloud COS Support</name>

diff --git a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java
@@ -12,7 +12,7 @@
 @InterfaceStability.Unstable
 public class CosNConfigKeys extends CommonConfigurationKeys {
     public static final String USER_AGENT = "fs.cosn.user.agent";
-    public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v8.0.5";
+    public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v8.0.7";
 
     public static final String TENCENT_EMR_VERSION_KEY = "fs.emr.version";
 

diff --git a/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java b/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java
@@ -1399,17 +1399,15 @@ private <X> Object callCOSClientWithRetry(X request) throws CosServiceException,
                 String errorCode = cse.getErrorCode();
                 LOG.debug("fail to retry statusCode {}, errorCode {}", statusCode, errorCode);
                 // 对5xx错误进行重试
-                if (request instanceof CopyObjectRequest && statusCode / 100 == 2
-                        && errorCode != null && !errorCode.isEmpty()) {
+                if (request instanceof CopyObjectRequest && hasErrorCode(statusCode, errorCode)) {
                     if (retryIndex <= this.maxRetryTimes) {
                         LOG.info(errMsg, cse);
                         ++retryIndex;
                     } else {
                         LOG.error(errMsg, cse);
                         throw new IOException(errMsg);
                     }
-                } else if (request instanceof  CompleteMultipartUploadRequest && statusCode / 100 ==2
-                        && errorCode != null && !errorCode.isEmpty()) {
+                } else if (request instanceof  CompleteMultipartUploadRequest && hasErrorCode(statusCode, errorCode)) {
                     // complete mpu error code might be in body when status code is 200
                     // double check to head object only works in big data job case which key is not same.
                     String key = ((CompleteMultipartUploadRequest) request).getKey();
@@ -1469,6 +1467,21 @@ private <X> Object callCOSClientWithRetry(X request) throws CosServiceException,
                                     throw cse;
                                 }
                             }
+
+                            // mpu might occur 503 access time out but already completed,
+                            // if direct retry may occur 403 not found the upload id.
+                            if (request instanceof CompleteMultipartUploadRequest && statusCode == 503) {
+                                String key = ((CompleteMultipartUploadRequest) request).getKey();
+                                FileMetadata fileMetadata = this.queryObjectMetadata(key);
+                                if (null != fileMetadata) {
+                                    // if file exist direct return.
+                                    LOG.info("complete mpu error might access time out, " +
+                                                    "but key {} already exist, length {}",
+                                            key, fileMetadata.getLength());
+                                    return new CompleteMultipartUploadResult();
+                                }
+                            }
+
                             Thread.sleep(
                                     ThreadLocalRandom.current().nextLong(sleepLeast, sleepBound));
                             ++retryIndex;
@@ -1488,6 +1501,10 @@ private <X> Object callCOSClientWithRetry(X request) throws CosServiceException,
         }
     }
 
+    private boolean hasErrorCode(int statusCode, String errCode) {
+        return statusCode / 100 == 2 && errCode != null && !errCode.isEmpty();
+    }
+
     private static String ensureValidAttributeName(String attributeName) {
         return attributeName.replace('.', '-').toLowerCase();
     }