diff --git a/pom.xml b/pom.xml index 4d8470ee..360aacc8 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.qcloud.cos hadoop-cos - 8.0.6 + 8.0.7 jar Apache Hadoop Tencent Cloud COS Support diff --git a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java index 3f06e0a9..2d0e3b6f 100644 --- a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java +++ b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java @@ -12,7 +12,7 @@ @InterfaceStability.Unstable public class CosNConfigKeys extends CommonConfigurationKeys { public static final String USER_AGENT = "fs.cosn.user.agent"; - public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v8.0.5"; + public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v8.0.7"; public static final String TENCENT_EMR_VERSION_KEY = "fs.emr.version"; diff --git a/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java b/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java index 9705a236..b588b0d3 100644 --- a/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java +++ b/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java @@ -1399,8 +1399,7 @@ private Object callCOSClientWithRetry(X request) throws CosServiceException, String errorCode = cse.getErrorCode(); LOG.debug("fail to retry statusCode {}, errorCode {}", statusCode, errorCode); // 对5xx错误进行重试 - if (request instanceof CopyObjectRequest && statusCode / 100 == 2 - && errorCode != null && !errorCode.isEmpty()) { + if (request instanceof CopyObjectRequest && hasErrorCode(statusCode, errorCode)) { if (retryIndex <= this.maxRetryTimes) { LOG.info(errMsg, cse); ++retryIndex; @@ -1408,8 +1407,7 @@ private Object callCOSClientWithRetry(X request) throws CosServiceException, LOG.error(errMsg, cse); throw new IOException(errMsg); } - } else if (request instanceof CompleteMultipartUploadRequest && statusCode / 100 ==2 - && errorCode != null && !errorCode.isEmpty()) { + } else if (request instanceof CompleteMultipartUploadRequest && hasErrorCode(statusCode, errorCode)) { // complete mpu error code might be in body when status code is 200 // double check to head object only works in big data job case which key is not same. String key = ((CompleteMultipartUploadRequest) request).getKey(); @@ -1469,6 +1467,21 @@ private Object callCOSClientWithRetry(X request) throws CosServiceException, throw cse; } } + + // mpu might occur 503 access time out but already completed, + // if direct retry may occur 403 not found the upload id. + if (request instanceof CompleteMultipartUploadRequest && statusCode == 503) { + String key = ((CompleteMultipartUploadRequest) request).getKey(); + FileMetadata fileMetadata = this.queryObjectMetadata(key); + if (null != fileMetadata) { + // if file exist direct return. + LOG.info("complete mpu error might access time out, " + + "but key {} already exist, length {}", + key, fileMetadata.getLength()); + return new CompleteMultipartUploadResult(); + } + } + Thread.sleep( ThreadLocalRandom.current().nextLong(sleepLeast, sleepBound)); ++retryIndex; @@ -1488,6 +1501,10 @@ private Object callCOSClientWithRetry(X request) throws CosServiceException, } } + private boolean hasErrorCode(int statusCode, String errCode) { + return statusCode / 100 == 2 && errCode != null && !errCode.isEmpty(); + } + private static String ensureValidAttributeName(String attributeName) { return attributeName.replace('.', '-').toLowerCase(); }