diff --git a/compile.sh b/compile.sh index e84cc24e..155ec627 100644 --- a/compile.sh +++ b/compile.sh @@ -2,7 +2,7 @@ base_dir=$(cd `dirname $0`;pwd) cd ${base_dir} -hadoop_version_array=("2.6.5" "2.7.5" "2.8.5" "3.1.0" "3.3.0") +hadoop_version_array=("2.6.0" "2.6.5" "2.7.5" "2.8.5" "3.1.0" "3.3.0") origin_version=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec) diff --git a/pom.xml b/pom.xml index 9d4df3bf..8fea7f91 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.qcloud.cos hadoop-cos - 5.9.0 + 5.9.1 jar Apache Hadoop Tencent Qcloud COS Support @@ -41,7 +41,7 @@ 1.7 1.7 3.3.0 - 5.6.32 + 5.6.35 24.1.1-jre 3.1 4.8 diff --git a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java index ee5d0855..f4610e2a 100644 --- a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java +++ b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java @@ -420,7 +420,8 @@ public FileStatus getFileStatus(Path f) throws IOException { return newDirectory(absolutePath); } - FileMetadata meta = store.retrieveMetadata(key); + CosResultInfo headInfo = new CosResultInfo(); + FileMetadata meta = store.retrieveMetadata(key, headInfo); if (meta != null) { if (meta.isFile()) { LOG.debug("Retrieve the cos key [{}] to find that it is a file.", key); @@ -435,12 +436,19 @@ public FileStatus getFileStatus(Path f) throws IOException { key += PATH_DELIMITER; } LOG.debug("List the cos key [{}] to judge whether it is a directory or not.", key); - PartialListing listing = store.list(key, 1); + CosResultInfo listInfo = new CosResultInfo(); + PartialListing listing = store.list(key, 1, listInfo); if (listing.getFiles().length > 0 || listing.getCommonPrefixes().length > 0) { LOG.debug("List the cos key [{}] to find that it is a directory.", key); return newDirectory(absolutePath); } + if (listInfo.isKeySameToPrefix()) { + LOG.info("List the cos key [{}] same to prefix, head-id:[{}], " + + "list-id:[{}], list-type:[{}], thread-id:[{}], thread-name:[{}]", + key, headInfo.getRequestID(), listInfo.getRequestID(), + listInfo.isKeySameToPrefix(), Thread.currentThread().getId(), Thread.currentThread().getName()); + } LOG.debug("Can not find the cos key [{}] on COS.", key); throw new FileNotFoundException("No such file or directory '" + absolutePath + "'"); diff --git a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java index e5e27649..3e65d3bc 100644 --- a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java +++ b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java @@ -10,7 +10,7 @@ @InterfaceStability.Unstable public class CosNConfigKeys extends CommonConfigurationKeys { public static final String USER_AGENT = "fs.cosn.user.agent"; - public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v5.8.7"; + public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v5.9.1"; public static final String TENCENT_EMR_VERSION_KEY = "fs.emr.version"; diff --git a/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java b/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java index 32ae8c4c..6fae8275 100644 --- a/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java +++ b/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java @@ -449,6 +449,11 @@ public int compare(PartETag o1, PartETag o2) { } private FileMetadata queryObjectMetadata(String key) throws IOException { + return queryObjectMetadata(key, null); + } + + private FileMetadata queryObjectMetadata(String key, + CosResultInfo info) throws IOException { LOG.debug("Query Object metadata. cos key: {}.", key); GetObjectMetadataRequest getObjectMetadataRequest = new GetObjectMetadataRequest(bucketName, key); @@ -494,10 +499,17 @@ private FileMetadata queryObjectMetadata(String key) throws IOException { new FileMetadata(key, fileSize, mtime, !key.endsWith(PATH_DELIMITER), ETag, crc64ecm, crc32cm, versionId, objectMetadata.getStorageClass(), userMetadata); + // record the last request result info + if (info != null) { + info.setRequestID(objectMetadata.getRequestId()); + } LOG.debug("Retrieve the file metadata. cos key: {}, ETag:{}, length:{}, crc64ecm: {}.", key, objectMetadata.getETag(), objectMetadata.getContentLength(), objectMetadata.getCrc64Ecma()); return fileMetadata; } catch (CosServiceException e) { + if (info != null) { + info.setRequestID(e.getRequestId()); + } if (e.getStatusCode() != 404) { String errorMsg = String.format("Retrieve the file metadata file failure. " + @@ -510,19 +522,26 @@ private FileMetadata queryObjectMetadata(String key) throws IOException { @Override public FileMetadata retrieveMetadata(String key) throws IOException { + return retrieveMetadata(key, null); + } + + // this method only used in getFileStatus to get the head request result info + @Override + public FileMetadata retrieveMetadata(String key, + CosResultInfo info) throws IOException { if (key.endsWith(PATH_DELIMITER)) { key = key.substring(0, key.length() - 1); } if (!key.isEmpty()) { - FileMetadata fileMetadata = queryObjectMetadata(key); + FileMetadata fileMetadata = queryObjectMetadata(key, info); if (fileMetadata != null) { return fileMetadata; } } // judge if the key is directory key = key + PATH_DELIMITER; - return queryObjectMetadata(key); + return queryObjectMetadata(key, info); } @Override @@ -782,15 +801,26 @@ public boolean retrieveBlock(String key, long byteRangeStart, @Override public PartialListing list(String prefix, int maxListingLength) throws IOException { - return list(prefix, maxListingLength, null, false); + return list(prefix, maxListingLength, null); + } + + @Override + public PartialListing list(String prefix, int maxListingLength, CosResultInfo info) throws IOException { + return list(prefix, maxListingLength, null, false, info); } @Override public PartialListing list(String prefix, int maxListingLength, String priorLastKey, boolean recurse) throws IOException { + return list(prefix, maxListingLength, priorLastKey, recurse, null); + } - return list(prefix, recurse ? null : PATH_DELIMITER, maxListingLength, priorLastKey); + @Override + public PartialListing list(String prefix, int maxListingLength, + String priorLastKey, + boolean recurse, CosResultInfo info) throws IOException { + return list(prefix, recurse ? null : PATH_DELIMITER, maxListingLength, priorLastKey, info); } /** @@ -806,7 +836,7 @@ public PartialListing list(String prefix, int maxListingLength, private PartialListing list(String prefix, String delimiter, int maxListingLength, - String priorLastKey) throws IOException { + String priorLastKey, CosResultInfo info) throws IOException { LOG.debug("List the cos key prefix: {}, max listing length: {}, delimiter: {}, prior last key: {}.", prefix, delimiter, maxListingLength, priorLastKey); @@ -846,12 +876,14 @@ private PartialListing list(String prefix, String delimiter, ArrayList commonPrefixArray = new ArrayList(); List summaries = objectListing.getObjectSummaries(); + boolean isKeySamePrefix = false; for (COSObjectSummary cosObjectSummary : summaries) { String filePath = cosObjectSummary.getKey(); if (!filePath.startsWith(PATH_DELIMITER)) { filePath = PATH_DELIMITER + filePath; } if (filePath.equals(prefix)) { + isKeySamePrefix = true; continue; } long mtime = 0; @@ -890,10 +922,20 @@ private PartialListing list(String prefix, String delimiter, // 如果truncated为false, 则表明已经遍历完 if (!objectListing.isTruncated()) { - return new PartialListing(null, fileMetadata, commonPrefixMetaData); + PartialListing ret = new PartialListing(null, fileMetadata, commonPrefixMetaData); + if (info != null) { + info.setRequestID(objectListing.getRequestId()); + info.setKeySameToPrefix(isKeySamePrefix); + } + return ret; } else { - return new PartialListing(objectListing.getNextMarker(), + PartialListing ret = new PartialListing(objectListing.getNextMarker(), fileMetadata, commonPrefixMetaData); + if (info != null) { + info.setRequestID(objectListing.getRequestId()); + info.setKeySameToPrefix(isKeySamePrefix); + } + return ret; } } diff --git a/src/main/java/org/apache/hadoop/fs/CosResultInfo.java b/src/main/java/org/apache/hadoop/fs/CosResultInfo.java new file mode 100644 index 00000000..90d91a9a --- /dev/null +++ b/src/main/java/org/apache/hadoop/fs/CosResultInfo.java @@ -0,0 +1,29 @@ +package org.apache.hadoop.fs; + +/** + * Used to record the cos client query result + */ +public class CosResultInfo { + private String requestID; + private boolean isKeySameToPrefix; + + CosResultInfo() { + requestID = ""; + isKeySameToPrefix = false; + } + + public void setRequestID(String requestID) { + this.requestID = requestID; + } + public String getRequestID() { + return this.requestID; + } + + public boolean isKeySameToPrefix() { + return this.isKeySameToPrefix; + } + + public void setKeySameToPrefix(boolean isKeySameToPrefix) { + this.isKeySameToPrefix = isKeySameToPrefix; + } +} diff --git a/src/main/java/org/apache/hadoop/fs/NativeFileSystemStore.java b/src/main/java/org/apache/hadoop/fs/NativeFileSystemStore.java index b4cf7475..5c9b812d 100644 --- a/src/main/java/org/apache/hadoop/fs/NativeFileSystemStore.java +++ b/src/main/java/org/apache/hadoop/fs/NativeFileSystemStore.java @@ -45,6 +45,8 @@ PartETag uploadPart(InputStream inputStream, String key, String uploadId, FileMetadata retrieveMetadata(String key) throws IOException; + FileMetadata retrieveMetadata(String key, CosResultInfo info) throws IOException; + byte[] retrieveAttribute(String key, String attribute) throws IOException; void storeDirAttribute(String key, String attribute, byte[] value) throws IOException; @@ -69,10 +71,16 @@ boolean retrieveBlock(String key, long byteRangeStart, long blockSize, PartialListing list(String prefix, int maxListingLength) throws IOException; + PartialListing list(String prefix, int maxListingLength, CosResultInfo info) throws IOException; + PartialListing list(String prefix, int maxListingLength, String priorLastKey, boolean recursive) throws IOException; + PartialListing list(String prefix, int maxListingLength, + String priorLastKey, boolean recursive, CosResultInfo info) + throws IOException; + void delete(String key) throws IOException; void copy(String srcKey, String dstKey) throws IOException;