From 81b8c9b152f57325cb23f11234aa53e51275f5d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?alantong=28=E4=BD=9F=E6=98=8E=E8=BE=BE=29?= Date: Fri, 8 Jan 2021 21:31:03 +0800 Subject: [PATCH] keep the way get result same to the head --- compile.sh | 2 +- .../org/apache/hadoop/fs/CosFileSystem.java | 13 +++++---- .../hadoop/fs/CosNativeFileSystemStore.java | 29 ++++++++++++++----- .../hadoop/fs/NativeFileSystemStore.java | 6 ++++ .../org/apache/hadoop/fs/PartialListing.java | 18 ------------ 5 files changed, 36 insertions(+), 32 deletions(-) diff --git a/compile.sh b/compile.sh index e84cc24e..155ec627 100644 --- a/compile.sh +++ b/compile.sh @@ -2,7 +2,7 @@ base_dir=$(cd `dirname $0`;pwd) cd ${base_dir} -hadoop_version_array=("2.6.5" "2.7.5" "2.8.5" "3.1.0" "3.3.0") +hadoop_version_array=("2.6.0" "2.6.5" "2.7.5" "2.8.5" "3.1.0" "3.3.0") origin_version=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec) diff --git a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java index 2ec263ba..f4610e2a 100644 --- a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java +++ b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java @@ -420,8 +420,8 @@ public FileStatus getFileStatus(Path f) throws IOException { return newDirectory(absolutePath); } - CosResultInfo info = new CosResultInfo(); - FileMetadata meta = store.retrieveMetadata(key, info); + CosResultInfo headInfo = new CosResultInfo(); + FileMetadata meta = store.retrieveMetadata(key, headInfo); if (meta != null) { if (meta.isFile()) { LOG.debug("Retrieve the cos key [{}] to find that it is a file.", key); @@ -436,17 +436,18 @@ public FileStatus getFileStatus(Path f) throws IOException { key += PATH_DELIMITER; } LOG.debug("List the cos key [{}] to judge whether it is a directory or not.", key); - PartialListing listing = store.list(key, 1); + CosResultInfo listInfo = new CosResultInfo(); + PartialListing listing = store.list(key, 1, listInfo); if (listing.getFiles().length > 0 || listing.getCommonPrefixes().length > 0) { LOG.debug("List the cos key [{}] to find that it is a directory.", key); return newDirectory(absolutePath); } - if (listing.isKeySamePrefix()) { + if (listInfo.isKeySameToPrefix()) { LOG.info("List the cos key [{}] same to prefix, head-id:[{}], " + "list-id:[{}], list-type:[{}], thread-id:[{}], thread-name:[{}]", - key, info.getRequestID(), listing.getRequestID(), - listing.isKeySamePrefix(), Thread.currentThread().getId(), Thread.currentThread().getName()); + key, headInfo.getRequestID(), listInfo.getRequestID(), + listInfo.isKeySameToPrefix(), Thread.currentThread().getId(), Thread.currentThread().getName()); } LOG.debug("Can not find the cos key [{}] on COS.", key); diff --git a/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java b/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java index 43e1797f..6fae8275 100644 --- a/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java +++ b/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java @@ -801,15 +801,26 @@ public boolean retrieveBlock(String key, long byteRangeStart, @Override public PartialListing list(String prefix, int maxListingLength) throws IOException { - return list(prefix, maxListingLength, null, false); + return list(prefix, maxListingLength, null); + } + + @Override + public PartialListing list(String prefix, int maxListingLength, CosResultInfo info) throws IOException { + return list(prefix, maxListingLength, null, false, info); } @Override public PartialListing list(String prefix, int maxListingLength, String priorLastKey, boolean recurse) throws IOException { + return list(prefix, maxListingLength, priorLastKey, recurse, null); + } - return list(prefix, recurse ? null : PATH_DELIMITER, maxListingLength, priorLastKey); + @Override + public PartialListing list(String prefix, int maxListingLength, + String priorLastKey, + boolean recurse, CosResultInfo info) throws IOException { + return list(prefix, recurse ? null : PATH_DELIMITER, maxListingLength, priorLastKey, info); } /** @@ -825,7 +836,7 @@ public PartialListing list(String prefix, int maxListingLength, private PartialListing list(String prefix, String delimiter, int maxListingLength, - String priorLastKey) throws IOException { + String priorLastKey, CosResultInfo info) throws IOException { LOG.debug("List the cos key prefix: {}, max listing length: {}, delimiter: {}, prior last key: {}.", prefix, delimiter, maxListingLength, priorLastKey); @@ -912,14 +923,18 @@ private PartialListing list(String prefix, String delimiter, // 如果truncated为false, 则表明已经遍历完 if (!objectListing.isTruncated()) { PartialListing ret = new PartialListing(null, fileMetadata, commonPrefixMetaData); - ret.setKeySamePrefix(isKeySamePrefix); - ret.setRequestID(objectListing.getRequestId()); + if (info != null) { + info.setRequestID(objectListing.getRequestId()); + info.setKeySameToPrefix(isKeySamePrefix); + } return ret; } else { PartialListing ret = new PartialListing(objectListing.getNextMarker(), fileMetadata, commonPrefixMetaData); - ret.setKeySamePrefix(isKeySamePrefix); - ret.setRequestID(objectListing.getRequestId()); + if (info != null) { + info.setRequestID(objectListing.getRequestId()); + info.setKeySameToPrefix(isKeySamePrefix); + } return ret; } } diff --git a/src/main/java/org/apache/hadoop/fs/NativeFileSystemStore.java b/src/main/java/org/apache/hadoop/fs/NativeFileSystemStore.java index 31664ff4..5c9b812d 100644 --- a/src/main/java/org/apache/hadoop/fs/NativeFileSystemStore.java +++ b/src/main/java/org/apache/hadoop/fs/NativeFileSystemStore.java @@ -71,10 +71,16 @@ boolean retrieveBlock(String key, long byteRangeStart, long blockSize, PartialListing list(String prefix, int maxListingLength) throws IOException; + PartialListing list(String prefix, int maxListingLength, CosResultInfo info) throws IOException; + PartialListing list(String prefix, int maxListingLength, String priorLastKey, boolean recursive) throws IOException; + PartialListing list(String prefix, int maxListingLength, + String priorLastKey, boolean recursive, CosResultInfo info) + throws IOException; + void delete(String key) throws IOException; void copy(String srcKey, String dstKey) throws IOException; diff --git a/src/main/java/org/apache/hadoop/fs/PartialListing.java b/src/main/java/org/apache/hadoop/fs/PartialListing.java index a6c458a9..3e782fe7 100644 --- a/src/main/java/org/apache/hadoop/fs/PartialListing.java +++ b/src/main/java/org/apache/hadoop/fs/PartialListing.java @@ -24,30 +24,12 @@ public class PartialListing { private final String priorLastKey; private final FileMetadata[] files; private final FileMetadata[] commonPrefixes; - private CosResultInfo resultInfo; public PartialListing(String priorLastKey, FileMetadata[] files, FileMetadata[] commonPrefixes) { this.priorLastKey = priorLastKey; this.files = files; this.commonPrefixes = commonPrefixes; - this.resultInfo = new CosResultInfo(); - } - - public void setKeySamePrefix(boolean isKeySamePrefix) { - this.resultInfo.setKeySameToPrefix(isKeySamePrefix); - } - - public boolean isKeySamePrefix() { - return this.resultInfo.isKeySameToPrefix(); - } - - public void setRequestID(String requestID) { - this.resultInfo.setRequestID(requestID); - } - - public String getRequestID() { - return this.resultInfo.getRequestID(); } public FileMetadata[] getFiles() {