Merge pull request #25 from vintmd/zuoyebang400

add the key requel prefix list and head requestid
tencentyun · Jan 8, 2021 · 428de4b · 428de4b
2 parents fe5a425 + 81b8c9b
commit 428de4b
Show file tree

Hide file tree

Showing 7 changed files with 100 additions and 13 deletions.
diff --git a/compile.sh b/compile.sh
@@ -2,7 +2,7 @@
 
 base_dir=$(cd `dirname $0`;pwd)
 cd ${base_dir}
-hadoop_version_array=("2.6.5" "2.7.5" "2.8.5" "3.1.0" "3.3.0")
+hadoop_version_array=("2.6.0" "2.6.5" "2.7.5" "2.8.5" "3.1.0" "3.3.0")
 
 origin_version=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec)
 

diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>com.qcloud.cos</groupId>
     <artifactId>hadoop-cos</artifactId>
-    <version>5.9.0</version>
+    <version>5.9.1</version>
     <packaging>jar</packaging>
 
     <name>Apache Hadoop Tencent Qcloud COS Support</name>
@@ -41,7 +41,7 @@
         <maven.compiler.source>1.7</maven.compiler.source>
         <maven.compiler.target>1.7</maven.compiler.target>
         <hadoop.version>3.3.0</hadoop.version>
-        <cos_api.version>5.6.32</cos_api.version>
+        <cos_api.version>5.6.35</cos_api.version>
         <google.guava.version>24.1.1-jre</google.guava.version>
         <commons_lang3.version>3.1</commons_lang3.version>
         <junit.version>4.8</junit.version>

diff --git a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java
@@ -420,7 +420,8 @@ public FileStatus getFileStatus(Path f) throws IOException {
             return newDirectory(absolutePath);
         }
 
-        FileMetadata meta = store.retrieveMetadata(key);
+        CosResultInfo headInfo = new CosResultInfo();
+        FileMetadata meta = store.retrieveMetadata(key, headInfo);
         if (meta != null) {
             if (meta.isFile()) {
                 LOG.debug("Retrieve the cos key [{}] to find that it is a file.", key);
@@ -435,12 +436,19 @@ public FileStatus getFileStatus(Path f) throws IOException {
             key += PATH_DELIMITER;
         }
         LOG.debug("List the cos key [{}] to judge whether it is a directory or not.", key);
-        PartialListing listing = store.list(key, 1);
+        CosResultInfo listInfo = new CosResultInfo();
+        PartialListing listing = store.list(key, 1, listInfo);
         if (listing.getFiles().length > 0 || listing.getCommonPrefixes().length > 0) {
             LOG.debug("List the cos key [{}] to find that it is a directory.", key);
             return newDirectory(absolutePath);
         }
 
+        if (listInfo.isKeySameToPrefix()) {
+            LOG.info("List the cos key [{}] same to prefix, head-id:[{}], " +
+                            "list-id:[{}], list-type:[{}], thread-id:[{}], thread-name:[{}]",
+                    key, headInfo.getRequestID(), listInfo.getRequestID(),
+                    listInfo.isKeySameToPrefix(), Thread.currentThread().getId(), Thread.currentThread().getName());
+        }
         LOG.debug("Can not find the cos key [{}] on COS.", key);
 
         throw new FileNotFoundException("No such file or directory '" + absolutePath + "'");

diff --git a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java
@@ -10,7 +10,7 @@
 @InterfaceStability.Unstable
 public class CosNConfigKeys extends CommonConfigurationKeys {
     public static final String USER_AGENT = "fs.cosn.user.agent";
-    public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v5.8.7";
+    public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v5.9.1";
 
     public static final String TENCENT_EMR_VERSION_KEY = "fs.emr.version";
 

diff --git a/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java b/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java
@@ -449,6 +449,11 @@ public int compare(PartETag o1, PartETag o2) {
     }
 
     private FileMetadata queryObjectMetadata(String key) throws IOException {
+        return queryObjectMetadata(key, null);
+    }
+
+    private FileMetadata queryObjectMetadata(String key,
+                                             CosResultInfo info) throws IOException {
         LOG.debug("Query Object metadata. cos key: {}.", key);
         GetObjectMetadataRequest getObjectMetadataRequest =
                 new GetObjectMetadataRequest(bucketName, key);
@@ -494,10 +499,17 @@ private FileMetadata queryObjectMetadata(String key) throws IOException {
                     new FileMetadata(key, fileSize, mtime, !key.endsWith(PATH_DELIMITER),
                             ETag, crc64ecm, crc32cm, versionId,
                             objectMetadata.getStorageClass(), userMetadata);
+            // record the last request result info
+            if (info != null)  {
+                info.setRequestID(objectMetadata.getRequestId());
+            }
             LOG.debug("Retrieve the file metadata. cos key: {}, ETag:{}, length:{}, crc64ecm: {}.", key,
                     objectMetadata.getETag(), objectMetadata.getContentLength(), objectMetadata.getCrc64Ecma());
             return fileMetadata;
         } catch (CosServiceException e) {
+            if (info != null) {
+                info.setRequestID(e.getRequestId());
+            }
             if (e.getStatusCode() != 404) {
                 String errorMsg =
                         String.format("Retrieve the file metadata file failure. " +
@@ -510,19 +522,26 @@ private FileMetadata queryObjectMetadata(String key) throws IOException {
 
     @Override
     public FileMetadata retrieveMetadata(String key) throws IOException {
+        return retrieveMetadata(key, null);
+    }
+
+    // this method only used in getFileStatus to get the head request result info
+    @Override
+    public FileMetadata retrieveMetadata(String key,
+                                         CosResultInfo info) throws IOException {
         if (key.endsWith(PATH_DELIMITER)) {
             key = key.substring(0, key.length() - 1);
         }
 
         if (!key.isEmpty()) {
-            FileMetadata fileMetadata = queryObjectMetadata(key);
+            FileMetadata fileMetadata = queryObjectMetadata(key, info);
             if (fileMetadata != null) {
                 return fileMetadata;
             }
         }
         // judge if the key is directory
         key = key + PATH_DELIMITER;
-        return queryObjectMetadata(key);
+        return queryObjectMetadata(key, info);
     }
 
     @Override
@@ -782,15 +801,26 @@ public boolean retrieveBlock(String key, long byteRangeStart,
 
     @Override
     public PartialListing list(String prefix, int maxListingLength) throws IOException {
-        return list(prefix, maxListingLength, null, false);
+        return list(prefix, maxListingLength, null);
+    }
+
+    @Override
+    public PartialListing list(String prefix, int maxListingLength, CosResultInfo info) throws IOException {
+        return list(prefix, maxListingLength, null, false, info);
     }
 
     @Override
     public PartialListing list(String prefix, int maxListingLength,
                                String priorLastKey,
                                boolean recurse) throws IOException {
+        return list(prefix, maxListingLength, priorLastKey, recurse, null);
+    }
 
-        return list(prefix, recurse ? null : PATH_DELIMITER, maxListingLength, priorLastKey);
+    @Override
+    public PartialListing list(String prefix, int maxListingLength,
+                               String priorLastKey,
+                               boolean recurse, CosResultInfo info) throws IOException {
+        return list(prefix, recurse ? null : PATH_DELIMITER, maxListingLength, priorLastKey, info);
     }
 
     /**
@@ -806,7 +836,7 @@ public PartialListing list(String prefix, int maxListingLength,
 
     private PartialListing list(String prefix, String delimiter,
                                 int maxListingLength,
-                                String priorLastKey) throws IOException {
+                                String priorLastKey, CosResultInfo info) throws IOException {
         LOG.debug("List the cos key prefix: {}, max listing length: {}, delimiter: {}, prior last key: {}.",
                 prefix,
                 delimiter, maxListingLength, priorLastKey);
@@ -846,12 +876,14 @@ private PartialListing list(String prefix, String delimiter,
         ArrayList<FileMetadata> commonPrefixArray =
                 new ArrayList<FileMetadata>();
         List<COSObjectSummary> summaries = objectListing.getObjectSummaries();
+        boolean isKeySamePrefix = false;
         for (COSObjectSummary cosObjectSummary : summaries) {
             String filePath = cosObjectSummary.getKey();
             if (!filePath.startsWith(PATH_DELIMITER)) {
                 filePath = PATH_DELIMITER + filePath;
             }
             if (filePath.equals(prefix)) {
+                isKeySamePrefix = true;
                 continue;
             }
             long mtime = 0;
@@ -890,10 +922,20 @@ private PartialListing list(String prefix, String delimiter,
 
         // 如果truncated为false, 则表明已经遍历完
         if (!objectListing.isTruncated()) {
-            return new PartialListing(null, fileMetadata, commonPrefixMetaData);
+            PartialListing ret = new PartialListing(null, fileMetadata, commonPrefixMetaData);
+            if (info != null) {
+                info.setRequestID(objectListing.getRequestId());
+                info.setKeySameToPrefix(isKeySamePrefix);
+            }
+            return ret;
         } else {
-            return new PartialListing(objectListing.getNextMarker(),
+            PartialListing ret =  new PartialListing(objectListing.getNextMarker(),
                     fileMetadata, commonPrefixMetaData);
+            if (info != null) {
+                info.setRequestID(objectListing.getRequestId());
+                info.setKeySameToPrefix(isKeySamePrefix);
+            }
+            return ret;
         }
     }
 

diff --git a/src/main/java/org/apache/hadoop/fs/CosResultInfo.java b/src/main/java/org/apache/hadoop/fs/CosResultInfo.java
@@ -0,0 +1,29 @@
+package org.apache.hadoop.fs;
+
+/**
+ * Used to record the cos client query result
+ */
+public class CosResultInfo {
+    private String requestID;
+    private boolean isKeySameToPrefix;
+
+    CosResultInfo() {
+        requestID = "";
+        isKeySameToPrefix = false;
+    }
+
+    public void setRequestID(String requestID) {
+        this.requestID = requestID;
+    }
+    public String getRequestID() {
+        return this.requestID;
+    }
+
+    public boolean isKeySameToPrefix() {
+        return this.isKeySameToPrefix;
+    }
+
+    public void setKeySameToPrefix(boolean isKeySameToPrefix) {
+        this.isKeySameToPrefix = isKeySameToPrefix;
+    }
+}
diff --git a/src/main/java/org/apache/hadoop/fs/NativeFileSystemStore.java b/src/main/java/org/apache/hadoop/fs/NativeFileSystemStore.java
@@ -45,6 +45,8 @@ PartETag uploadPart(InputStream inputStream, String key, String uploadId,
 
     FileMetadata retrieveMetadata(String key) throws IOException;
 
+    FileMetadata retrieveMetadata(String key, CosResultInfo info) throws IOException;
+
     byte[] retrieveAttribute(String key, String attribute) throws IOException;
 
     void storeDirAttribute(String key, String attribute, byte[] value) throws IOException;
@@ -69,10 +71,16 @@ boolean retrieveBlock(String key, long byteRangeStart, long blockSize,
 
     PartialListing list(String prefix, int maxListingLength) throws IOException;
 
+    PartialListing list(String prefix, int maxListingLength, CosResultInfo info) throws IOException;
+
     PartialListing list(String prefix, int maxListingLength,
                         String priorLastKey, boolean recursive)
             throws IOException;
 
+    PartialListing list(String prefix, int maxListingLength,
+                        String priorLastKey, boolean recursive, CosResultInfo info)
+            throws IOException;
+
     void delete(String key) throws IOException;
 
     void copy(String srcKey, String dstKey) throws IOException;