fix appid and expose connection relate config (#139)

tencentyun · May 7, 2024 · d7dffc4 · d7dffc4
1 parent 9a094cf
commit d7dffc4
Show file tree

Hide file tree

Showing 6 changed files with 34 additions and 10 deletions.
diff --git a/pom.xml b/pom.xml
@@ -91,7 +91,7 @@
         <dependency>
             <groupId>com.qcloud</groupId>
             <artifactId>chdfs_hadoop_plugin_network</artifactId>
-            <version>3.2</version>
+            <version>3.5</version>
             <scope>compile</scope>
         </dependency>
 

diff --git a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java
@@ -140,6 +140,21 @@ public class CosNConfigKeys extends CommonConfigurationKeys {
     public static final String MAX_CONNECTION_NUM = "fs.cosn.max.connection.num";
     public static final int DEFAULT_MAX_CONNECTION_NUM = 1024;
 
+    public static final String COSN_USE_SHORT_CONNECTION = "fs.cosn.use.short.connection";
+
+    public static final boolean DEFAULT_COSN_USE_SHORT_CONNECTION = false;
+
+    public static final String COSN_CONNECTION_REQUEST_TIMEOUT = "fs.cosn.connection.request.timeout";
+    public static final int DEFAULT_COSN_CONNECTION_REQUEST_TIMEOUT = -1;
+
+    public static final String COSN_CONNECTION_TIMEOUT = "fs.cosn.connection.timeout";
+
+    public static final int DEFAULT_COSN_CONNECTION_TIMEOUT = 10000;
+
+    public static final String COSN_IDLE_CONNECTION_ALIVE = "fs.cosn.idle.connection.alive";
+
+    public static final int DEFAULT_COSN_IDLE_CONNECTION_ALIVE = 60000;
+
     public static final String CUSTOMER_DOMAIN = "fs.cosn.customer.domain";
     public static final String COSN_SERVER_SIDE_ENCRYPTION_ALGORITHM = "fs.cosn.server-side-encryption.algorithm";
     public static final String COSN_SERVER_SIDE_ENCRYPTION_KEY = "fs.cosn.server-side-encryption.key";

diff --git a/src/main/java/org/apache/hadoop/fs/CosNUtils.java b/src/main/java/org/apache/hadoop/fs/CosNUtils.java
@@ -19,9 +19,6 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import javax.crypto.KeyGenerator;
-import javax.crypto.SecretKey;
-import javax.crypto.spec.SecretKeySpec;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -220,7 +217,7 @@ private static Method getFactoryMethod(Class<?> cl, Class<?> returnType,
         }
     }
 
-    public static String getBucketNameWithoutAppid(String originBucketName, String appidStr) {
+    public static String getBucketNameWithAppid(String originBucketName, String appidStr) {
         if (appidStr == null || appidStr.isEmpty()) {
             return originBucketName;
         }

diff --git a/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java b/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java
@@ -292,11 +292,24 @@ private void initCOSClient(URI uri, Configuration conf) throws IOException {
         LOG.info("hadoop cos retry times: {}, cos client retry times: {}",
                 this.maxRetryTimes, clientMaxRetryTimes);
 
-        // 设置连接池的最大连接数目
+        if (conf.getBoolean(CosNConfigKeys.COSN_USE_SHORT_CONNECTION,
+                CosNConfigKeys.DEFAULT_COSN_USE_SHORT_CONNECTION)) {
+           config.setShortConnection();
+        }
+
         config.setMaxConnectionsCount(
                 conf.getInt(
                         CosNConfigKeys.MAX_CONNECTION_NUM,
                         CosNConfigKeys.DEFAULT_MAX_CONNECTION_NUM));
+        config.setConnectionRequestTimeout(
+                conf.getInt(CosNConfigKeys.COSN_CONNECTION_REQUEST_TIMEOUT,
+                        CosNConfigKeys.DEFAULT_COSN_CONNECTION_REQUEST_TIMEOUT));
+        config.setConnectionTimeout(
+                conf.getInt(CosNConfigKeys.COSN_CONNECTION_TIMEOUT,
+                        CosNConfigKeys.DEFAULT_COSN_CONNECTION_TIMEOUT));
+        config.setIdleConnectionAlive(
+                conf.getInt(CosNConfigKeys.COSN_IDLE_CONNECTION_ALIVE,
+                        CosNConfigKeys.DEFAULT_COSN_IDLE_CONNECTION_ALIVE));
 
         // 设置是否进行服务器端加密
         String serverSideEncryptionAlgorithm = conf.get(CosNConfigKeys.COSN_SERVER_SIDE_ENCRYPTION_ALGORITHM, "");

diff --git a/src/main/java/org/apache/hadoop/fs/RangerCredentialsClient.java b/src/main/java/org/apache/hadoop/fs/RangerCredentialsClient.java
@@ -74,7 +74,7 @@ public void doCheckPermission(Path f, RangerAccessType rangerAccessType, String
         }
 
         PermissionRequest permissionReq = new PermissionRequest(ServiceType.COS, accessType,
-                CosNUtils.getBucketNameWithoutAppid(this.bucket, this.conf.get(CosNConfigKeys.COSN_APPID_KEY)),
+                CosNUtils.getBucketNameWithAppid(this.bucket, this.conf.get(CosNConfigKeys.COSN_APPID_KEY)),
                 allowKey, "", "");
         boolean allowed = false;
         String checkPermissionActualUserName = ownerId;
@@ -101,7 +101,7 @@ public void doCheckCustomAuth(Configuration conf) throws IOException {
         }
 
         GetSTSResponse stsResp = this.rangerQcloudObjectStorageStorageClient.getSTS(bucketRegion,
-                bucket);
+                CosNUtils.getBucketNameWithAppid(this.bucket, this.conf.get(CosNConfigKeys.COSN_APPID_KEY)));
         if (!stsResp.isCheckAuthPass()) {
             throw new IOException(String.format("Permission denied, [operation: %s], please check user and " +
                     "password", CUSTOM_AUTHENTICATION));

diff --git a/src/main/java/org/apache/hadoop/fs/auth/RangerCredentialsProvider.java b/src/main/java/org/apache/hadoop/fs/auth/RangerCredentialsProvider.java
@@ -7,7 +7,6 @@
 import org.apache.hadoop.fs.CosNConfigKeys;
 import org.apache.hadoop.fs.CosNUtils;
 import org.apache.hadoop.fs.RangerCredentialsClient;
-import org.apache.hadoop.fs.cosn.ranger.client.RangerQcloudObjectStorageClient;
 import org.apache.hadoop.fs.cosn.ranger.security.sts.GetSTSResponse;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -32,7 +31,7 @@ public RangerCredentialsProvider(@Nullable URI uri, Configuration conf,
         super(uri, conf);
         if (null != conf) {
             this.appId = conf.get(CosNConfigKeys.COSN_APPID_KEY);
-            this.bucketNameWithoutAppid = CosNUtils.getBucketNameWithoutAppid(
+            this.bucketNameWithoutAppid = CosNUtils.getBucketNameWithAppid(
                     uri.getHost(), conf.get(CosNConfigKeys.COSN_APPID_KEY));
             this.bucketRegion = conf.get(CosNConfigKeys.COSN_REGION_KEY);
             // native store keep the ranger client not null.