diff --git a/pom.xml b/pom.xml index fedb8657..2793caed 100644 --- a/pom.xml +++ b/pom.xml @@ -91,7 +91,7 @@ com.qcloud chdfs_hadoop_plugin_network - 3.2 + 3.5 compile diff --git a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java index 5ef27945..efb54e2d 100644 --- a/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java +++ b/src/main/java/org/apache/hadoop/fs/CosNConfigKeys.java @@ -140,6 +140,21 @@ public class CosNConfigKeys extends CommonConfigurationKeys { public static final String MAX_CONNECTION_NUM = "fs.cosn.max.connection.num"; public static final int DEFAULT_MAX_CONNECTION_NUM = 1024; + public static final String COSN_USE_SHORT_CONNECTION = "fs.cosn.use.short.connection"; + + public static final boolean DEFAULT_COSN_USE_SHORT_CONNECTION = false; + + public static final String COSN_CONNECTION_REQUEST_TIMEOUT = "fs.cosn.connection.request.timeout"; + public static final int DEFAULT_COSN_CONNECTION_REQUEST_TIMEOUT = -1; + + public static final String COSN_CONNECTION_TIMEOUT = "fs.cosn.connection.timeout"; + + public static final int DEFAULT_COSN_CONNECTION_TIMEOUT = 10000; + + public static final String COSN_IDLE_CONNECTION_ALIVE = "fs.cosn.idle.connection.alive"; + + public static final int DEFAULT_COSN_IDLE_CONNECTION_ALIVE = 60000; + public static final String CUSTOMER_DOMAIN = "fs.cosn.customer.domain"; public static final String COSN_SERVER_SIDE_ENCRYPTION_ALGORITHM = "fs.cosn.server-side-encryption.algorithm"; public static final String COSN_SERVER_SIDE_ENCRYPTION_KEY = "fs.cosn.server-side-encryption.key"; diff --git a/src/main/java/org/apache/hadoop/fs/CosNUtils.java b/src/main/java/org/apache/hadoop/fs/CosNUtils.java index a1545c0a..20ac34b6 100644 --- a/src/main/java/org/apache/hadoop/fs/CosNUtils.java +++ b/src/main/java/org/apache/hadoop/fs/CosNUtils.java @@ -19,9 +19,6 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; -import javax.crypto.KeyGenerator; -import javax.crypto.SecretKey; -import javax.crypto.spec.SecretKeySpec; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -220,7 +217,7 @@ private static Method getFactoryMethod(Class cl, Class returnType, } } - public static String getBucketNameWithoutAppid(String originBucketName, String appidStr) { + public static String getBucketNameWithAppid(String originBucketName, String appidStr) { if (appidStr == null || appidStr.isEmpty()) { return originBucketName; } diff --git a/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java b/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java index 53a03c45..1dd38fa2 100644 --- a/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java +++ b/src/main/java/org/apache/hadoop/fs/CosNativeFileSystemStore.java @@ -292,11 +292,24 @@ private void initCOSClient(URI uri, Configuration conf) throws IOException { LOG.info("hadoop cos retry times: {}, cos client retry times: {}", this.maxRetryTimes, clientMaxRetryTimes); - // 设置连接池的最大连接数目 + if (conf.getBoolean(CosNConfigKeys.COSN_USE_SHORT_CONNECTION, + CosNConfigKeys.DEFAULT_COSN_USE_SHORT_CONNECTION)) { + config.setShortConnection(); + } + config.setMaxConnectionsCount( conf.getInt( CosNConfigKeys.MAX_CONNECTION_NUM, CosNConfigKeys.DEFAULT_MAX_CONNECTION_NUM)); + config.setConnectionRequestTimeout( + conf.getInt(CosNConfigKeys.COSN_CONNECTION_REQUEST_TIMEOUT, + CosNConfigKeys.DEFAULT_COSN_CONNECTION_REQUEST_TIMEOUT)); + config.setConnectionTimeout( + conf.getInt(CosNConfigKeys.COSN_CONNECTION_TIMEOUT, + CosNConfigKeys.DEFAULT_COSN_CONNECTION_TIMEOUT)); + config.setIdleConnectionAlive( + conf.getInt(CosNConfigKeys.COSN_IDLE_CONNECTION_ALIVE, + CosNConfigKeys.DEFAULT_COSN_IDLE_CONNECTION_ALIVE)); // 设置是否进行服务器端加密 String serverSideEncryptionAlgorithm = conf.get(CosNConfigKeys.COSN_SERVER_SIDE_ENCRYPTION_ALGORITHM, ""); diff --git a/src/main/java/org/apache/hadoop/fs/RangerCredentialsClient.java b/src/main/java/org/apache/hadoop/fs/RangerCredentialsClient.java index acd3e009..b6aefc64 100644 --- a/src/main/java/org/apache/hadoop/fs/RangerCredentialsClient.java +++ b/src/main/java/org/apache/hadoop/fs/RangerCredentialsClient.java @@ -74,7 +74,7 @@ public void doCheckPermission(Path f, RangerAccessType rangerAccessType, String } PermissionRequest permissionReq = new PermissionRequest(ServiceType.COS, accessType, - CosNUtils.getBucketNameWithoutAppid(this.bucket, this.conf.get(CosNConfigKeys.COSN_APPID_KEY)), + CosNUtils.getBucketNameWithAppid(this.bucket, this.conf.get(CosNConfigKeys.COSN_APPID_KEY)), allowKey, "", ""); boolean allowed = false; String checkPermissionActualUserName = ownerId; @@ -101,7 +101,7 @@ public void doCheckCustomAuth(Configuration conf) throws IOException { } GetSTSResponse stsResp = this.rangerQcloudObjectStorageStorageClient.getSTS(bucketRegion, - bucket); + CosNUtils.getBucketNameWithAppid(this.bucket, this.conf.get(CosNConfigKeys.COSN_APPID_KEY))); if (!stsResp.isCheckAuthPass()) { throw new IOException(String.format("Permission denied, [operation: %s], please check user and " + "password", CUSTOM_AUTHENTICATION)); diff --git a/src/main/java/org/apache/hadoop/fs/auth/RangerCredentialsProvider.java b/src/main/java/org/apache/hadoop/fs/auth/RangerCredentialsProvider.java index 6e0c9b4c..6e1601eb 100644 --- a/src/main/java/org/apache/hadoop/fs/auth/RangerCredentialsProvider.java +++ b/src/main/java/org/apache/hadoop/fs/auth/RangerCredentialsProvider.java @@ -7,7 +7,6 @@ import org.apache.hadoop.fs.CosNConfigKeys; import org.apache.hadoop.fs.CosNUtils; import org.apache.hadoop.fs.RangerCredentialsClient; -import org.apache.hadoop.fs.cosn.ranger.client.RangerQcloudObjectStorageClient; import org.apache.hadoop.fs.cosn.ranger.security.sts.GetSTSResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,7 +31,7 @@ public RangerCredentialsProvider(@Nullable URI uri, Configuration conf, super(uri, conf); if (null != conf) { this.appId = conf.get(CosNConfigKeys.COSN_APPID_KEY); - this.bucketNameWithoutAppid = CosNUtils.getBucketNameWithoutAppid( + this.bucketNameWithoutAppid = CosNUtils.getBucketNameWithAppid( uri.getHost(), conf.get(CosNConfigKeys.COSN_APPID_KEY)); this.bucketRegion = conf.get(CosNConfigKeys.COSN_REGION_KEY); // native store keep the ranger client not null.