From da38a2f870f70902276208f7708f1bda0e8d023a Mon Sep 17 00:00:00 2001 From: iabetor Date: Thu, 19 May 2022 09:54:17 +0800 Subject: [PATCH] return real user and add custom auth check (#57) Co-authored-by: vinsonruan --- .../org/apache/hadoop/fs/CosFileSystem.java | 53 ++++++++++++++++++- .../fs/auth/RangerCredentialsProvider.java | 13 +++-- .../org/apache/hadoop/fs/cosn/Constants.java | 2 + 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java index a7887b8c..0b1f402d 100644 --- a/src/main/java/org/apache/hadoop/fs/CosFileSystem.java +++ b/src/main/java/org/apache/hadoop/fs/CosFileSystem.java @@ -6,6 +6,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.cosn.Constants; +import org.apache.hadoop.fs.cosn.ranger.security.sts.GetSTSResponse; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -30,6 +31,8 @@ import java.util.List; import java.util.Map; +import static org.apache.hadoop.fs.cosn.Constants.CUSTOM_AUTHENTICATION; + /** * A {@link FileSystem} for reading and writing files stored on @@ -103,6 +106,8 @@ public void initialize(URI uri, Configuration conf) throws IOException { this.nativeStore.initialize(uri, conf); this.isDefaultNativeStore = true; } + // required checkCustomAuth if ranger is enabled and custom authentication is enabled + checkCustomAuth(conf); this.isPosixFSStore = this.nativeStore.headBucket(bucket).isMergeBucket(); LOG.info("The cos bucket {} bucket.", isPosixFSStore ? "is the posix" : "is the normal"); @@ -494,13 +499,59 @@ private void checkPermission(Path f, RangerAccessType rangerAccessType) throws I CosNUtils.getBucketNameWithoutAppid(this.bucket, this.getConf().get(CosNConfigKeys.COSN_APPID_KEY)), allowKey, "", ""); boolean allowed = false; + String checkPermissionActualUserName = getOwnerId(); PermissionResponse permission = rangerQcloudObjectStorageStorageClient.checkPermission(permissionReq); if (permission != null) { allowed = permission.isAllowed(); + if (permission.getRealUserName() != null && !permission.getRealUserName().isEmpty()) { + checkPermissionActualUserName = permission.getRealUserName(); + } } if (!allowed) { throw new IOException(String.format("Permission denied, [key: %s], [user: %s], [operation: %s]", - allowKey, this.userGroupInformation.getShortUserName(), rangerAccessType.name())); + allowKey, checkPermissionActualUserName, rangerAccessType.name())); + } + } + + private String getOwnerId() { + UserGroupInformation currentUgi; + try { + currentUgi = UserGroupInformation.getCurrentUser(); + } catch (IOException e) { + LOG.warn("get current user failed! use user.name prop", e); + return System.getProperty("user.name"); + } + + String shortUserName = ""; + if (currentUgi != null) { + shortUserName = currentUgi.getShortUserName(); + } + + if (shortUserName == null) { + LOG.warn("get short user name failed! use user.name prop"); + shortUserName = System.getProperty("user.name"); + } + return shortUserName; + } + + /** + * @param conf + * @throws IOException + */ + private void checkCustomAuth(Configuration conf) throws IOException { + if (!enableRangerPluginPermissionCheck) { + return; + } + String bucketRegion = conf.get(CosNConfigKeys.COSN_REGION_KEY); + if (bucketRegion == null || bucketRegion.isEmpty()) { + bucketRegion = conf.get(CosNConfigKeys.COSN_REGION_PREV_KEY); + } + + GetSTSResponse stsResp = CosFileSystem.rangerQcloudObjectStorageStorageClient.getSTS(bucketRegion, + bucket); + if (!stsResp.isCheckAuthPass()) { + throw new IOException(String.format("Permission denied, [operation: %s], please check user and " + + "password", CUSTOM_AUTHENTICATION)); } } diff --git a/src/main/java/org/apache/hadoop/fs/auth/RangerCredentialsProvider.java b/src/main/java/org/apache/hadoop/fs/auth/RangerCredentialsProvider.java index 19a2f442..ac360e2d 100644 --- a/src/main/java/org/apache/hadoop/fs/auth/RangerCredentialsProvider.java +++ b/src/main/java/org/apache/hadoop/fs/auth/RangerCredentialsProvider.java @@ -44,7 +44,7 @@ public RangerCredentialsProvider(@Nullable URI uri, Configuration conf) { } } - class RangerCredentialsFetcher { + class RangerCredentialsFetcher { private int refreshIntervalSeconds; private AtomicReference lastCredentialsRef; private AtomicLong lastGetCredentialsTimeStamp; @@ -78,10 +78,17 @@ private COSCredentials fetchNewCredentials() { try { GetSTSResponse stsResp = CosFileSystem.rangerQcloudObjectStorageStorageClient.getSTS(bucketRegion, bucketNameWithoutAppid); - + /** + * some customers feel that kerberos authentication is heavy, so we have implemented a relatively + * lightweight authentication method + */ + // if the custom authentication fails, there will be no temporary AK/SK + if (!stsResp.isCheckAuthPass()) { + return null; + } COSCredentials cosCredentials = null; if (appId != null) { - cosCredentials = new BasicSessionCredentials(appId, stsResp.getTempAK(), stsResp.getTempSK(), + cosCredentials = new BasicSessionCredentials(appId, stsResp.getTempAK(), stsResp.getTempSK(), stsResp.getTempToken()); } else { cosCredentials = new BasicSessionCredentials(stsResp.getTempAK(), stsResp.getTempSK(), diff --git a/src/main/java/org/apache/hadoop/fs/cosn/Constants.java b/src/main/java/org/apache/hadoop/fs/cosn/Constants.java index 252c3a08..011ff1ca 100644 --- a/src/main/java/org/apache/hadoop/fs/cosn/Constants.java +++ b/src/main/java/org/apache/hadoop/fs/cosn/Constants.java @@ -35,4 +35,6 @@ private Constants() { public static final String COSN_POSIX_BUCKET_FS_COSN_IMPL = "org.apache.hadoop.fs.CosNFileSystem"; public static final String COSN_POSIX_BUCKET_FS_CHDFS_IMPL="com.qcloud.chdfs.fs.CHDFSHadoopFileSystemAdapter"; + + public static final String CUSTOM_AUTHENTICATION = "custom authentication"; }