Merge pull request #36 from Nike-Inc/bugfix/avoid_kms_policy_validati…

…on_api_limit

Avoid hitting API limit on KMS policy validation

README.md

@@ -59,19 +59,24 @@ That will setup the default policy and generate a token for CMS and output:
 
 There are a few parameters that need to be configured for CMS to run properly, they are defined in this table. 
 
-property                    | required | notes
---------------------------- | -------- | ----------
-JDBC.url                    | Yes | The JDBC url for the mysql db
-JDBC.username               | Yes | The JDBC user name for the mysql db
-JDBC.password               | Yes | The JDBC JDBC.password for the mysql db
-root.user.arn               | Yes | The arn for the root AWS user, needed to make the KMS keys deletable.
-admin.role.arn              | Yes | The arn for an AWS user, needed to make the KMS keys deletable.
-cms.role.arn                | Yes | The arn for the Instance profile for CMS instances, so they can admin KMS keys that they create.
-cms.admin.group             | Yes | Group that user can be identified by to get admin privileges, currently this just enables users to access `/v1/metadata` see API.md
-cms.admin.roles             | No  | Comma seperated list of ARNs that can auth and access admin endpoints.
-cms.auth.connector          | Yes | The user authentication connector implementation to use for user auth.
-cms.user.token.ttl.override | No  | By default user tokens are created with a TTL of 1h, you can override that with this param
-cms.iam.token.ttl.override  | No  | By default IAM tokens are created with a TTL of 1h, you can override that with this param
+property                                            | required | notes
+---------------------------                         | -------- | ----------
+JDBC.url                                            | Yes      | The JDBC url for the mysql db
+JDBC.username                                       | Yes      | The JDBC user name for the mysql db
+JDBC.password                                       | Yes      | The JDBC JDBC.password for the mysql db
+root.user.arn                                       | Yes      | The arn for the root AWS user, needed to make the KMS keys deletable.
+admin.role.arn                                      | Yes      | The arn for an AWS user, needed to make the KMS keys deletable.
+cms.role.arn                                        | Yes      | The arn for the Instance profile for CMS instances, so they can admin KMS keys that they create.
+cms.admin.group                                     | Yes      | Group that user can be identified by to get admin privileges, currently this just enables users to access `/v1/metadata` see API.md
+cms.admin.roles                                     | No       | Comma separated list of ARNs that can auth and access admin endpoints.
+cms.auth.connector                                  | Yes      | The user authentication connector implementation to use for user auth.
+cms.user.token.ttl.override                         | No       | By default user tokens are created with a TTL of 1h, you can override that with this param
+cms.iam.token.ttl.override                          | No       | By default IAM tokens are created with a TTL of 1h, you can override that with this param
+cms.kms.policy.validation.interval.millis.override  | No       | By default CMS validates KMS key policies no more than once per minute, you can override that with this param
+
+KMS Policies are bound to IAM Principal IDs rather than ARNs themselves. Because of this, we validate the policy at authentication time
+to ensure that if an IAM role has been deleted and re-created, that we grant access to the new principal ID.
+The API limit for this call is low, so the `cms.kms.policy.validation.interval.millis.override` property is used to throttle this validation.
 
 For local dev see `Running CMS Locally`.
 

gradle.properties

@@ -14,6 +14,6 @@
 # limitations under the License.
 #
 
-version=0.17.0
+version=0.18.0
 groupId=com.nike.cerberus
 artifactId=cms

gradle/develop.gradle

@@ -18,8 +18,8 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import groovyx.net.http.RESTClient
 import static groovyx.net.http.ContentType.*
 
-def dashboardRelease = 'v0.12.0'
-def vaultVersion = "0.6.4"
+def dashboardRelease = 'v1.0.0'
+def vaultVersion = "0.7.0"
 
 buildscript {
     apply from: file('gradle/buildscript.gradle'), to: buildscript

src/main/java/com/nike/cerberus/dao/AwsIamRoleDao.java

@@ -76,4 +76,8 @@ public Optional<AwsIamRoleKmsKeyRecord> getKmsKey(final String awsIamRoleId, fin
     public int createIamRoleKmsKey(final AwsIamRoleKmsKeyRecord record) {
         return awsIamRoleMapper.createIamRoleKmsKey(record);
     }
+
+    public int updateIamRoleKmsKey(final AwsIamRoleKmsKeyRecord record) {
+        return awsIamRoleMapper.updateIamRoleKmsKey(record);
+    }
 }

src/main/java/com/nike/cerberus/mapper/AwsIamRoleMapper.java

@@ -49,4 +49,6 @@ int deleteIamRolePermission(@Param("safeDepositBoxId") String safeDepositBoxId,
     List<AwsIamRolePermissionRecord> getIamRolePermissions(@Param("safeDepositBoxId") String safeDepositBoxId);
 
     int deleteIamRolePermissions(@Param("safeDepositBoxId") String safeDepositBoxId);
+
+    int updateIamRoleKmsKey(@Param("record") AwsIamRoleKmsKeyRecord record);
 }

src/main/java/com/nike/cerberus/record/AwsIamRoleKmsKeyRecord.java

@@ -40,6 +40,8 @@ public class AwsIamRoleKmsKeyRecord {
 
     private OffsetDateTime lastUpdatedTs;
 
+    private OffsetDateTime lastValidatedTs;
+
     public String getId() {
         return id;
     }
@@ -112,6 +114,15 @@ public AwsIamRoleKmsKeyRecord setLastUpdatedTs(OffsetDateTime lastUpdatedTs) {
         return this;
     }
 
+    public OffsetDateTime getLastValidatedTs() {
+        return lastValidatedTs;
+    }
+
+    public AwsIamRoleKmsKeyRecord setLastValidatedTs(OffsetDateTime lastValidatedTs) {
+        this.lastValidatedTs = lastValidatedTs;
+        return this;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -124,11 +135,12 @@ public boolean equals(Object o) {
                 Objects.equals(createdBy, that.createdBy) &&
                 Objects.equals(lastUpdatedBy, that.lastUpdatedBy) &&
                 Objects.equals(createdTs, that.createdTs) &&
-                Objects.equals(lastUpdatedTs, that.lastUpdatedTs);
+                Objects.equals(lastUpdatedTs, that.lastUpdatedTs) &&
+                Objects.equals(lastValidatedTs, that.lastValidatedTs);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(id, awsIamRoleId, awsRegion, awsKmsKeyId, createdBy, lastUpdatedBy, createdTs, lastUpdatedTs);
+        return Objects.hash(id, awsIamRoleId, awsRegion, awsKmsKeyId, createdBy, lastUpdatedBy, createdTs, lastUpdatedTs, lastValidatedTs);
     }
 }

src/main/java/com/nike/cerberus/service/AuthenticationService.java

@@ -57,9 +57,14 @@
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.http.HttpStatus;
+import org.joda.time.Interval;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
+import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
@@ -73,6 +78,8 @@
 @Singleton
 public class AuthenticationService {
 
+    private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
     public static final String SYSTEM_USER = "system";
     public static final String ADMIN_GROUP_PROPERTY = "cms.admin.group";
     public static final String ADMIN_IAM_ROLES_PROPERTY = "cms.admin.roles";
@@ -203,7 +210,7 @@ public IamRoleAuthResponse authenticate(IamPrincipalCredentials credentials) {
         return authenticate(credentials, vaultAuthPrincipalMetadata);
     }
 
-    public IamRoleAuthResponse authenticate(IamPrincipalCredentials credentials, Map<String, String> vaultAuthPrincipalMetadata) {
+    private IamRoleAuthResponse authenticate(IamPrincipalCredentials credentials, Map<String, String> vaultAuthPrincipalMetadata) {
         final String keyId;
         try {
             keyId = getKeyId(credentials);
@@ -360,7 +367,7 @@ private Set<String> buildPolicySet(final String iamRoleArn) {
      * @param credentials IAM role credentials
      * @return KMS Key id
      */
-    private String getKeyId(IamPrincipalCredentials credentials) {
+    protected String getKeyId(IamPrincipalCredentials credentials) {
         final Optional<AwsIamRoleRecord> iamRole = awsIamRoleDao.getIamRole(credentials.getIamPrincipalArn());
 
         if (!iamRole.isPresent()) {
@@ -374,14 +381,16 @@ private String getKeyId(IamPrincipalCredentials credentials) {
         final Optional<AwsIamRoleKmsKeyRecord> kmsKey = awsIamRoleDao.getKmsKey(iamRole.get().getId(), credentials.getRegion());
 
         final String kmsKeyId;
+        final AwsIamRoleKmsKeyRecord kmsKeyRecord;
+        final OffsetDateTime now = dateTimeSupplier.get();
 
         if (!kmsKey.isPresent()) {
             kmsKeyId = kmsService.provisionKmsKey(iamRole.get().getId(), credentials.getIamPrincipalArn(),
-                    credentials.getRegion(), SYSTEM_USER, dateTimeSupplier.get());
+                    credentials.getRegion(), SYSTEM_USER, now);
         } else {
-            kmsKeyId = kmsKey.get().getAwsKmsKeyId();
-            String keyRegion = credentials.getRegion();
-            kmsService.validatePolicy(kmsKeyId, credentials.getIamPrincipalArn(), keyRegion);
+            kmsKeyRecord = kmsKey.get();
+            kmsKeyId = kmsKeyRecord.getAwsKmsKeyId();
+            kmsService.validatePolicy(kmsKeyRecord, credentials.getIamPrincipalArn());
         }
 
         return kmsKeyId;

src/main/java/com/nike/cerberus/service/KmsService.java

@@ -26,11 +26,13 @@
 import com.amazonaws.services.kms.model.KeyMetadata;
 import com.amazonaws.services.kms.model.KeyUsageType;
 import com.amazonaws.services.kms.model.PutKeyPolicyRequest;
+import com.google.inject.name.Named;
 import com.nike.backstopper.exception.ApiException;
 import com.nike.cerberus.aws.KmsClientFactory;
 import com.nike.cerberus.dao.AwsIamRoleDao;
 import com.nike.cerberus.error.DefaultApiError;
 import com.nike.cerberus.record.AwsIamRoleKmsKeyRecord;
+import com.nike.cerberus.util.DateTimeSupplier;
 import com.nike.cerberus.util.UuidSupplier;
 import org.mybatis.guice.transactional.Transactional;
 import org.slf4j.Logger;
@@ -39,6 +41,10 @@
 import javax.inject.Inject;
 import javax.inject.Singleton;
 import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
+import java.util.Optional;
+
+import static com.nike.cerberus.service.AuthenticationService.SYSTEM_USER;
 
 /**
  * Abstracts interactions with the AWS KMS service.
@@ -49,6 +55,9 @@ public class KmsService {
     private final Logger logger = LoggerFactory.getLogger(this.getClass());
 
     private static final String KMS_ALIAS_FORMAT = "alias/cerberus/%s";
+    public static final String KMS_POLICY_VALIDATION_INTERVAL_OVERRIDE = "cms.kms.policy.validation.interval.millis.override";
+    public static final Integer DEFAULT_KMS_VALIDATION_INTERVAL = 6000; // in milliseconds
+
 
     private final AwsIamRoleDao awsIamRoleDao;
 
@@ -58,15 +67,23 @@ public class KmsService {
 
     private final KmsPolicyService kmsPolicyService;
 
+    private final DateTimeSupplier dateTimeSupplier;
+
+    @com.google.inject.Inject(optional=true)
+    @Named(KMS_POLICY_VALIDATION_INTERVAL_OVERRIDE)
+    Integer kmsKeyPolicyValidationInterval = DEFAULT_KMS_VALIDATION_INTERVAL;
+
     @Inject
     public KmsService(final AwsIamRoleDao awsIamRoleDao,
                       final UuidSupplier uuidSupplier,
                       final KmsClientFactory kmsClientFactory,
-                      final KmsPolicyService kmsPolicyService) {
+                      final KmsPolicyService kmsPolicyService,
+                      final DateTimeSupplier dateTimeSupplier) {
         this.awsIamRoleDao = awsIamRoleDao;
         this.uuidSupplier = uuidSupplier;
         this.kmsClientFactory = kmsClientFactory;
         this.kmsPolicyService = kmsPolicyService;
+        this.dateTimeSupplier = dateTimeSupplier;
     }
 
     /**
@@ -111,12 +128,47 @@ public String provisionKmsKey(final String iamRoleId,
         awsIamRoleKmsKeyRecord.setLastUpdatedBy(user);
         awsIamRoleKmsKeyRecord.setCreatedTs(dateTime);
         awsIamRoleKmsKeyRecord.setLastUpdatedTs(dateTime);
+        awsIamRoleKmsKeyRecord.setLastValidatedTs(dateTime);
 
         awsIamRoleDao.createIamRoleKmsKey(awsIamRoleKmsKeyRecord);
 
         return result.getKeyMetadata().getArn();
     }
 
+    /**
+     * Updates the KMS CMK record for the specified IAM role and region
+     * @param awsIamRoleId    The IAM role that this CMK will be associated with
+     * @param awsRegion       The region to provision the key in
+     * @param user            The user requesting it
+     * @param lastedUpdatedTs The date when the record was last updated
+     * @param lastValidatedTs The date when the record was last validated
+     */
+    @Transactional
+    public void updateKmsKey(final String awsIamRoleId,
+                             final String awsRegion,
+                             final String user,
+                             final OffsetDateTime lastedUpdatedTs,
+                             final OffsetDateTime lastValidatedTs) {
+        final Optional<AwsIamRoleKmsKeyRecord> kmsKey = awsIamRoleDao.getKmsKey(awsIamRoleId, awsRegion);
+
+        if (!kmsKey.isPresent()) {
+            throw ApiException.newBuilder()
+                    .withApiErrors(DefaultApiError.ENTITY_NOT_FOUND)
+                    .withExceptionMessage("Unable to update a KMS key that does not exist.")
+                    .build();
+        }
+
+        AwsIamRoleKmsKeyRecord kmsKeyRecord = kmsKey.get();
+
+        AwsIamRoleKmsKeyRecord updatedKmsKeyRecord = new AwsIamRoleKmsKeyRecord();
+        updatedKmsKeyRecord.setAwsIamRoleId(kmsKeyRecord.getAwsIamRoleId());
+        updatedKmsKeyRecord.setLastUpdatedBy(user);
+        updatedKmsKeyRecord.setLastUpdatedTs(lastedUpdatedTs);
+        updatedKmsKeyRecord.setLastValidatedTs(lastValidatedTs);
+        updatedKmsKeyRecord.setAwsRegion(kmsKeyRecord.getAwsRegion());
+        awsIamRoleDao.updateIamRoleKmsKey(updatedKmsKeyRecord);
+    }
+
     protected String getAliasName(String awsIamRoleKmsKeyId) {
         return String.format(KMS_ALIAS_FORMAT, awsIamRoleKmsKeyId);
     }
@@ -134,34 +186,51 @@ protected String getAliasName(String awsIamRoleKmsKeyId) {
      * statement has been deleted the ARN is replaced by the ID. We can validate that principal matches an ARN pattern
      * or recreate the policy.
      *
-     * @param keyId - The CMK Id to validate the policies on.
+     * @param kmsKeyRecord - The CMK record to validate policy on
      * @param iamPrincipalArn - The principal ARN that should have decrypt permission
-     * @param kmsCMKRegion - The region that the key was provisioned for
      */
-    public void validatePolicy(String keyId, String iamPrincipalArn, String kmsCMKRegion) {
+    public void validatePolicy(AwsIamRoleKmsKeyRecord kmsKeyRecord, String iamPrincipalArn) {
+
+        if (! kmsPolicyNeedsValidation(kmsKeyRecord)) {
+            return;
+        }
+
+        String kmsCMKRegion = kmsKeyRecord.getAwsRegion();
+        String awsKmsKeyArn = kmsKeyRecord.getAwsKmsKeyId();
         AWSKMSClient kmsClient = kmsClientFactory.getClient(kmsCMKRegion);
-        GetKeyPolicyResult policyResult = null;
         try {
-            policyResult = kmsClient.getKeyPolicy(new GetKeyPolicyRequest().withKeyId(keyId).withPolicyName("default"));
+            GetKeyPolicyResult policyResult = kmsClient.getKeyPolicy(new GetKeyPolicyRequest().withKeyId(awsKmsKeyArn).withPolicyName("default"));
+
+            if (!kmsPolicyService.isPolicyValid(policyResult.getPolicy(), iamPrincipalArn)) {
+                logger.info("The KMS key: {} generated for IAM principal: {} contained an invalid policy, regenerating",
+                        awsKmsKeyArn, iamPrincipalArn);
+                String updatedPolicy = kmsPolicyService.generateStandardKmsPolicy(iamPrincipalArn);
+                kmsClient.putKeyPolicy(new PutKeyPolicyRequest()
+                        .withKeyId(awsKmsKeyArn)
+                        .withPolicyName("default")
+                        .withPolicy(updatedPolicy)
+                );
+            }
+
+            // update last validated timestamp
+            OffsetDateTime now = dateTimeSupplier.get();
+            updateKmsKey(kmsKeyRecord.getAwsIamRoleId(), kmsCMKRegion, SYSTEM_USER, now, now);
         } catch (AmazonServiceException e) {
-            throw ApiException.newBuilder()
-                    .withApiErrors(DefaultApiError.FAILED_TO_VALIDATE_KMS_KEY_POLICY)
-                    .withExceptionCause(e)
-                    .withExceptionMessage(
-                            String.format("Failed to validate KMS key policy for keyId: " +
-                                    "%s for IAM principal: %s in region: %s", keyId, iamPrincipalArn, kmsCMKRegion))
-                    .build();
+            logger.warn(String.format("Failed to validate KMS policy for keyId: %s for IAM principal: %s in region: %s. API limit" +
+                    " may have been reached for validate call.", awsKmsKeyArn, iamPrincipalArn, kmsCMKRegion), e);
         }
+    }
 
-        if (!kmsPolicyService.isPolicyValid(policyResult.getPolicy(), iamPrincipalArn)) {
-            logger.info("The KMS key: {} generated for IAM principal: {} contained an invalid policy, regenerating",
-                    keyId, iamPrincipalArn);
-            String updatedPolicy = kmsPolicyService.generateStandardKmsPolicy(iamPrincipalArn);
-            kmsClient.putKeyPolicy(new PutKeyPolicyRequest()
-                    .withKeyId(keyId)
-                    .withPolicyName("default")
-                    .withPolicy(updatedPolicy)
-            );
-        }
+    /**
+     * Determines if given KMS policy should be validated
+     * @param kmsKeyRecord - KMS key record to check for validation
+     * @return True if needs validation, False if not
+     */
+    protected boolean kmsPolicyNeedsValidation(AwsIamRoleKmsKeyRecord kmsKeyRecord) {
+
+        OffsetDateTime now = dateTimeSupplier.get();
+        long timeSinceLastValidatedInMillis = ChronoUnit.MILLIS.between(kmsKeyRecord.getLastValidatedTs(), now);
+
+        return timeSinceLastValidatedInMillis >= kmsKeyPolicyValidationInterval;
     }
 }