From 2bd227e0f28c4bec0f40480b60dc0e36fe6e85e6 Mon Sep 17 00:00:00 2001 From: Sean Lin Date: Mon, 23 Sep 2019 12:59:33 -0700 Subject: [PATCH] feat(caching): Enable data key caching in AWS crypto operations (#203) * Enable data key caching in AWS crypto operations * Add logging * Add feature flag * Update cms.conf * Update CmsGuiceModule.java --- gradle.properties | 2 +- .../server/config/guice/CmsGuiceModule.java | 111 +++++++++++++++++- .../cerberus/service/EncryptionService.java | 57 ++++++--- src/main/resources/cms.conf | 2 + 4 files changed, 154 insertions(+), 18 deletions(-) diff --git a/gradle.properties b/gradle.properties index cff260ecb..9c6e9da63 100644 --- a/gradle.properties +++ b/gradle.properties @@ -14,6 +14,6 @@ # limitations under the License. # -version=3.31.1 +version=3.32.0 groupId=com.nike.cerberus artifactId=cms diff --git a/src/main/java/com/nike/cerberus/server/config/guice/CmsGuiceModule.java b/src/main/java/com/nike/cerberus/server/config/guice/CmsGuiceModule.java index c5f95ca1a..f409384db 100644 --- a/src/main/java/com/nike/cerberus/server/config/guice/CmsGuiceModule.java +++ b/src/main/java/com/nike/cerberus/server/config/guice/CmsGuiceModule.java @@ -17,6 +17,15 @@ package com.nike.cerberus.server.config.guice; +import com.amazonaws.encryptionsdk.CryptoMaterialsManager; +import com.amazonaws.encryptionsdk.DefaultCryptoMaterialsManager; +import com.amazonaws.encryptionsdk.MasterKeyProvider; +import com.amazonaws.encryptionsdk.caching.CachingCryptoMaterialsManager; +import com.amazonaws.encryptionsdk.caching.CryptoMaterialsCache; +import com.amazonaws.encryptionsdk.caching.LocalCryptoMaterialsCache; +import com.amazonaws.encryptionsdk.kms.KmsMasterKey; +import com.amazonaws.regions.Region; +import com.amazonaws.regions.Regions; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.*; import com.google.inject.name.Names; @@ -61,8 +70,11 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static com.nike.cerberus.service.EncryptionService.*; + public class CmsGuiceModule extends AbstractModule { private static final String AUTH_CONNECTOR_IMPL_KEY = "cms.auth.connector"; @@ -191,8 +203,8 @@ public List> authProtectedEndpoints(@Named("appEndpoints") Set keyProvider = initializeKeyProvider(cmkArns, currentRegion); + if (cacheEnabled) { + int maxSize = kmsDataKeyCachingOptionalPropertyHolder.encryptMaxSize; + int maxAge = kmsDataKeyCachingOptionalPropertyHolder.encryptMaxAge; + int messageUseLimit = kmsDataKeyCachingOptionalPropertyHolder.encryptMessageUseLimit; + logger.info("Initializing caching encryptCryptoMaterialsManager with CMK: {}, maxSize: {}, maxAge: {}, " + + "messageUseLimit: {}", cmkArns, maxSize, maxAge, messageUseLimit); + CryptoMaterialsCache cache = new LocalCryptoMaterialsCache(maxSize); + CryptoMaterialsManager cachingCmm = + CachingCryptoMaterialsManager.newBuilder().withMasterKeyProvider(keyProvider) + .withCache(cache) + .withMaxAge(maxAge, TimeUnit.SECONDS) + .withMessageUseLimit(messageUseLimit) + .build(); + return cachingCmm; + } else { + logger.info("Initializing encryptCryptoMaterialsManager with CMK: {}", cmkArns); + return new DefaultCryptoMaterialsManager(keyProvider); + } + } + + @Provides + @Singleton + @Named("decryptCryptoMaterialsManager") + public CryptoMaterialsManager decryptCryptoMaterialsManager(@Named("cms.encryption.cmk.arns") String cmkArns, + @Named("cms.cache.enabled") boolean cacheEnabled, + KmsDataKeyCachingOptionalPropertyHolder kmsDataKeyCachingOptionalPropertyHolder, + Region currentRegion) { + MasterKeyProvider keyProvider = initializeKeyProvider(cmkArns, currentRegion); + if (cacheEnabled) { + int maxSize = kmsDataKeyCachingOptionalPropertyHolder.decryptMaxSize; + int maxAge = kmsDataKeyCachingOptionalPropertyHolder.decryptMaxAge; + logger.info("Initializing caching decryptCryptoMaterialsManager with CMK: {}, maxSize: {}, maxAge: {}", + cmkArns, maxSize, maxAge); + CryptoMaterialsCache cache = new LocalCryptoMaterialsCache(maxSize); + CryptoMaterialsManager cachingCmm = + CachingCryptoMaterialsManager.newBuilder().withMasterKeyProvider(keyProvider) + .withCache(cache) + .withMaxAge(maxAge, TimeUnit.SECONDS) + .build(); + return cachingCmm; + } else { + logger.info("Initializing decryptCryptoMaterialsManager with CMK: {}", cmkArns); + return new DefaultCryptoMaterialsManager(keyProvider); + } + } + + /** + * Returns the current region + * @return current region + */ + @Provides + @Singleton + public Region currentRegion() { + Region region = Regions.getCurrentRegion(); + Region currentRegion = region == null ? Region.getRegion(Regions.DEFAULT_REGION ) : region; + return currentRegion; + } + + /** + * This 'holder' class allows optional injection of KMS-data-key-caching-specific properties that are only necessary when + * SignalFx metrics reporting is enabled. + * + * The 'optional=true' parameter to Guice @Inject cannot be used in combination with the @Provides annotation + * or with constructor injection. + * + * https://github.com/google/guice/wiki/FrequentlyAskedQuestions + */ + static class KmsDataKeyCachingOptionalPropertyHolder { + @Inject(optional=true) + @com.google.inject.name.Named("cms.cache.encryption.encrypt.maxSize") + int encryptMaxSize = 0; + + @Inject(optional=true) + @com.google.inject.name.Named("cms.cache.encryption.encrypt.maxAge") + int encryptMaxAge = 0; + + @Inject(optional=true) + @com.google.inject.name.Named("cms.cache.encryption.encrypt.messageUseLimit") + int encryptMessageUseLimit = 0; + + @Inject(optional=true) + @com.google.inject.name.Named("cms.cache.encryption.decrypt.maxSize") + int decryptMaxSize = 0; + + @Inject(optional=true) + @com.google.inject.name.Named("cms.cache.encryption.decrypt.maxAge") + int decryptMaxAge = 0; + } } diff --git a/src/main/java/com/nike/cerberus/service/EncryptionService.java b/src/main/java/com/nike/cerberus/service/EncryptionService.java index ccbe9b7ad..1d373c455 100644 --- a/src/main/java/com/nike/cerberus/service/EncryptionService.java +++ b/src/main/java/com/nike/cerberus/service/EncryptionService.java @@ -1,13 +1,14 @@ package com.nike.cerberus.service; import com.amazonaws.encryptionsdk.AwsCrypto; +import com.amazonaws.encryptionsdk.CryptoMaterialsManager; +import com.amazonaws.encryptionsdk.DefaultCryptoMaterialsManager; import com.amazonaws.encryptionsdk.MasterKeyProvider; import com.amazonaws.encryptionsdk.ParsedCiphertext; import com.amazonaws.encryptionsdk.kms.KmsMasterKey; import com.amazonaws.encryptionsdk.kms.KmsMasterKeyProvider; import com.amazonaws.encryptionsdk.multi.MultipleProviderFactory; import com.amazonaws.regions.Region; -import com.amazonaws.regions.Regions; import com.google.common.collect.Lists; import com.nike.cerberus.util.CiphertextUtils; import org.apache.commons.lang3.StringUtils; @@ -39,20 +40,26 @@ public class EncryptionService { private final Logger log = LoggerFactory.getLogger(getClass()); private final AwsCrypto awsCrypto; - private final MasterKeyProvider encryptProvider; + private List cmkArnList; private final String cmsVersion; private final Region currentRegion; + private final CryptoMaterialsManager decryptCryptoMaterialsManager; + private final CryptoMaterialsManager encryptCryptoMaterialsManager; @Inject public EncryptionService(AwsCrypto awsCrypto, @Named("cms.encryption.cmk.arns") String cmkArns, - @Named("service.version") String cmsVersion) { - - Region region = Regions.getCurrentRegion(); - currentRegion = region == null ? Region.getRegion(Regions.DEFAULT_REGION ) : region; + @Named("service.version") String cmsVersion, + @Named("decryptCryptoMaterialsManager") CryptoMaterialsManager decryptCryptoMaterialsManager, + @Named("encryptCryptoMaterialsManager") CryptoMaterialsManager encryptCryptoMaterialsManager, + Region currentRegion) { + this.currentRegion = currentRegion; this.awsCrypto = awsCrypto; - this.encryptProvider = initializeKeyProvider(splitArns(cmkArns), currentRegion); + log.info("CMK ARNs " + cmkArns); + this.cmkArnList = splitArns(cmkArns); this.cmsVersion = cmsVersion; + this.decryptCryptoMaterialsManager = decryptCryptoMaterialsManager; + this.encryptCryptoMaterialsManager = encryptCryptoMaterialsManager; } /** @@ -66,11 +73,11 @@ public EncryptionService(AwsCrypto awsCrypto, * @param sdbPath the SDB path where these secrets are being stored (added to EncryptionContext) */ public String encrypt(String plainTextPayload, String sdbPath) { - return awsCrypto.encryptString(encryptProvider, plainTextPayload, buildEncryptionContext(sdbPath)).getResult(); + return awsCrypto.encryptString(encryptCryptoMaterialsManager, plainTextPayload, buildEncryptionContext(sdbPath)).getResult(); } public byte[] encrypt(byte[] bytes, String sdbPath) { - return awsCrypto.encryptData(encryptProvider, bytes, buildEncryptionContext(sdbPath)).getResult(); + return awsCrypto.encryptData(encryptCryptoMaterialsManager, bytes, buildEncryptionContext(sdbPath)).getResult(); } /** @@ -118,8 +125,17 @@ private String decrypt(ParsedCiphertext parsedCiphertext, String sdbPath) { // Parses the ARNs out of the encryptedPayload so that you can manually rotate the CMKs, if desired // Whatever CMKs were used in the encrypt operation will be used to decrypt List cmkArns = CiphertextUtils.getCustomerMasterKeyArns(parsedCiphertext); - MasterKeyProvider decryptProvider = initializeKeyProvider(cmkArns, currentRegion); - return new String(awsCrypto.decryptData(decryptProvider, parsedCiphertext).getResult(), StandardCharsets.UTF_8); + CryptoMaterialsManager cryptoMaterialsManager = getCryptoMaterialsManager(cmkArns, currentRegion); + return new String(awsCrypto.decryptData(cryptoMaterialsManager, parsedCiphertext).getResult(), StandardCharsets.UTF_8); + } + + private CryptoMaterialsManager getCryptoMaterialsManager(List cmkArns, Region currentRegion) { + if (cmkArnList.containsAll(cmkArns)) { + return decryptCryptoMaterialsManager; + } else { + MasterKeyProvider provider = initializeKeyProvider(cmkArns, currentRegion); + return new DefaultCryptoMaterialsManager(provider); + } } /** @@ -155,8 +171,8 @@ private byte[] decryptToBytes(ParsedCiphertext parsedCiphertext, String sdbPath) // Parses the ARNs out of the encryptedPayload so that you can manually rotate the CMKs, if desired // Whatever CMKs were used in the encrypt operation will be used to decrypt List cmkArns = CiphertextUtils.getCustomerMasterKeyArns(parsedCiphertext); - MasterKeyProvider decryptProvider = initializeKeyProvider(cmkArns, currentRegion); - return awsCrypto.decryptData(decryptProvider, parsedCiphertext).getResult(); + CryptoMaterialsManager cryptoMaterialsManager = getCryptoMaterialsManager(cmkArns, currentRegion); + return awsCrypto.decryptData(cryptoMaterialsManager, parsedCiphertext).getResult(); } /** @@ -194,8 +210,7 @@ private void validateEncryptionContext(ParsedCiphertext parsedCiphertext, String /** * Split the ARNs from a single comma delimited string into a list. */ - protected List splitArns(String cmkArns) { - log.info("CMK ARNs " + cmkArns); + public static List splitArns(String cmkArns) { List keyArns = Lists.newArrayList(StringUtils.split(cmkArns, ",")); if (keyArns.size() < 2) { throw new IllegalArgumentException("At least 2 CMK ARNs are required for high availability, size:" + keyArns.size()); @@ -209,7 +224,7 @@ protected List splitArns(String cmkArns) { * For encrypt, KMS in all regions must be available. * For decrypt, KMS in at least one region must be available. */ - protected static MasterKeyProvider initializeKeyProvider(List cmkArns, Region currentRegion) { + public static MasterKeyProvider initializeKeyProvider(List cmkArns, Region currentRegion) { List> providers = getSortedArnListByCurrentRegion(cmkArns, currentRegion).stream() .map(KmsMasterKeyProvider::new) @@ -217,6 +232,16 @@ protected static MasterKeyProvider initializeKeyProvider(List) MultipleProviderFactory.buildMultiProvider(providers); } + /** + * Initialize a Multi-KMS-MasterKeyProvider. + *

+ * For encrypt, KMS in all regions must be available. + * For decrypt, KMS in at least one region must be available. + */ + public static MasterKeyProvider initializeKeyProvider(String cmkArns, Region currentRegion) { + return initializeKeyProvider(splitArns(cmkArns), currentRegion); + } + /** * ARN with current region should always go first to minimize latency */ diff --git a/src/main/resources/cms.conf b/src/main/resources/cms.conf index 5f6476719..e9f47d5bd 100644 --- a/src/main/resources/cms.conf +++ b/src/main/resources/cms.conf @@ -196,3 +196,5 @@ cms.iam.token.ttl=1h # When true, if an SDB grants access to AD group 'Lst-foo', then users in group 'Lst-Foo' will not have access # When false, if an SDB grants access to AD group 'Lst-foo', then users in group 'Lst-Foo' will have access cms.user.groups.caseSensitive=true + +cms.cache.enabled=false