diff --git a/.travis.yml b/.travis.yml index 4720b5d81..db990f10b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,4 +11,4 @@ deploy: file: build/libs/cms.jar on: repo: Nike-Inc/cerberus-management-service - tags: true + tags: true \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 04bb84a8b..97b5b558e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -14,6 +14,6 @@ # limitations under the License. # -version=0.29.0 +version=0.30.0-hystrixRC5 groupId=com.nike.cerberus artifactId=cms diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index c41b93e8e..5af666e93 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -28,7 +28,7 @@ def groovyVersion = '2.3.9' dependencies { compile ( - "com.nike:vault-client:1.4.0", + "com.nike:vault-client:1.6.0", "com.nike.riposte:riposte-spi:$riposteVersion", "com.nike.riposte:riposte-core:$riposteVersion", "com.nike.riposte:riposte-typesafe-config:$riposteVersion", @@ -38,6 +38,7 @@ dependencies { "com.nike.riposte:riposte-metrics-codahale:$riposteVersion", "com.nike.riposte:riposte-metrics-codahale-signalfx:$riposteVersion", + "com.netflix.hystrix:hystrix-core:1.5.12", "javax:javaee-api:7.0", "org.codehaus.groovy:groovy-all:$groovyVersion", // For logback groovy config processing "ch.qos.logback:logback-classic:$logbackVersion", diff --git a/gradle/develop.gradle b/gradle/develop.gradle index bf251c393..ede18d6a9 100644 --- a/gradle/develop.gradle +++ b/gradle/develop.gradle @@ -19,7 +19,7 @@ import groovyx.net.http.RESTClient import static groovyx.net.http.ContentType.* def dashboardRelease = 'v1.6.0' -def vaultVersion = "0.7.3" +def vaultVersion = "0.8.1" def reverseProxyPort = 9001 def nodeServerPort = 8000 diff --git a/src/main/java/com/nike/cerberus/aws/KmsClientFactory.java b/src/main/java/com/nike/cerberus/aws/KmsClientFactory.java index 69bb3e280..c1c3568f2 100644 --- a/src/main/java/com/nike/cerberus/aws/KmsClientFactory.java +++ b/src/main/java/com/nike/cerberus/aws/KmsClientFactory.java @@ -29,7 +29,6 @@ /** * Factory for AWS KMS clients. Caches clients by region as they are requested. */ -@Singleton public class KmsClientFactory { private final Map kmsClientMap = Maps.newConcurrentMap(); diff --git a/src/main/java/com/nike/cerberus/hystrix/HystrixKmsClient.java b/src/main/java/com/nike/cerberus/hystrix/HystrixKmsClient.java new file mode 100644 index 000000000..31c53bd87 --- /dev/null +++ b/src/main/java/com/nike/cerberus/hystrix/HystrixKmsClient.java @@ -0,0 +1,102 @@ +package com.nike.cerberus.hystrix; + +import com.amazonaws.services.kms.AWSKMSClient; +import com.amazonaws.services.kms.model.CreateAliasRequest; +import com.amazonaws.services.kms.model.CreateAliasResult; +import com.amazonaws.services.kms.model.CreateKeyRequest; +import com.amazonaws.services.kms.model.CreateKeyResult; +import com.amazonaws.services.kms.model.DescribeKeyRequest; +import com.amazonaws.services.kms.model.DescribeKeyResult; +import com.amazonaws.services.kms.model.EncryptRequest; +import com.amazonaws.services.kms.model.EncryptResult; +import com.amazonaws.services.kms.model.GetKeyPolicyRequest; +import com.amazonaws.services.kms.model.GetKeyPolicyResult; +import com.amazonaws.services.kms.model.PutKeyPolicyRequest; +import com.amazonaws.services.kms.model.PutKeyPolicyResult; +import com.amazonaws.services.kms.model.ScheduleKeyDeletionRequest; +import com.amazonaws.services.kms.model.ScheduleKeyDeletionResult; +import com.netflix.hystrix.HystrixCommand; +import com.netflix.hystrix.HystrixCommandGroupKey; +import com.netflix.hystrix.HystrixCommandKey; +import com.netflix.hystrix.HystrixThreadPoolKey; + +import java.util.function.Supplier; + +/** + * Hystrix wrapper around AWSKMSClient + *

+ * Most of these commands should execute in their own Thread Pools because they have unique limits. + * http://docs.aws.amazon.com/kms/latest/developerguide/limits.html#requests-per-second-table + */ +public class HystrixKmsClient extends AWSKMSClient { + + private static final String KMS = "KMS"; + + private final AWSKMSClient client; + + public HystrixKmsClient(AWSKMSClient client) { + this.client = client; + } + + public EncryptResult encrypt(EncryptRequest request) { + // Default AWS limit was 1200 shared as of Aug 2017 + return execute("KmsEncryptDecrypt", "KmsEncrypt", () -> client.encrypt(request)); + } + + public CreateKeyResult createKey(CreateKeyRequest request) { + // Default AWS limit was 5 as of Aug 2017 + return execute("KmsCreateKey", () -> client.createKey(request)); + } + + public CreateAliasResult createAlias(CreateAliasRequest request) { + // Default AWS limit was 5 as of Aug 2017 + return execute("KmsCreateAlias", () -> client.createAlias(request)); + } + + public DescribeKeyResult describeKey(DescribeKeyRequest request) { + // Default AWS limit was 30 as of Aug 2017 + return execute("KmsDescribeKey", () -> client.describeKey(request)); + } + + public ScheduleKeyDeletionResult scheduleKeyDeletion(ScheduleKeyDeletionRequest request) { + // Default AWS limit was 5 as of Aug 2017 + return execute("KmsScheduleKeyDeletion", () -> client.scheduleKeyDeletion(request)); + } + + public GetKeyPolicyResult getKeyPolicy(GetKeyPolicyRequest request) { + // Default AWS limit was 30 as of Aug 2017 + return execute("KmsGetKeyPolicy", () -> client.getKeyPolicy(request)); + } + + public PutKeyPolicyResult putKeyPolicy(PutKeyPolicyRequest request) { + // Default AWS limit was 5 as of Aug 2017 + return execute("KmsPutKeyPolicy", () -> client.putKeyPolicy(request)); + } + + /** + * Execute a function that returns a value in a ThreadPool unique to that command. + */ + private static T execute(String commandKey, Supplier function) { + return execute(commandKey, commandKey, function); + } + + /** + * Execute a function that returns a value in a specified ThreadPool + */ + private static T execute(String threadPoolName, String commandKey, Supplier function) { + return new HystrixCommand(buildSetter(threadPoolName, commandKey)) { + + @Override + protected T run() throws Exception { + return function.get(); + } + }.execute(); + } + + private static HystrixCommand.Setter buildSetter(String threadPoolName, String commandKey) { + return HystrixCommand.Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey(KMS)) + .andThreadPoolKey(HystrixThreadPoolKey.Factory.asKey(threadPoolName)) + .andCommandKey(HystrixCommandKey.Factory.asKey(commandKey)); + } + +} diff --git a/src/main/java/com/nike/cerberus/hystrix/HystrixKmsClientFactory.java b/src/main/java/com/nike/cerberus/hystrix/HystrixKmsClientFactory.java new file mode 100644 index 000000000..62d8ee088 --- /dev/null +++ b/src/main/java/com/nike/cerberus/hystrix/HystrixKmsClientFactory.java @@ -0,0 +1,23 @@ +package com.nike.cerberus.hystrix; + +import com.amazonaws.regions.Region; +import com.nike.cerberus.aws.KmsClientFactory; + +public class HystrixKmsClientFactory extends KmsClientFactory { + + private final KmsClientFactory kmsClientFactory; + + public HystrixKmsClientFactory(KmsClientFactory kmsClientFactory) { + this.kmsClientFactory = kmsClientFactory; + } + + @Override + public HystrixKmsClient getClient(Region region) { + return new HystrixKmsClient(kmsClientFactory.getClient(region)); + } + + @Override + public HystrixKmsClient getClient(String regionName) { + return new HystrixKmsClient(kmsClientFactory.getClient(regionName)); + } +} \ No newline at end of file diff --git a/src/main/java/com/nike/cerberus/hystrix/HystrixMetricsLogger.java b/src/main/java/com/nike/cerberus/hystrix/HystrixMetricsLogger.java new file mode 100644 index 000000000..f7361f63c --- /dev/null +++ b/src/main/java/com/nike/cerberus/hystrix/HystrixMetricsLogger.java @@ -0,0 +1,75 @@ +package com.nike.cerberus.hystrix; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.google.inject.name.Named; +import com.netflix.hystrix.HystrixCircuitBreaker; +import com.netflix.hystrix.HystrixCommandMetrics; +import com.netflix.hystrix.HystrixThreadPoolMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * Periodically print Hystrix metrics to the log. + */ +@Singleton +public class HystrixMetricsLogger implements Runnable { + + private static final Logger LOGGER = LoggerFactory.getLogger(HystrixMetricsLogger.class); + + private final ScheduledExecutorService executor; + + @Inject + public HystrixMetricsLogger(@Named("hystrixExecutor") ScheduledExecutorService executor, + @Named("cms.hystrix.metricsLoggingInitialIntervalSeconds") long initialIntervalSeconds, + @Named("cms.hystrix.metricsLoggingIntervalSeconds") long intervalSeconds) { + this.executor = executor; + LOGGER.info("Hystrix metrics logging initialIntervalSeconds:{}, intervalSeconds:{}", initialIntervalSeconds, intervalSeconds); + this.executor.scheduleWithFixedDelay(this, initialIntervalSeconds, intervalSeconds, TimeUnit.SECONDS); + } + + @Override + public void run() { + try { + printHystrixCommandMetrics(); + printHystrixThreadPoolMetrics(); + } catch (Exception e) { + LOGGER.warn("Error printing Hystrix metrics", e); + } + } + + public void printHystrixCommandMetrics() { + for (HystrixCommandMetrics metrics : HystrixCommandMetrics.getInstances()) { + boolean isCircuitOpen = HystrixCircuitBreaker.Factory.getInstance(metrics.getCommandKey()).isOpen(); + + LOGGER.info("group:{}, commandKey:{}, CircuitOpen:{}, Mean:{}, 95%:{}, 99%:{}, 99.5%:{}, {}", + metrics.getCommandGroup().name(), + metrics.getCommandKey().name(), + isCircuitOpen, + metrics.getExecutionTimeMean(), + metrics.getExecutionTimePercentile(95.0), + metrics.getExecutionTimePercentile(99.5), + metrics.getExecutionTimePercentile(99.5), + metrics.getHealthCounts() + ); + } + } + + public void printHystrixThreadPoolMetrics() { + for (HystrixThreadPoolMetrics metrics : HystrixThreadPoolMetrics.getInstances()) { + LOGGER.info("threadPool:{}, rollingCounts[rejected:{}, executed:{}, maxActiveThreads:{}], cumulativeCounts[rejected:{}, executed:{}], {}", + metrics.getThreadPoolKey().name(), + metrics.getRollingCountThreadsRejected(), + metrics.getRollingCountThreadsExecuted(), + metrics.getRollingMaxActiveThreads(), + metrics.getCumulativeCountThreadsRejected(), + metrics.getCumulativeCountThreadsExecuted(), + metrics.getThreadPool() + ); + } + } + +} \ No newline at end of file diff --git a/src/main/java/com/nike/cerberus/hystrix/HystrixRequestAndResponseFilter.java b/src/main/java/com/nike/cerberus/hystrix/HystrixRequestAndResponseFilter.java new file mode 100644 index 000000000..eabf4933a --- /dev/null +++ b/src/main/java/com/nike/cerberus/hystrix/HystrixRequestAndResponseFilter.java @@ -0,0 +1,46 @@ +package com.nike.cerberus.hystrix; + +import com.netflix.hystrix.strategy.concurrency.HystrixRequestContext; +import com.nike.riposte.server.http.RequestInfo; +import com.nike.riposte.server.http.ResponseInfo; +import com.nike.riposte.server.http.filter.RequestAndResponseFilter; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Riposte filter that sets up and shuts down a HystrixRequestContext + */ +public class HystrixRequestAndResponseFilter implements RequestAndResponseFilter { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Override + public RequestInfo filterRequestFirstChunkNoPayload(RequestInfo currentRequestInfo, + ChannelHandlerContext ctx) { + HystrixRequestContext context = HystrixRequestContext.initializeContext(); + currentRequestInfo.addRequestAttribute("HystrixRequestContext", context); + return currentRequestInfo; + } + + @Override + public RequestInfo filterRequestLastChunkWithFullPayload(RequestInfo currentRequestInfo, + ChannelHandlerContext ctx) { + // Nothing to do - the other filterRequest method already handled Hystrix initialization. + // Returning null just means use the passed-in response, which is what we want. + return null; + } + + @Override + public ResponseInfo filterResponse(ResponseInfo currentResponseInfo, RequestInfo requestInfo, + ChannelHandlerContext ctx) { + try { + ((HystrixRequestContext) requestInfo.getRequestAttributes().get("HystrixRequestContext")).shutdown(); + } catch (Throwable t) { + logger.error("An unexpected error occurred trying to shutdown the HystrixRequestContext for this request.", t); + } + + // Returning null just means use the passed-in response, which is what we want. + return null; + } +} \ No newline at end of file diff --git a/src/main/java/com/nike/cerberus/hystrix/HystrixVaultAdminClient.java b/src/main/java/com/nike/cerberus/hystrix/HystrixVaultAdminClient.java new file mode 100644 index 000000000..3ae9da0a6 --- /dev/null +++ b/src/main/java/com/nike/cerberus/hystrix/HystrixVaultAdminClient.java @@ -0,0 +1,103 @@ +package com.nike.cerberus.hystrix; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.netflix.hystrix.HystrixCommand; +import com.netflix.hystrix.HystrixCommandGroupKey; +import com.netflix.hystrix.HystrixCommandKey; +import com.nike.vault.client.VaultAdminClient; +import com.nike.vault.client.model.VaultAuthResponse; +import com.nike.vault.client.model.VaultClientTokenResponse; +import com.nike.vault.client.model.VaultListResponse; +import com.nike.vault.client.model.VaultPolicy; +import com.nike.vault.client.model.VaultTokenAuthRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.function.Supplier; + + +/** + * Hystrix wrapper for VaultAdminClient. + */ +@Singleton +public class HystrixVaultAdminClient { + + private static final String VAULT = "Vault"; + + private static final Logger LOGGER = LoggerFactory.getLogger(HystrixVaultAdminClient.class); + private final VaultAdminClient vaultAdminClient; + + @Inject + public HystrixVaultAdminClient(final VaultAdminClient vaultAdminClient) { + this.vaultAdminClient = vaultAdminClient; + } + + public VaultAuthResponse createOrphanToken(VaultTokenAuthRequest vaultTokenAuthRequest) { + return execute("VaultCreateOrphanToken", () -> { + try { + return vaultAdminClient.createOrphanToken(vaultTokenAuthRequest); + } + catch (Exception e) { + LOGGER.warn("createOrphanToken failed, retrying...", e); + return vaultAdminClient.createOrphanToken(vaultTokenAuthRequest); + } + }); + } + + public VaultClientTokenResponse lookupToken(String vaultToken) { + return execute("VaultLookupToken", () -> vaultAdminClient.lookupToken(vaultToken)); + } + + public void revokeOrphanToken(final String vaultToken) { + execute("VaultRevokeOrphanToken", () -> vaultAdminClient.revokeOrphanToken(vaultToken)); + } + + public VaultListResponse list(final String path) { + return execute("VaultList", () -> vaultAdminClient.list(path)); + } + + public void delete(final String path) { + execute("VaultDelete", () -> vaultAdminClient.delete(path)); + } + + public void putPolicy(final String name, final VaultPolicy policy) { + execute("VaultPutPolicy", () -> vaultAdminClient.putPolicy(name, policy)); + } + + public void deletePolicy(final String name) { + execute("VaultDeletePolicy", () -> vaultAdminClient.deletePolicy(name)); + } + + /** + * Execute a function that returns a value + */ + private static T execute(String commandKey, Supplier function) { + return new HystrixCommand(buildSetter(commandKey)) { + + @Override + protected T run() throws Exception { + return function.get(); + } + }.execute(); + } + + /** + * Execute a function with void return type + */ + private static void execute(String commandKey, Runnable function) { + new HystrixCommand(buildSetter(commandKey)) { + + @Override + protected Void run() throws Exception { + function.run(); + return null; + } + }.execute(); + } + + private static HystrixCommand.Setter buildSetter(String commandKey) { + return HystrixCommand.Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey(VAULT)) + .andCommandKey(HystrixCommandKey.Factory.asKey(commandKey)); + } +} diff --git a/src/main/java/com/nike/cerberus/security/CmsRequestSecurityValidator.java b/src/main/java/com/nike/cerberus/security/CmsRequestSecurityValidator.java index ec6350e32..8cd93f634 100644 --- a/src/main/java/com/nike/cerberus/security/CmsRequestSecurityValidator.java +++ b/src/main/java/com/nike/cerberus/security/CmsRequestSecurityValidator.java @@ -18,7 +18,7 @@ import com.nike.backstopper.exception.ApiException; import com.nike.cerberus.error.DefaultApiError; -import com.nike.vault.client.VaultAdminClient; +import com.nike.cerberus.hystrix.HystrixVaultAdminClient; import com.nike.vault.client.VaultClientException; import com.nike.vault.client.VaultServerException; import com.nike.vault.client.model.VaultClientTokenResponse; @@ -48,10 +48,10 @@ public class CmsRequestSecurityValidator implements RequestSecurityValidator { private final Collection> endpointsToValidate; - private final VaultAdminClient vaultAdminClient; + private final HystrixVaultAdminClient vaultAdminClient; public CmsRequestSecurityValidator(final Collection> endpointsToValidate, - final VaultAdminClient vaultAdminClient) { + final HystrixVaultAdminClient vaultAdminClient) { this.endpointsToValidate = endpointsToValidate; this.vaultAdminClient = vaultAdminClient; this.endpointsToValidate.forEach(endpoint -> log.info("auth protected: {}", endpoint.getClass().getName())); diff --git a/src/main/java/com/nike/cerberus/server/config/CmsConfig.java b/src/main/java/com/nike/cerberus/server/config/CmsConfig.java index 2e3896e69..e1b945d1e 100644 --- a/src/main/java/com/nike/cerberus/server/config/CmsConfig.java +++ b/src/main/java/com/nike/cerberus/server/config/CmsConfig.java @@ -17,8 +17,16 @@ package com.nike.cerberus.server.config; import com.google.inject.util.Modules; +import com.netflix.config.ConfigurationManager; +import com.netflix.config.DynamicPropertyFactory; import com.nike.backstopper.handler.riposte.config.guice.BackstopperRiposteConfigGuiceModule; -import com.nike.cerberus.server.config.guice.*; +import com.nike.cerberus.server.config.guice.CmsFlywayModule; +import com.nike.cerberus.server.config.guice.CmsGuiceModule; +import com.nike.cerberus.server.config.guice.CmsMyBatisModule; +import com.nike.cerberus.server.config.guice.GuiceProvidedServerConfigValues; +import com.nike.cerberus.server.config.guice.MetricsGuiceModule; +import com.nike.cerberus.server.config.guice.OneLoginGuiceModule; +import com.nike.cerberus.util.ArchaiusUtils; import com.nike.guice.PropertiesRegistrationGuiceModule; import com.nike.guice.typesafeconfig.TypesafeConfigPropertiesRegistrationGuiceModule; import com.nike.riposte.metrics.MetricsListener; @@ -29,6 +37,7 @@ import com.nike.riposte.server.error.validation.RequestSecurityValidator; import com.nike.riposte.server.error.validation.RequestValidator; import com.nike.riposte.server.http.Endpoint; +import com.nike.riposte.server.http.filter.RequestAndResponseFilter; import com.nike.riposte.server.logging.AccessLogger; import com.fasterxml.jackson.databind.DeserializationFeature; @@ -46,6 +55,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Properties; import java.util.concurrent.CompletableFuture; /** @@ -84,9 +94,10 @@ protected CmsConfig(Config appConfig, PropertiesRegistrationGuiceModule properti throw new IllegalArgumentException("appConfig cannot be null"); this.appConfig = appConfig; - this.objectMapper = configureObjectMapper(); + ArchaiusUtils.initializeArchiaus(appConfig); + // Create a Guice Injector for this app. List appGuiceModules = new ArrayList<>(); appGuiceModules.add(propertiesRegistrationGuiceModule); @@ -211,4 +222,9 @@ public int numWorkerThreads() { public int maxRequestSizeInBytes() { return guiceValues.maxRequestSizeInBytes; } + + @Override + public List requestAndResponseFilters() { + return guiceValues.requestAndResponseFilters; + } } diff --git a/src/main/java/com/nike/cerberus/server/config/guice/CmsGuiceModule.java b/src/main/java/com/nike/cerberus/server/config/guice/CmsGuiceModule.java index b7d40a22a..ffa824189 100644 --- a/src/main/java/com/nike/cerberus/server/config/guice/CmsGuiceModule.java +++ b/src/main/java/com/nike/cerberus/server/config/guice/CmsGuiceModule.java @@ -21,8 +21,10 @@ import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.google.inject.name.Names; +import com.netflix.config.ConfigurationManager; import com.nike.backstopper.apierror.projectspecificinfo.ProjectApiErrors; import com.nike.cerberus.auth.connector.AuthConnector; +import com.nike.cerberus.aws.KmsClientFactory; import com.nike.cerberus.config.CmsEnvPropertiesLoader; import com.nike.cerberus.endpoints.HealthCheckEndpoint; import com.nike.cerberus.endpoints.admin.CleanUpInactiveOrOrphanedRecords; @@ -49,7 +51,12 @@ import com.nike.cerberus.endpoints.sdb.UpdateSafeDepositBoxV1; import com.nike.cerberus.endpoints.sdb.UpdateSafeDepositBoxV2; import com.nike.cerberus.error.DefaultApiErrorsImpl; +import com.nike.cerberus.auth.connector.AuthConnector; +import com.nike.cerberus.hystrix.HystrixKmsClientFactory; +import com.nike.cerberus.hystrix.HystrixMetricsLogger; +import com.nike.cerberus.hystrix.HystrixVaultAdminClient; import com.nike.cerberus.security.CmsRequestSecurityValidator; +import com.nike.cerberus.util.ArchaiusUtils; import com.nike.cerberus.util.UuidSupplier; import com.nike.cerberus.vault.CmsVaultCredentialsProvider; import com.nike.cerberus.vault.CmsVaultUrlResolver; @@ -72,11 +79,15 @@ import javax.validation.Validation; import javax.validation.Validator; import java.util.Arrays; +import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.stream.Collectors; public class CmsGuiceModule extends AbstractModule { @@ -124,6 +135,8 @@ protected void configure() { } catch(ClassCastException cce) { throw new IllegalArgumentException("class: " + className + " is the wrong type", cce); } + + bind(HystrixMetricsLogger.class).asEagerSingleton(); } private void loadEnvProperties() { @@ -140,6 +153,9 @@ private void loadEnvProperties() { // bind the props to named props for guice Names.bindProperties(binder(), properties); + // properties from cms.conf may be overridden in environment.properties + ArchaiusUtils.loadProperties(properties); + for (String propertyName : properties.stringPropertyNames()) { logger.info("Successfully loaded: {} from the env data stored in S3", propertyName); appConfig = appConfig.withValue(propertyName, ConfigValueFactory.fromAnyRef(properties.getProperty(propertyName))); @@ -223,9 +239,31 @@ public UuidSupplier uuidSupplier() { @Provides public VaultAdminClient vaultAdminClient(UrlResolver urlResolver, VaultCredentialsProvider vaultCredentialsProvider, - @Named("vault.maxRequestsPerHost") int vaultMaxRequestsPerHost) { - logger.info("Vault clientVersion={}, maxRequestsPerHost={}, url={}", ClientVersion.getVersion(), vaultMaxRequestsPerHost, urlResolver.resolve()); - return VaultClientFactory.getAdminClient(urlResolver, vaultCredentialsProvider, vaultMaxRequestsPerHost); + @Named("vault.maxRequests") int vaultMaxRequests, + @Named("vault.maxRequestsPerHost") int vaultMaxRequestsPerHost, + @Named("vault.connectTimeoutMillis") int vaultConnectTimeoutMillis, + @Named("vault.readTimeoutMillis")int vaultReadTimeoutMillis, + @Named("vault.writeTimeoutMillis") int vaultWriteTimeoutMillis, + @Named("service.version") String cmsVersion) { + String version = ClientVersion.getVersion(); + logger.info("Vault clientVersion={}, maxRequests={}, maxRequestsPerHost={}, connectTimeoutMillis={}, readTimeoutMillis={}, writeTimeoutMillis={}, url={}", + version, + vaultMaxRequests, + vaultMaxRequestsPerHost, + vaultConnectTimeoutMillis, + vaultReadTimeoutMillis, + vaultWriteTimeoutMillis, + urlResolver.resolve()); + + return VaultClientFactory.getAdminClient(urlResolver, + vaultCredentialsProvider, + vaultMaxRequests, + vaultMaxRequestsPerHost, + vaultConnectTimeoutMillis, + vaultReadTimeoutMillis, + vaultWriteTimeoutMillis, + new HashMap<>() + ); } @Provides @@ -243,7 +281,7 @@ public List> authProtectedEndpoints(@Named("appEndpoints") Set> authProtectedEndpoints, - VaultAdminClient vaultAdminClient) { + HystrixVaultAdminClient vaultAdminClient) { return new CmsRequestSecurityValidator(authProtectedEndpoints, vaultAdminClient); } @@ -253,4 +291,17 @@ public CmsRequestSecurityValidator authRequestSecurityValidator( public CompletableFuture appInfoFuture(AsyncHttpClientHelper asyncHttpClientHelper) { return AwsUtil.getAppInfoFutureWithAwsInfo(asyncHttpClientHelper); } + + @Provides + @Singleton + @Named("hystrixExecutor") + public ScheduledExecutorService executor() { + return Executors.newSingleThreadScheduledExecutor(); + } + + @Provides + @Singleton + public KmsClientFactory hystrixKmsClientFactory() { + return new HystrixKmsClientFactory(new KmsClientFactory()); + } } diff --git a/src/main/java/com/nike/cerberus/server/config/guice/GuiceProvidedServerConfigValues.java b/src/main/java/com/nike/cerberus/server/config/guice/GuiceProvidedServerConfigValues.java index b136eb6a2..ff9da84ef 100644 --- a/src/main/java/com/nike/cerberus/server/config/guice/GuiceProvidedServerConfigValues.java +++ b/src/main/java/com/nike/cerberus/server/config/guice/GuiceProvidedServerConfigValues.java @@ -16,6 +16,8 @@ package com.nike.cerberus.server.config.guice; +import com.google.common.collect.Lists; +import com.nike.cerberus.hystrix.HystrixRequestAndResponseFilter; import com.nike.cerberus.security.CmsRequestSecurityValidator; import com.nike.cerberus.server.config.CmsConfig; import com.nike.riposte.metrics.codahale.CodahaleMetricsListener; @@ -25,7 +27,9 @@ import com.nike.riposte.server.error.handler.RiposteUnhandledErrorHandler; import com.nike.riposte.server.error.validation.RequestValidator; import com.nike.riposte.server.http.Endpoint; +import com.nike.riposte.server.http.filter.RequestAndResponseFilter; +import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -47,6 +51,7 @@ public class GuiceProvidedServerConfigValues extends DependencyInjectionProvided public final CodahaleMetricsListener metricsListener; public final CompletableFuture appInfoFuture; public final CmsRequestSecurityValidator cmsRequestSecurityValidator; + public final List requestAndResponseFilters; @Inject public GuiceProvidedServerConfigValues(@Named("endpoints.port") Integer endpointsPort, @@ -63,7 +68,8 @@ public GuiceProvidedServerConfigValues(@Named("endpoints.port") Integer endpoint RequestValidator validationService, @Nullable CodahaleMetricsListener metricsListener, @Named("appInfoFuture") CompletableFuture appInfoFuture, - CmsRequestSecurityValidator cmsRequestSecurityValidator + CmsRequestSecurityValidator cmsRequestSecurityValidator, + HystrixRequestAndResponseFilter hystrixRequestAndResponseFilter ) { super(endpointsPort, endpointsSslPort, endpointsUseSsl, numBossThreads, numWorkerThreads, maxRequestSizeInBytes, appEndpoints, debugActionsEnabled, debugChannelLifecycleLoggingEnabled); @@ -74,5 +80,6 @@ public GuiceProvidedServerConfigValues(@Named("endpoints.port") Integer endpoint this.metricsListener = metricsListener; this.appInfoFuture = appInfoFuture; this.cmsRequestSecurityValidator = cmsRequestSecurityValidator; + this.requestAndResponseFilters = Lists.newArrayList(hystrixRequestAndResponseFilter); } } diff --git a/src/main/java/com/nike/cerberus/service/AuthenticationService.java b/src/main/java/com/nike/cerberus/service/AuthenticationService.java index b5a16d552..38ab25de7 100644 --- a/src/main/java/com/nike/cerberus/service/AuthenticationService.java +++ b/src/main/java/com/nike/cerberus/service/AuthenticationService.java @@ -46,6 +46,7 @@ import com.nike.cerberus.domain.MfaCheckRequest; import com.nike.cerberus.domain.UserCredentials; import com.nike.cerberus.error.DefaultApiError; +import com.nike.cerberus.hystrix.HystrixVaultAdminClient; import com.nike.cerberus.record.AwsIamRoleKmsKeyRecord; import com.nike.cerberus.record.AwsIamRoleRecord; import com.nike.cerberus.record.SafeDepositBoxRoleRecord; @@ -97,7 +98,7 @@ public class AuthenticationService { private final AuthConnector authServiceConnector; private final KmsService kmsService; private final KmsClientFactory kmsClientFactory; - private final VaultAdminClient vaultAdminClient; + private final HystrixVaultAdminClient vaultAdminClient; private final VaultPolicyService vaultPolicyService; private final ObjectMapper objectMapper; private final String adminGroup; @@ -126,7 +127,7 @@ public AuthenticationService(final SafeDepositBoxDao safeDepositBoxDao, final AuthConnector authConnector, final KmsService kmsService, final KmsClientFactory kmsClientFactory, - final VaultAdminClient vaultAdminClient, + final HystrixVaultAdminClient vaultAdminClient, final VaultPolicyService vaultPolicyService, final ObjectMapper objectMapper, @Named(ADMIN_GROUP_PROPERTY) final String adminGroup, @@ -243,6 +244,7 @@ private IamRoleAuthResponse authenticate(IamPrincipalCredentials credentials, Ma .setTtl(iamTokenTTL) .setNoDefaultPolicy(true); + VaultAuthResponse authResponse = vaultAdminClient.createOrphanToken(tokenAuthRequest); byte[] authResponseJson; diff --git a/src/main/java/com/nike/cerberus/service/SafeDepositBoxService.java b/src/main/java/com/nike/cerberus/service/SafeDepositBoxService.java index f5d99ba4a..8755f26b1 100644 --- a/src/main/java/com/nike/cerberus/service/SafeDepositBoxService.java +++ b/src/main/java/com/nike/cerberus/service/SafeDepositBoxService.java @@ -30,6 +30,7 @@ import com.nike.cerberus.domain.SafeDepositBoxV2; import com.nike.cerberus.domain.UserGroupPermission; import com.nike.cerberus.error.DefaultApiError; +import com.nike.cerberus.hystrix.HystrixVaultAdminClient; import com.nike.cerberus.record.RoleRecord; import com.nike.cerberus.record.SafeDepositBoxRecord; import com.nike.cerberus.record.UserGroupRecord; @@ -38,7 +39,6 @@ import com.nike.cerberus.util.UuidSupplier; import com.nike.cerberus.util.DateTimeSupplier; import com.nike.cerberus.util.Slugger; -import com.nike.vault.client.VaultAdminClient; import com.nike.vault.client.VaultClientException; import com.nike.vault.client.model.VaultListResponse; import org.apache.commons.lang3.StringUtils; @@ -73,7 +73,7 @@ public class SafeDepositBoxService { private final RoleService roleService; - private final VaultAdminClient vaultAdminClient; + private final HystrixVaultAdminClient vaultAdminClient; private final VaultPolicyService vaultPolicyService; @@ -93,7 +93,7 @@ public SafeDepositBoxService(final SafeDepositBoxDao safeDepositBoxDao, final UuidSupplier uuidSupplier, final CategoryService categoryService, final RoleService roleService, - final VaultAdminClient vaultAdminClient, + final HystrixVaultAdminClient vaultAdminClient, final VaultPolicyService vaultPolicyService, final UserGroupPermissionService userGroupPermissionService, final IamPrincipalPermissionService iamPrincipalPermissionService, diff --git a/src/main/java/com/nike/cerberus/service/VaultPolicyService.java b/src/main/java/com/nike/cerberus/service/VaultPolicyService.java index 9b3c90856..bcd431ac9 100644 --- a/src/main/java/com/nike/cerberus/service/VaultPolicyService.java +++ b/src/main/java/com/nike/cerberus/service/VaultPolicyService.java @@ -19,9 +19,9 @@ import com.google.common.base.Preconditions; import com.nike.backstopper.exception.ApiException; import com.nike.cerberus.error.DefaultApiError; +import com.nike.cerberus.hystrix.HystrixVaultAdminClient; import com.nike.cerberus.record.RoleRecord; import com.nike.cerberus.util.Slugger; -import com.nike.vault.client.VaultAdminClient; import com.nike.vault.client.VaultClientException; import com.nike.vault.client.model.VaultPolicy; import org.apache.commons.lang3.StringUtils; @@ -49,12 +49,12 @@ public class VaultPolicyService { private final Logger logger = LoggerFactory.getLogger(getClass()); - private final VaultAdminClient vaultAdminClient; + private final HystrixVaultAdminClient vaultAdminClient; private final Slugger slugger; @Inject - public VaultPolicyService(final VaultAdminClient vaultAdminClient, final Slugger slugger) { + public VaultPolicyService(final HystrixVaultAdminClient vaultAdminClient, final Slugger slugger) { this.vaultAdminClient = vaultAdminClient; this.slugger = slugger; } diff --git a/src/main/java/com/nike/cerberus/util/ArchaiusUtils.java b/src/main/java/com/nike/cerberus/util/ArchaiusUtils.java new file mode 100644 index 000000000..ed5f6e9e3 --- /dev/null +++ b/src/main/java/com/nike/cerberus/util/ArchaiusUtils.java @@ -0,0 +1,34 @@ +package com.nike.cerberus.util; + +import com.netflix.config.ConfigurationManager; +import com.netflix.config.DynamicPropertyFactory; +import com.typesafe.config.Config; + +import java.util.Properties; + +public class ArchaiusUtils { + + /** + * Hystrix expects configuration via Archaius so we initialize it here + */ + public static void initializeArchiaus(Config appConfig) { + // Initialize Archaius + DynamicPropertyFactory.getInstance(); + // Load properties from Typesafe config for Hystrix, etc. + ConfigurationManager.loadProperties(toProperties(appConfig)); + } + + public static void loadProperties(Properties properties) { + ConfigurationManager.loadProperties(properties); + } + + /** + * Convert Typesafe config to Properties + * From https://github.com/typesafehub/config/issues/357 + */ + public static Properties toProperties(Config config) { + Properties properties = new Properties(); + config.entrySet().forEach(e -> properties.setProperty(e.getKey(), config.getString(e.getKey()))); + return properties; + } +} diff --git a/src/main/resources/cms.conf b/src/main/resources/cms.conf index f1d86223a..18e747039 100644 --- a/src/main/resources/cms.conf +++ b/src/main/resources/cms.conf @@ -79,7 +79,49 @@ c3p0.maxIdleTimeExcessConnections=4200 c3p0.preferredTestQuery=SELECT 1 # Vault Admin client -vault.maxRequestsPerHost=200 +vault.maxRequests=1000 +vault.maxRequestsPerHost=1000 +vault.connectTimeoutMillis=300 +vault.readTimeoutMillis=1000 +vault.writeTimeoutMillis=1000 + +# Hystrix Logging +cms.hystrix.metricsLoggingInitialIntervalSeconds=0 +cms.hystrix.metricsLoggingIntervalSeconds=600 + +# Hystrix Configuration +# See https://github.com/Netflix/Hystrix/wiki/Configuration + +# Vault Command Configuration +hystrix.command.VaultCreateOrphanToken.execution.isolation.thread.timeoutInMilliseconds=2100 + +# Vault Thread Pool Configuration +hystrix.threadpool.Vault.coreSize=20 + + +# KMS Thread Pool Configuration + +# Default AWS limit was 1200 shared as of Aug 2017 +hystrix.threadpool.KmsEncryptDecrypt.coreSize=30 + +# Default AWS limit was 5 as of Aug 2017 +hystrix.threadpool.KmsCreateKey.coreSize=5 + +# Default AWS limit was 5 as of Aug 2017 +hystrix.threadpool.KmsCreateAlias.coreSize=5 + +# Default AWS limit was 30 as of Aug 2017 +hystrix.threadpool.KmsDescribeKey.coreSize=10 + +# Default AWS limit was 5 as of Aug 2017 +hystrix.threadpool.KmsScheduleKeyDeletion.coreSize=5 + +# Default AWS limit was 30 as of Aug 2017 +hystrix.threadpool.KmsGetKeyPolicy.coreSize=10 + +# Default AWS limit was 5 as of Aug 2017 +hystrix.threadpool.KmsPutKeyPolicy.coreSize=5 + # Application name cms.app.name=cms diff --git a/src/test/java/com/nike/cerberus/security/CmsRequestSecurityValidatorTest.java b/src/test/java/com/nike/cerberus/security/CmsRequestSecurityValidatorTest.java index a4b5a85dc..92dd66783 100644 --- a/src/test/java/com/nike/cerberus/security/CmsRequestSecurityValidatorTest.java +++ b/src/test/java/com/nike/cerberus/security/CmsRequestSecurityValidatorTest.java @@ -19,7 +19,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.nike.backstopper.exception.ApiException; -import com.nike.vault.client.VaultAdminClient; +import com.nike.cerberus.hystrix.HystrixVaultAdminClient; import com.nike.vault.client.VaultClientException; import com.nike.vault.client.VaultServerException; import com.nike.vault.client.model.VaultClientTokenResponse; @@ -51,13 +51,13 @@ public class CmsRequestSecurityValidatorTest { private final Collection> securedEndpoints = Lists.newArrayList(securedEndpoint); - private VaultAdminClient vaultAdminClient; + private HystrixVaultAdminClient vaultAdminClient; private CmsRequestSecurityValidator subject; @Before public void setUp() throws Exception { - vaultAdminClient = mock(VaultAdminClient.class); + vaultAdminClient = mock(HystrixVaultAdminClient.class); subject = new CmsRequestSecurityValidator(securedEndpoints, vaultAdminClient); } diff --git a/src/test/java/com/nike/cerberus/service/AuthenticationServiceTest.java b/src/test/java/com/nike/cerberus/service/AuthenticationServiceTest.java index cc3c89171..b131e5522 100644 --- a/src/test/java/com/nike/cerberus/service/AuthenticationServiceTest.java +++ b/src/test/java/com/nike/cerberus/service/AuthenticationServiceTest.java @@ -26,6 +26,7 @@ import com.nike.cerberus.dao.SafeDepositBoxDao; import com.nike.cerberus.domain.IamPrincipalCredentials; import com.nike.cerberus.error.DefaultApiError; +import com.nike.cerberus.hystrix.HystrixVaultAdminClient; import com.nike.cerberus.record.AwsIamRoleKmsKeyRecord; import com.nike.cerberus.record.AwsIamRoleRecord; import com.nike.cerberus.record.SafeDepositBoxRoleRecord; @@ -88,7 +89,7 @@ public class AuthenticationServiceTest { private KmsClientFactory kmsClientFactory; @Mock - private VaultAdminClient vaultAdminClient; + private HystrixVaultAdminClient vaultAdminClient; @Mock private VaultPolicyService vaultPolicyService; diff --git a/src/test/java/com/nike/cerberus/service/VaultPolicyServiceTest.java b/src/test/java/com/nike/cerberus/service/VaultPolicyServiceTest.java index 6e7ac871f..38c486433 100644 --- a/src/test/java/com/nike/cerberus/service/VaultPolicyServiceTest.java +++ b/src/test/java/com/nike/cerberus/service/VaultPolicyServiceTest.java @@ -1,5 +1,6 @@ package com.nike.cerberus.service; +import com.nike.cerberus.hystrix.HystrixVaultAdminClient; import com.nike.cerberus.util.Slugger; import com.nike.vault.client.VaultAdminClient; import com.nike.vault.client.model.VaultPolicy; @@ -17,12 +18,12 @@ public class VaultPolicyServiceTest { private Slugger slugger = new Slugger(); - private VaultAdminClient vaultAdminClient; + private HystrixVaultAdminClient vaultAdminClient; private VaultPolicyService vaultPolicyService; @Before public void setup() { - vaultAdminClient = mock(VaultAdminClient.class); + vaultAdminClient = mock(HystrixVaultAdminClient.class); vaultPolicyService = new VaultPolicyService(vaultAdminClient, slugger); }