Performance rc mergable (#217)

* Add timeout for db connection pool waiting and retry with back off AWS STS Get Caller Identity requests
* Extra logging and hystrix tweaks

gradle.properties

@@ -14,6 +14,6 @@
 # limitations under the License.
 #
 
-version=3.34.0
+version=3.35.0
 groupId=com.nike.cerberus
 artifactId=cms

gradle/dependencies.gradle

@@ -77,8 +77,9 @@ dependencies {
             "com.okta.sdk:okta-sdk-httpclient:1.2.0",
             "com.okta.authn.sdk:okta-authn-sdk-impl:0.1.0",
             "org.reflections:reflections:0.9.11",
-            "com.github.ben-manes.caffeine:caffeine:2.8.0"
+            "com.github.ben-manes.caffeine:caffeine:2.8.0",
 
+            "io.github.resilience4j:resilience4j-all:1.1.0"
     )
 
     testCompile (

src/main/java/com/nike/cerberus/aws/sts/AwsStsClient.java

@@ -17,23 +17,40 @@
 package com.nike.cerberus.aws.sts;
 
 import com.google.inject.Singleton;
+import io.github.resilience4j.retry.IntervalFunction;
+import io.github.resilience4j.retry.Retry;
+import io.github.resilience4j.retry.RetryConfig;
 
 import javax.inject.Inject;
 
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
+
+import static io.github.resilience4j.decorators.Decorators.ofSupplier;
+
 /**
  * Client for calling AWS STS APIs
  */
 @Singleton
 public class AwsStsClient {
     private final AwsStsHttpClient httpClient;
 
+    private static final RetryConfig RETRY_CONFIG = RetryConfig.custom()
+        .maxAttempts(5)
+        .intervalFunction(IntervalFunction.ofExponentialBackoff(Duration.of(250, ChronoUnit.MILLIS)))
+        .build();
+
+    private final Retry retry = Retry.of(this.getClass().getName(), RETRY_CONFIG);
+
     @Inject
     public AwsStsClient(AwsStsHttpClient httpClient) {
         this.httpClient = httpClient;
     }
 
     public GetCallerIdentityResponse getCallerIdentity(AwsStsHttpHeader header) {
-        GetCallerIdentityFullResponse response = httpClient.execute(header.getRegion(), header.generateHeaders(), GetCallerIdentityFullResponse.class);
-        return response.getGetCallerIdentityResponse();
+        return ofSupplier(() -> {
+            GetCallerIdentityFullResponse response = httpClient.execute(header.getRegion(), header.generateHeaders(), GetCallerIdentityFullResponse.class);
+            return response.getGetCallerIdentityResponse();
+        }).withRetry(retry).decorate().get();
     }
 }

src/main/resources/cms.conf

@@ -68,6 +68,7 @@ c3p0.initialPoolSize=40
 c3p0.minPoolSize=40
 c3p0.maxPoolSize=400
 c3p0.acquireIncrement=10
+c3p0.checkoutTimeout=5000
 c3p0.testConnectionOnCheckin=true
 c3p0.idleConnectionTestPeriod=300
 c3p0.maxIdleTimeExcessConnections=4200
@@ -80,7 +81,9 @@ c3p0.preferredTestQuery=SELECT 1
 # KMS Thread Pool Configuration
 
 # Default AWS limit was 1200 shared as of Aug 2017
-hystrix.threadpool.KmsEncryptDecrypt.coreSize=1000
+hystrix.threadpool.KmsEncryptDecrypt.allowMaximumSizeToDivergeFromCoreSize=true
+hystrix.threadpool.KmsEncryptDecrypt.coreSize=10
+hystrix.threadpool.KmsEncryptDecrypt.maximumSize=1000
 hystrix.command.KmsEncrypt.execution.isolation.thread.timeoutInMilliseconds=3000
 
 # Default AWS limit was 5 as of Aug 2017
@@ -107,6 +110,12 @@ hystrix.threadpool.KmsPutKeyPolicy.coreSize=5
 hystrix.threadpool.ListKeysRequest.coreSize=5
 hystrix.command.ListKeysRequest.execution.isolation.thread.timeoutInMilliseconds=10000
 
+# Application Events
+hystrix.threadpool.event-processor-tp.allowMaximumSizeToDivergeFromCoreSize=true
+hystrix.threadpool.event-processor-tp.coreSize=10
+hystrix.threadpool.event-processor-tp.maximumSize=1000
+hystrix.command.process-event-command.execution.isolation.thread.timeoutInMilliseconds=2500
+
 # Application name
 cms.app.name=cms