From 1d5ee847d649fd437c2be140f9317762f854eaea Mon Sep 17 00:00:00 2001
From: Sean Lin <sean.lin@nike.com>
Date: Thu, 7 Nov 2019 15:43:04 -0800
Subject: [PATCH] Performance rc mergable (#217)

* Add timeout for db connection pool waiting and retry with back off AWS STS Get Caller Identity requests
* Extra logging and hystrix tweaks
---
 gradle.properties                             |  2 +-
 gradle/dependencies.gradle                    |  3 ++-
 .../nike/cerberus/aws/sts/AwsStsClient.java   | 21 +++++++++++++++++--
 src/main/resources/cms.conf                   | 11 +++++++++-
 4 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/gradle.properties b/gradle.properties
index a53107515..ca6df7cfb 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -14,6 +14,6 @@
 # limitations under the License.
 #
 
-version=3.34.0
+version=3.35.0
 groupId=com.nike.cerberus
 artifactId=cms
diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle
index cc297acfe..8a7e8fe79 100644
--- a/gradle/dependencies.gradle
+++ b/gradle/dependencies.gradle
@@ -77,8 +77,9 @@ dependencies {
             "com.okta.sdk:okta-sdk-httpclient:1.2.0",
             "com.okta.authn.sdk:okta-authn-sdk-impl:0.1.0",
             "org.reflections:reflections:0.9.11",
-            "com.github.ben-manes.caffeine:caffeine:2.8.0"
+            "com.github.ben-manes.caffeine:caffeine:2.8.0",
 
+            "io.github.resilience4j:resilience4j-all:1.1.0"
     )
 
     testCompile (
diff --git a/src/main/java/com/nike/cerberus/aws/sts/AwsStsClient.java b/src/main/java/com/nike/cerberus/aws/sts/AwsStsClient.java
index 76c471b23..f5bf87680 100644
--- a/src/main/java/com/nike/cerberus/aws/sts/AwsStsClient.java
+++ b/src/main/java/com/nike/cerberus/aws/sts/AwsStsClient.java
@@ -17,9 +17,17 @@
 package com.nike.cerberus.aws.sts;
 
 import com.google.inject.Singleton;
+import io.github.resilience4j.retry.IntervalFunction;
+import io.github.resilience4j.retry.Retry;
+import io.github.resilience4j.retry.RetryConfig;
 
 import javax.inject.Inject;
 
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
+
+import static io.github.resilience4j.decorators.Decorators.ofSupplier;
+
 /**
  * Client for calling AWS STS APIs
  */
@@ -27,13 +35,22 @@
 public class AwsStsClient {
     private final AwsStsHttpClient httpClient;
 
+    private static final RetryConfig RETRY_CONFIG = RetryConfig.custom()
+        .maxAttempts(5)
+        .intervalFunction(IntervalFunction.ofExponentialBackoff(Duration.of(250, ChronoUnit.MILLIS)))
+        .build();
+
+    private final Retry retry = Retry.of(this.getClass().getName(), RETRY_CONFIG);
+
     @Inject
     public AwsStsClient(AwsStsHttpClient httpClient) {
         this.httpClient = httpClient;
     }
 
     public GetCallerIdentityResponse getCallerIdentity(AwsStsHttpHeader header) {
-        GetCallerIdentityFullResponse response = httpClient.execute(header.getRegion(), header.generateHeaders(), GetCallerIdentityFullResponse.class);
-        return response.getGetCallerIdentityResponse();
+        return ofSupplier(() -> {
+            GetCallerIdentityFullResponse response = httpClient.execute(header.getRegion(), header.generateHeaders(), GetCallerIdentityFullResponse.class);
+            return response.getGetCallerIdentityResponse();
+        }).withRetry(retry).decorate().get();
     }
 }
diff --git a/src/main/resources/cms.conf b/src/main/resources/cms.conf
index 81ba03e26..5cf2ee6f1 100644
--- a/src/main/resources/cms.conf
+++ b/src/main/resources/cms.conf
@@ -68,6 +68,7 @@ c3p0.initialPoolSize=40
 c3p0.minPoolSize=40
 c3p0.maxPoolSize=400
 c3p0.acquireIncrement=10
+c3p0.checkoutTimeout=5000
 c3p0.testConnectionOnCheckin=true
 c3p0.idleConnectionTestPeriod=300
 c3p0.maxIdleTimeExcessConnections=4200
@@ -80,7 +81,9 @@ c3p0.preferredTestQuery=SELECT 1
 # KMS Thread Pool Configuration
 
 # Default AWS limit was 1200 shared as of Aug 2017
-hystrix.threadpool.KmsEncryptDecrypt.coreSize=1000
+hystrix.threadpool.KmsEncryptDecrypt.allowMaximumSizeToDivergeFromCoreSize=true
+hystrix.threadpool.KmsEncryptDecrypt.coreSize=10
+hystrix.threadpool.KmsEncryptDecrypt.maximumSize=1000
 hystrix.command.KmsEncrypt.execution.isolation.thread.timeoutInMilliseconds=3000
 
 # Default AWS limit was 5 as of Aug 2017
@@ -107,6 +110,12 @@ hystrix.threadpool.KmsPutKeyPolicy.coreSize=5
 hystrix.threadpool.ListKeysRequest.coreSize=5
 hystrix.command.ListKeysRequest.execution.isolation.thread.timeoutInMilliseconds=10000
 
+# Application Events
+hystrix.threadpool.event-processor-tp.allowMaximumSizeToDivergeFromCoreSize=true
+hystrix.threadpool.event-processor-tp.coreSize=10
+hystrix.threadpool.event-processor-tp.maximumSize=1000
+hystrix.command.process-event-command.execution.isolation.thread.timeoutInMilliseconds=2500
+
 # Application name
 cms.app.name=cms