feat(caching): add rate triggered MyBatis database caching. (#213)

* feat(caching): add rate triggered MyBatis database caching.

* chore: update default settings and docs

* fix: grammer in cache docs

* refactor: use Metric Reporting Cache instead of duplicating logic

src/main/java/com/nike/cerberus/cache/DatabaseCache.java

@@ -0,0 +1,207 @@
+/*
+ * Copyright (c) 2019 Nike, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.nike.cerberus.cache;
+
+import com.codahale.metrics.Counter;
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Injector;
+import com.nike.cerberus.server.config.guice.StaticInjector;
+import com.nike.cerberus.service.MetricsService;
+import com.typesafe.config.Config;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.ibatis.builder.InitializingObject;
+import org.apache.ibatis.cache.Cache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReadWriteLock;
+
+import static com.github.benmanes.caffeine.cache.Caffeine.newBuilder;
+import static java.util.Optional.ofNullable;
+
+/**
+ * This is a custom MyBatis Cache, that allows use to do the following
+ * 1. Report cache statistics via Dropwizard
+ * 2. Expire items automatically after some TTL from when they were cached. (To avoid needing to deal with distributed cache busting, this basically makes cached data eventually consistent up to the defined TTL)
+ * 3. Only cache items after it has been proven via repeat reads that they should be cached. (To avoid unnecessary eventual consistency in the dashboard, only make the items under heavy reads eventually consistent)
+ * See cms.conf for all the configuration settings.
+ */
+public class DatabaseCache implements Cache, InitializingObject {
+
+  private final Logger log = LoggerFactory.getLogger(getClass());
+  private Integer repeatReadThreshold;
+
+  protected static final String GLOBAL_DATA_TTL_IN_SECONDS = "cms.mybatis.cache.global.dataTtlInSeconds";
+  protected static final String DATA_TTL_IN_SECONDS_OVERRIDE_PATH_TEMPLATE = "cms.mybatis.cache.%s.dataTtlInSeconds";
+  protected static final String GLOBAL_REPEAT_READ_COUNTER_EXPIRE_IN_SECONDS = "cms.mybatis.cache.global.repeatReadCounterResetInSeconds";
+  protected static final String REPEAT_READ_COUNTER_EXPIRE_IN_SECONDS_OVERRIDE_PATH_TEMPLATE = "cms.mybatis.cache.%s.repeatReadCounterResetInSeconds";
+  protected static final String GLOBAL_REPEAT_READ_THRESHOLD = "cms.mybatis.cache.global.repeatReadThreshold";
+  protected static final String REPEAT_READ_THRESHOLD_OVERRIDE_PATH_TEMPLATE = "cms.mybatis.cache.%s.repeatReadThreshold";
+  protected static final int DEFAULT_GLOBAL_DATA_TTL_IN_SECONDS = 10;
+  protected static final int DEFAULT_REPEAT_READ_COUNTER_EXPIRE_IN_SECONDS = 2;
+  protected static final int DEFAULT_REPEAT_READ_THRESHOLD = 2;
+
+  protected final String id;
+  protected MetricReportingCache<Object, Object> dataCache;
+  protected com.github.benmanes.caffeine.cache.Cache<Object, Counter> autoExpiringRepeatReadCounterMap;
+
+  public DatabaseCache(String id) {
+    this.id = id;
+  }
+
+  /**
+   * This method gets called after this class is instantiated by MyBatis and all the properties have been set.
+   */
+  @Override
+  public void initialize() {
+    // Util we can get the MyBatis Guice Module updated, this is our best bet, for getting the Guice instances.
+    // https://groups.google.com/forum/#!msg/mybatis-user/Ekd1LTNVIDc/t2xGuvETBgAJ
+    Injector injector = StaticInjector.getInstance();
+
+    Config config = injector.getInstance(Config.class);
+    MetricsService metricsService = injector.getInstance(MetricsService.class);
+
+    String mapperKey = StringUtils.uncapitalize(id.replaceFirst("com.nike.cerberus.mapper.", ""));
+    int expireTimeInSeconds = getExpireTimeInSeconds(config, mapperKey);
+    int counterExpireTimeInSeconds = getRepeatReadCounterExpireTimeInSeconds(config, mapperKey);
+    repeatReadThreshold = getRepeatReadThreshold(config, mapperKey);
+
+    log.info("Database cache created with mapperKey: {}, expireTimeInSeconds: {}, counterExpireTimeInSeconds: {}, repeatReadThreshold: {}",
+        mapperKey, expireTimeInSeconds, counterExpireTimeInSeconds, repeatReadThreshold);
+
+    dataCache = new MetricReportingCache<>("mybatis", expireTimeInSeconds, metricsService,
+        ImmutableMap.of("namespace", this.id));
+
+    autoExpiringRepeatReadCounterMap = newBuilder()
+        .expireAfterAccess(counterExpireTimeInSeconds, TimeUnit.SECONDS)
+        .build();
+  }
+
+  /**
+   * @param config The application config
+   * @param mapperKey The key for this mapper
+   * @return The amount of time in seconds that the mapper cache will keep an item in memory before it purges itself.
+   */
+  protected int getExpireTimeInSeconds(Config config, String mapperKey) {
+    int globalExpireTimeInSeconds = config.hasPath(GLOBAL_DATA_TTL_IN_SECONDS) ? config.getInt(GLOBAL_DATA_TTL_IN_SECONDS) : DEFAULT_GLOBAL_DATA_TTL_IN_SECONDS;
+    String globalDataTtlInSecondsOverridePathTemplate = String.format(DATA_TTL_IN_SECONDS_OVERRIDE_PATH_TEMPLATE, mapperKey);
+    return config.hasPath(globalDataTtlInSecondsOverridePathTemplate) ? config.getInt(globalDataTtlInSecondsOverridePathTemplate) : globalExpireTimeInSeconds;
+  }
+
+  /**
+   * @param config The application config
+   * @param mapperKey The key for this mapper
+   * @return The amount of time in seconds that must pass without consecutive reads to reset the counter.
+   */
+  protected int getRepeatReadCounterExpireTimeInSeconds(Config config, String mapperKey) {
+    int globalCounterExpireTimeInSeconds = config.hasPath(GLOBAL_REPEAT_READ_COUNTER_EXPIRE_IN_SECONDS) ? config.getInt(GLOBAL_REPEAT_READ_COUNTER_EXPIRE_IN_SECONDS) : DEFAULT_REPEAT_READ_COUNTER_EXPIRE_IN_SECONDS;
+    String counterMapperOverrideTtlPath = String.format(REPEAT_READ_COUNTER_EXPIRE_IN_SECONDS_OVERRIDE_PATH_TEMPLATE, mapperKey);
+    return config.hasPath(counterMapperOverrideTtlPath) ? config.getInt(counterMapperOverrideTtlPath) : globalCounterExpireTimeInSeconds;
+  }
+
+  /**
+   * @param config The application config
+   * @param mapperKey The key for this mapper
+   * @return The number of reads that must be exceeding while counts are being chained before caching of that object is enabled.
+   */
+  protected int getRepeatReadThreshold(Config config, String mapperKey) {
+    int globalRepeatReadThreshold = config.hasPath(GLOBAL_REPEAT_READ_THRESHOLD) ? config.getInt(GLOBAL_REPEAT_READ_THRESHOLD) : DEFAULT_REPEAT_READ_THRESHOLD;
+    String repeatReadThresholdOverridePath = String.format(REPEAT_READ_THRESHOLD_OVERRIDE_PATH_TEMPLATE, mapperKey);
+    return config.hasPath(repeatReadThresholdOverridePath) ? config.getInt(repeatReadThresholdOverridePath) : globalRepeatReadThreshold;
+  }
+
+  @Override
+  public String getId() {
+    return id;
+  }
+
+  @Override
+  public void putObject(Object key, Object value) {
+    if (key == null || value == null) {
+      return;
+    }
+
+    // If the read counter exists and is greater than the threshold then we are receiving
+    // burst repeat reads and we will cache that entry.
+    ofNullable(autoExpiringRepeatReadCounterMap.getIfPresent(key)).ifPresent(counter -> {
+      if (counter.getCount() > repeatReadThreshold) {
+        dataCache.put(key, value);
+      }
+    });
+  }
+
+  @Override
+  public Object getObject(Object key) {
+    // Increment the read counter, which resets after counterExpireTimeInSeconds.
+    Counter counter = autoExpiringRepeatReadCounterMap.getIfPresent(key);
+    if (counter != null) {
+      counter.inc();
+    } else {
+      counter = new Counter();
+      counter.inc();
+      autoExpiringRepeatReadCounterMap.put(key, counter);
+    }
+
+    return dataCache.getIfPresent(key);
+  }
+
+  @Override
+  public Object removeObject(Object key) {
+    Object res = dataCache.getIfPresent(key);
+    dataCache.invalidate(key);
+    return res;
+  }
+
+  @Override
+  public void clear() {
+    // NO-OP, my batis by default clears the entire namespaced cache when a write action occurs,
+    // we do not want that here, we are expiring the cache / making reads eventually consistent.
+    // Since we run Cerberus in a cluster anyways and each instance will have it's own generated cache, a simple small
+    // time window where items purge themselves is adequate.
+  }
+
+  @Override
+  public int getSize() {
+    try {
+      return Math.toIntExact(dataCache.estimatedSize());
+    } catch (ArithmeticException e) {
+      return Integer.MAX_VALUE;
+    }
+  }
+
+  @Override
+  public ReadWriteLock getReadWriteLock() {
+    return null;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(dataCache, getId());
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (!(o instanceof DatabaseCache)) return false;
+    DatabaseCache that = (DatabaseCache) o;
+    return dataCache.equals(that.dataCache) &&
+        getId().equals(that.getId());
+  }
+}

src/main/java/com/nike/cerberus/cache/MetricReportingCache.java

@@ -45,7 +45,7 @@ public class MetricReportingCache<K, V> implements Cache<K, V> {
     private final Counter hitCounter;
     private final Counter missCounter;
 
-    public MetricReportingCache(String namespace,int expireTimeInSeconds, MetricsService metricsService,
+    public MetricReportingCache(String namespace, int expireTimeInSeconds, MetricsService metricsService,
                                 Map<String, String> dimensions) {
         log.info("Cerberus cache with namespace: {} has been initialized with ttl: {}", namespace, expireTimeInSeconds);
 
@@ -141,4 +141,12 @@ public void invalidateAll(Iterable keys) {
     public @NonNull Map getAllPresent(Iterable keys) {
         return delegate.getAllPresent(keys);
     }
-}
+
+    public Counter getHitCounter() {
+        return hitCounter;
+    }
+
+    public Counter getMissCounter() {
+        return missCounter;
+    }
+}

src/main/java/com/nike/cerberus/server/config/guice/CmsGuiceModule.java

@@ -22,7 +22,6 @@
 import com.amazonaws.encryptionsdk.MasterKeyProvider;
 import com.amazonaws.encryptionsdk.caching.CachingCryptoMaterialsManager;
 import com.amazonaws.encryptionsdk.caching.CryptoMaterialsCache;
-import com.amazonaws.encryptionsdk.caching.LocalCryptoMaterialsCache;
 import com.amazonaws.encryptionsdk.kms.KmsMasterKey;
 import com.amazonaws.regions.Region;
 import com.amazonaws.regions.Regions;
@@ -53,6 +52,7 @@
 import com.nike.riposte.util.AwsUtil;
 import com.okta.authn.sdk.client.AuthenticationClient;
 import com.okta.authn.sdk.client.AuthenticationClients;
+import com.typesafe.config.Config;
 import io.netty.handler.ssl.SslContext;
 import io.netty.handler.ssl.SslContextBuilder;
 import io.netty.handler.ssl.util.SelfSignedCertificate;
@@ -79,7 +79,6 @@
 import java.util.stream.Collectors;
 
 import static com.nike.cerberus.service.EncryptionService.*;
-import static com.github.benmanes.caffeine.cache.Caffeine.newBuilder;
 
 public class CmsGuiceModule extends AbstractModule {
 
@@ -101,6 +100,7 @@ public CmsGuiceModule(ObjectMapper objectMapper) {
 
     @Override
     protected void configure() {
+        requestStaticInjection(StaticInjector.class);
         bind(ObjectMapper.class).toInstance(objectMapper);
         bind(ConfigService.class).toInstance(configService);
 
@@ -145,6 +145,12 @@ public List<ServerShutdownHook> shutdownHooks(Injector injector) {
         return shutdownHooks;
     }
 
+    @Provides
+    @Singleton
+    public Config config() {
+        return configService.getAppConfigMergedWithCliGeneratedProperties();
+    }
+
     @Provides
     @Singleton
     @Named("appEndpoints")

src/main/java/com/nike/cerberus/server/config/guice/CmsMyBatisModule.java

@@ -16,6 +16,8 @@
 
 package com.nike.cerberus.server.config.guice;
 
+import com.nike.cerberus.service.ConfigService;
+import com.typesafe.config.Config;
 import org.apache.ibatis.transaction.jdbc.JdbcTransactionFactory;
 import org.mybatis.guice.MyBatisModule;
 import org.mybatis.guice.datasource.c3p0.C3p0DataSourceProvider;
@@ -25,14 +27,16 @@
  */
 public class CmsMyBatisModule extends MyBatisModule {
 
+    private final Config config = ConfigService.getInstance().getAppConfigMergedWithCliGeneratedProperties();
+
     @Override
     protected void initialize() {
         bindDataSourceProviderType(C3p0DataSourceProvider.class);
         bindTransactionFactoryType(JdbcTransactionFactory.class);
 
         addSimpleAliases("com.nike.cerberus.record");
         addMapperClasses("com.nike.cerberus.mapper");
-        useCacheEnabled(false);
+        useCacheEnabled(config.getBoolean("cms.mybatis.cache.enabled"));
         failFast(true);
     }
 }

src/main/java/com/nike/cerberus/server/config/guice/StaticInjector.java

@@ -0,0 +1,16 @@
+package com.nike.cerberus.server.config.guice;
+
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+
+/**
+ * This is needed for Classes created outside our normal process that can be created with Guice such as MyBatis caches.
+ */
+public class StaticInjector {
+
+  @Inject static Injector injector;
+
+  public static Injector getInstance() {
+    return injector;
+  }
+}

src/main/resources/cms-local.conf

@@ -19,4 +19,4 @@ cms.jobs.enabled=false
 # Disable tls for local dev
 endpoints.useSsl=false
 
-include "cms-local-overrides.conf"
+include "cms-local-overrides.conf"

src/main/resources/cms.conf

@@ -197,5 +197,66 @@ cms.iam.token.ttl=1h
 #   When false, if an SDB grants access to AD group 'Lst-foo', then users in group 'Lst-Foo' will have access
 cms.user.groups.caseSensitive=true
 
+
+####################################################################################################################################
+#
+# CACHE SETTINGS,
+#
+# By default all caching is disabled, Enabling caching will allow this app to scale but comes with trade offs.
+# Potential Trade offs:
+#   - Eventualy consistency of data written
+#   - Potatentally exposesing your encrypted secrets and the key to decrypt them
+#     to meltdown / spectre style attacks: https://meltdownattack.com/
+#
+# You can use the Cerberus Lifecycle CLI `update-cms-config` modify these settings and `reboot-cms` command to apply the settings.
+#
+####################################################################################################################################
+#
+# Data Key Caching for the SDK Crypto Client
+# Why you would want to enable this:
+# KMS has an account wide API limit for KMS of 1200 RPS.
+# Cerberus uses KMS to encrypt and decrypt its data, and by default will make an API call to KMS at least 1 per request.
+# Enabling this cache will allow CMS to to skip calls to KMS for repeat reads of the same data.
+#
 cms.encryption.cache.enabled=false
+// Below are the encryption cache settings for when tcms.encryption.cache.enabled is set to true;
+//cms.encryption.cache.encrypt.maxSize=1000
+//cms.encryption.cache.encrypt.maxAgeInSecods=60
+//cms.encryption.cache.encrypt.messageUseLimit=100
+//cms.encryption.cache.decrypt.maxSize=1000
+//cms.encryption.cache.decrypt.maxAgeInSecods=60
+####################################################################################################################################
+#
+# Enable caching the gernerated encrypted AWS IAM KMS Auth object, so that burst auths of the same principal do not result in repeat KMS encrypt calls
+# Why you would want to enable this:
+# KMS has an account wide API limit for KMS of 1200 RPS.
+# Cerberus uses KMS to encrypt the auth payload, and by default will make an API call to KMS at least 1 per request.
+# Enabling this cache will allow CMS to to skip calls to KMS for repeat authenicates of the same IAM principal.
+#
 cms.auth.iam.kms.cache.enabled=false
+####################################################################################################################################
+#
+# MyBaits Caching
+# CMS uses mybatis as its ORM for the mysql data store.
+# Enabling this cache will cause CMS to cache certain repeat reads that exceed a configurable threshold from various mappers (Some mappers excluded, Lock mapper for example) for some TTL.
+# This will protect the DB from high bursts of repeat reads at the cost of potentially making some of its data eventually consistent.
+# Ex: if a secret / property has been cached it will remain there for upto the whole TTL after a new value has been written.
+# This cache is not distributed and will be in mem on each instance of CMS.
+#
+# The way the repeat read threshold works is as follows
+#
+# Assume the default settings of dataTtlInSeconds=10, repeatReadCounterResetInSeconds=2, repeatReadThreshold=2
+# Everytime a read occurs on a individual instance of CMS a counter is incremented.
+# Everytime a fetch from the DB occurs MyBaitis calls put on the cache object.
+# When the put call happens we check to see if the count has exceeded the repeatReadThreshold threshold, if so cache it for the dataTtlInSeconds ttl, if not don't cache.
+# If there hasn't been any reads in repeatReadCounterResetInSeconds secons the counter is reset.
+#
+cms.mybatis.cache.enabled=false
+//cms.mybatis.cache.global.dataTtlInSeconds=10 // <- Override the global TTL for items after write here, defaults to 10.
+//cms.mybatis.cache.global.repeatReadCounterResetInSeconds=2 // Override the the global Repeat Read counter reset interval here
+//cms.mybatis.cache.global.repeatReadThreshold=2 // Override the global repeat read count threshold here
+// All of the global settings can be overridden for any mapper in resources/com/nike/cerberus/mapper.
+//cms.mybatis.cache.categoryMapper.dataTtlInSeconds=10
+//cms.mybatis.cache.categoryMapper.repeatReadCounterResetInSeconds=2
+//cms.mybatis.cache.categoryMapper.repeatReadThreshold=2
+####################################################################################################################################