diff --git a/cerberus-web/build.gradle b/cerberus-web/build.gradle index 085410154..149c7f6d0 100644 --- a/cerberus-web/build.gradle +++ b/cerberus-web/build.gradle @@ -55,7 +55,6 @@ dependencies { implementation "io.spinnaker.kork:kork-secrets:${versions.kork}" // Misc - implementation "com.netflix.hystrix:hystrix-core:1.5.18" // TODO remove hystrix and use resilience4j implementation "com.google.guava:guava:${versions.guava}" // todo, pretty sure with Java 11, we can get ride of this, we mostly use this for the collection builders. implementation 'com.google.code.gson:gson:2.9.1' // todo delete, should use jackson. see cipher text utils diff --git a/cerberus-web/src/main/java/com/nike/cerberus/jobs/HystrixMetricsProcessingJob.java b/cerberus-web/src/main/java/com/nike/cerberus/jobs/HystrixMetricsProcessingJob.java deleted file mode 100644 index ba5b4f7fd..000000000 --- a/cerberus-web/src/main/java/com/nike/cerberus/jobs/HystrixMetricsProcessingJob.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright (c) 2020 Nike, inc. - * - * Licensed under the Apache License, Version 2.0 (the "License") - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.nike.cerberus.jobs; - -import com.google.common.collect.ImmutableMap; -import com.netflix.hystrix.HystrixCircuitBreaker; -import com.netflix.hystrix.HystrixCommandKey; -import com.netflix.hystrix.HystrixCommandMetrics; -import com.netflix.hystrix.HystrixThreadPoolMetrics; -import com.nike.cerberus.metric.MetricsService; -import java.util.Collection; -import java.util.Map; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; - -/** Periodically print Hystrix metrics to the log. */ -@Slf4j -@ConditionalOnProperty("cerberus.jobs.hystrixMetricsProcessingJob.enabled") -@Component -public class HystrixMetricsProcessingJob { - private final MetricsService metricsService; - - @Autowired - public HystrixMetricsProcessingJob(MetricsService metricsService) { - this.metricsService = metricsService; - } - - @Scheduled(cron = "${cerberus.jobs.hystrixMetricsProcessingJob.cronExpression}") - public void execute() { - log.debug("Running hystrix metrics processing job"); - try { - processHystrixCommandMetrics(); - processHystrixThreadPoolMetrics(); - } catch (Exception e) { - log.warn("Error processing Hystrix metrics", e); - } - } - - public void processHystrixCommandMetrics() { - for (HystrixCommandMetrics metrics : getHystrixCommandMetrics()) { - Map dimensions = - ImmutableMap.of( - "key", metrics.getCommandKey().name(), - "group", metrics.getCommandGroup().name()); - boolean isCircuitOpen = getHystrixCircuitBreaker(metrics.getCommandKey()).isOpen(); - - log.debug( - "group:{}, commandKey:{}, CircuitOpen:{}, Mean:{}, 95%:{}, 99%:{}, 99.5%:{}, {}", - metrics.getCommandGroup().name(), - metrics.getCommandKey().name(), - isCircuitOpen, - metrics.getExecutionTimeMean(), - metrics.getExecutionTimePercentile(95.0), - metrics.getExecutionTimePercentile(99.0), - metrics.getExecutionTimePercentile(99.5), - metrics.getHealthCounts()); - - metricsService.getOrCreateCallbackGauge( - "hystrix.command.circuit_open", () -> isCircuitOpen ? 1 : 0, dimensions); - metricsService.getOrCreateCallbackGauge( - "hystrix.command.exec_time.mean", metrics::getExecutionTimeMean, dimensions); - metricsService.getOrCreateCallbackGauge( - "hystrix.command.exec_time.95th", - () -> metrics.getExecutionTimePercentile(95.0), - dimensions); - metricsService.getOrCreateCallbackGauge( - "hystrix.command.exec_time.99th", - () -> metrics.getExecutionTimePercentile(99.0), - dimensions); - metricsService.getOrCreateCallbackGauge( - "hystrix.command.rolling.max_concurrent_execs", - metrics::getRollingMaxConcurrentExecutions, - dimensions); - metricsService.getOrCreateCallbackGauge( - "hystrix.command.total_count", - () -> metrics.getHealthCounts().getTotalRequests(), - dimensions); - metricsService.getOrCreateCallbackGauge( - "hystrix.command.error_count", - () -> metrics.getHealthCounts().getErrorCount(), - dimensions); - } - } - - public void processHystrixThreadPoolMetrics() { - for (HystrixThreadPoolMetrics metrics : getHystrixThreadPoolMetrics()) { - Map dimensions = ImmutableMap.of("name", metrics.getThreadPoolKey().name()); - log.debug( - "threadPool:{}, rollingCounts[rejected:{}, executed:{}, maxActiveThreads:{}], cumulativeCounts[rejected:{}, executed:{}], {}", - metrics.getThreadPoolKey().name(), - metrics.getRollingCountThreadsRejected(), - metrics.getRollingCountThreadsExecuted(), - metrics.getRollingMaxActiveThreads(), - metrics.getCumulativeCountThreadsRejected(), - metrics.getCumulativeCountThreadsExecuted(), - metrics.getThreadPool()); - - metricsService.getOrCreateCallbackGauge( - "hystrix.threads.rolling.rejected", metrics::getRollingCountThreadsRejected, dimensions); - metricsService.getOrCreateCallbackGauge( - "hystrix.threads.rolling.executed", metrics::getRollingCountThreadsExecuted, dimensions); - metricsService.getOrCreateCallbackGauge( - "hystrix.threads.rolling.maxActiveThreads", - metrics::getRollingMaxActiveThreads, - dimensions); - metricsService.getOrCreateCallbackGauge( - "hystrix.threads.cumulative.rejected", - metrics::getCumulativeCountThreadsRejected, - dimensions); - metricsService.getOrCreateCallbackGauge( - "hystrix.threads.cumulative.executed", - metrics::getCumulativeCountThreadsExecuted, - dimensions); - metricsService.getOrCreateCallbackGauge( - "hystrix.threads.activeThreads", - () -> metrics.getThreadPool().getActiveCount(), - dimensions); - metricsService.getOrCreateCallbackGauge( - "hystrix.threads.queuedTasks", - () -> metrics.getThreadPool().getQueue().size(), - dimensions); - metricsService.getOrCreateCallbackGauge( - "hystrix.threads.completedTasks", - () -> metrics.getThreadPool().getCompletedTaskCount(), - dimensions); - } - } - - Collection getHystrixCommandMetrics() { - return HystrixCommandMetrics.getInstances(); - } - - Collection getHystrixThreadPoolMetrics() { - return HystrixThreadPoolMetrics.getInstances(); - } - - HystrixCircuitBreaker getHystrixCircuitBreaker(HystrixCommandKey hystrixCommandKey) { - return HystrixCircuitBreaker.Factory.getInstance(hystrixCommandKey); - } -} diff --git a/cerberus-web/src/main/java/com/nike/cerberus/jobs/OrphanedKmsKeyCleanUpJob.java b/cerberus-web/src/main/java/com/nike/cerberus/jobs/OrphanedKmsKeyCleanUpJob.java index cd3bc2e0c..ad5baff5b 100644 --- a/cerberus-web/src/main/java/com/nike/cerberus/jobs/OrphanedKmsKeyCleanUpJob.java +++ b/cerberus-web/src/main/java/com/nike/cerberus/jobs/OrphanedKmsKeyCleanUpJob.java @@ -25,7 +25,7 @@ import java.util.*; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; diff --git a/cerberus-web/src/main/resources/cerberus.yaml b/cerberus-web/src/main/resources/cerberus.yaml index 22feed4a9..c2a46ce5c 100644 --- a/cerberus-web/src/main/resources/cerberus.yaml +++ b/cerberus-web/src/main/resources/cerberus.yaml @@ -233,11 +233,6 @@ cerberus: # Every ten minutes cronExpression: "0 */10 * ? * *" - # Generates drop wizard metrics on the status of Hystrix for monitoring and alerting - hystrixMetricsProcessingJob: - enabled: true - # Every ten seconds - cronExpression: "0/10 * * ? * *" # This job will generate Key Performance Indicator Metrics (KPIs) for the Cerberus environment and use Drop Wizard to report these Metrics # By default unless you provide your own reporter impl, these metrics are just logged via an SLF4j Logger. diff --git a/cerberus-web/src/test/java/com/nike/cerberus/jobs/HystrixMetricsProcessingJobTest.java b/cerberus-web/src/test/java/com/nike/cerberus/jobs/HystrixMetricsProcessingJobTest.java deleted file mode 100644 index 0134f14cc..000000000 --- a/cerberus-web/src/test/java/com/nike/cerberus/jobs/HystrixMetricsProcessingJobTest.java +++ /dev/null @@ -1,229 +0,0 @@ -package com.nike.cerberus.jobs; - -import com.google.common.collect.ImmutableMap; -import com.netflix.hystrix.*; -import com.nike.cerberus.metric.MetricsService; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.function.Supplier; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; - -public class HystrixMetricsProcessingJobTest { - - @Mock private MetricsService metricsService; - - @InjectMocks private HystrixMetricsProcessingJob hystrixMetricsProcessingJob; - - @Captor private ArgumentCaptor> supplierArgumentCaptor; - - @Captor private ArgumentCaptor> mapArgumentCaptor; - - @Before - public void setup() { - MockitoAnnotations.initMocks(this); - } - - @Test - @SuppressFBWarnings - public void testExecute() { - HystrixMetricsProcessingJob hystrixMetricsProcessingJobSpy = - Mockito.spy(hystrixMetricsProcessingJob); - List hystrixCommandMetricsList = getHystrixCommandMetricsList(); - Mockito.doReturn(hystrixCommandMetricsList) - .when(hystrixMetricsProcessingJobSpy) - .getHystrixCommandMetrics(); - HystrixCircuitBreaker hystrixCircuitBreaker = Mockito.mock(HystrixCircuitBreaker.class); - HystrixCommandKey commandKey = hystrixCommandMetricsList.get(0).getCommandKey(); - Mockito.doReturn(hystrixCircuitBreaker) - .when(hystrixMetricsProcessingJobSpy) - .getHystrixCircuitBreaker(commandKey); - List hystrixThreadPoolMetrics = getHystrixThreadPoolMetrics(); - Mockito.doReturn(hystrixThreadPoolMetrics) - .when(hystrixMetricsProcessingJobSpy) - .getHystrixThreadPoolMetrics(); - hystrixMetricsProcessingJobSpy.execute(); - verifyCommandMetricsOperations(); - verifyThreadPoolOperations(); - } - - private List getHystrixCommandMetricsList() { - List hystrixCommandMetricsList = new ArrayList<>(); - HystrixCommandMetrics hystrixCommandMetrics = Mockito.mock(HystrixCommandMetrics.class); - HystrixCommandKey hystrixCommandKey = Mockito.mock(HystrixCommandKey.class); - Mockito.when(hystrixCommandKey.name()).thenReturn("name"); - Mockito.when(hystrixCommandMetrics.getCommandKey()).thenReturn(hystrixCommandKey); - HystrixCommandGroupKey hystrixCommandGroupKey = Mockito.mock(HystrixCommandGroupKey.class); - Mockito.when(hystrixCommandGroupKey.name()).thenReturn("groupKey"); - Mockito.when(hystrixCommandMetrics.getCommandGroup()).thenReturn(hystrixCommandGroupKey); - HystrixCommandMetrics.HealthCounts healthCounts = HystrixCommandMetrics.HealthCounts.empty(); - Mockito.when(hystrixCommandMetrics.getHealthCounts()).thenReturn(healthCounts); - Mockito.when(hystrixCommandMetrics.getExecutionTimeMean()).thenReturn(100); - Mockito.when(hystrixCommandMetrics.getExecutionTimePercentile(95.0)).thenReturn(95); - Mockito.when(hystrixCommandMetrics.getExecutionTimePercentile(99.0)).thenReturn(99); - Mockito.when(hystrixCommandMetrics.getExecutionTimePercentile(99.5)).thenReturn(100); - Mockito.when(hystrixCommandMetrics.getRollingMaxConcurrentExecutions()).thenReturn(43l); - hystrixCommandMetricsList.add(hystrixCommandMetrics); - return hystrixCommandMetricsList; - } - - private void verifyCommandMetricsOperations() { - Map dimensions = - ImmutableMap.of( - "key", "name", - "group", "groupKey"); - Mockito.verify(metricsService) - .getOrCreateCallbackGauge( - Mockito.eq("hystrix.command.circuit_open"), - supplierArgumentCaptor.capture(), - mapArgumentCaptor.capture()); - Assert.assertEquals(0, supplierArgumentCaptor.getValue().get()); - Assert.assertEquals(dimensions, mapArgumentCaptor.getValue()); - Mockito.verify(metricsService) - .getOrCreateCallbackGauge( - Mockito.eq("hystrix.command.exec_time.mean"), - supplierArgumentCaptor.capture(), - mapArgumentCaptor.capture()); - Assert.assertEquals(100, supplierArgumentCaptor.getValue().get()); - Assert.assertEquals(dimensions, mapArgumentCaptor.getValue()); - Mockito.verify(metricsService) - .getOrCreateCallbackGauge( - Mockito.eq("hystrix.command.exec_time.95th"), - supplierArgumentCaptor.capture(), - mapArgumentCaptor.capture()); - Assert.assertEquals(95, supplierArgumentCaptor.getValue().get()); - Assert.assertEquals(dimensions, mapArgumentCaptor.getValue()); - Mockito.verify(metricsService) - .getOrCreateCallbackGauge( - Mockito.eq("hystrix.command.exec_time.99th"), - supplierArgumentCaptor.capture(), - mapArgumentCaptor.capture()); - Assert.assertEquals(99, supplierArgumentCaptor.getValue().get()); - Assert.assertEquals(dimensions, mapArgumentCaptor.getValue()); - Mockito.verify(metricsService) - .getOrCreateCallbackGauge( - Mockito.eq("hystrix.command.rolling.max_concurrent_execs"), - supplierArgumentCaptor.capture(), - mapArgumentCaptor.capture()); - Assert.assertEquals(43l, supplierArgumentCaptor.getValue().get()); - Assert.assertEquals(dimensions, mapArgumentCaptor.getValue()); - Mockito.verify(metricsService) - .getOrCreateCallbackGauge( - Mockito.eq("hystrix.command.total_count"), - supplierArgumentCaptor.capture(), - mapArgumentCaptor.capture()); - Assert.assertEquals(0l, supplierArgumentCaptor.getValue().get()); - Assert.assertEquals(dimensions, mapArgumentCaptor.getValue()); - Mockito.verify(metricsService) - .getOrCreateCallbackGauge( - Mockito.eq("hystrix.command.error_count"), - supplierArgumentCaptor.capture(), - mapArgumentCaptor.capture()); - Assert.assertEquals(0l, supplierArgumentCaptor.getValue().get()); - Assert.assertEquals(dimensions, mapArgumentCaptor.getValue()); - } - - private List getHystrixThreadPoolMetrics() { - List hystrixCommandMetricsList = new ArrayList<>(); - HystrixThreadPoolMetrics hystrixThreadPoolMetrics = - Mockito.mock(HystrixThreadPoolMetrics.class); - hystrixCommandMetricsList.add(hystrixThreadPoolMetrics); - HystrixThreadPoolKey hystrixThreadPoolKey = Mockito.mock(HystrixThreadPoolKey.class); - Mockito.when(hystrixThreadPoolKey.name()).thenReturn("name"); - Mockito.when(hystrixThreadPoolMetrics.getThreadPoolKey()).thenReturn(hystrixThreadPoolKey); - Mockito.when(hystrixThreadPoolMetrics.getRollingCountThreadsRejected()).thenReturn(100l); - Mockito.when(hystrixThreadPoolMetrics.getRollingCountThreadsExecuted()).thenReturn(200l); - Mockito.when(hystrixThreadPoolMetrics.getRollingMaxActiveThreads()).thenReturn(300l); - Mockito.when(hystrixThreadPoolMetrics.getCumulativeCountThreadsRejected()).thenReturn(400l); - Mockito.when(hystrixThreadPoolMetrics.getCumulativeCountThreadsExecuted()).thenReturn(500l); - ThreadPoolExecutor threadPoolExecutor = Mockito.mock(ThreadPoolExecutor.class); - Mockito.when(threadPoolExecutor.getActiveCount()).thenReturn(30); - Mockito.when(threadPoolExecutor.getCompletedTaskCount()).thenReturn(40l); - BlockingQueue blockingQueue = Mockito.mock(BlockingQueue.class); - Mockito.when(blockingQueue.size()).thenReturn(10); - Mockito.when(threadPoolExecutor.getQueue()).thenReturn(blockingQueue); - Mockito.when(hystrixThreadPoolMetrics.getThreadPool()).thenReturn(threadPoolExecutor); - return hystrixCommandMetricsList; - } - - private void verifyThreadPoolOperations() { - Map dimensions = ImmutableMap.of("name", "name"); - Mockito.verify(metricsService) - .getOrCreateCallbackGauge( - Mockito.eq("hystrix.threads.rolling.rejected"), - supplierArgumentCaptor.capture(), - mapArgumentCaptor.capture()); - Assert.assertEquals(100l, supplierArgumentCaptor.getValue().get()); - Assert.assertEquals(dimensions, mapArgumentCaptor.getValue()); - Mockito.verify(metricsService) - .getOrCreateCallbackGauge( - Mockito.eq("hystrix.threads.rolling.executed"), - supplierArgumentCaptor.capture(), - mapArgumentCaptor.capture()); - Assert.assertEquals(200l, supplierArgumentCaptor.getValue().get()); - Assert.assertEquals(dimensions, mapArgumentCaptor.getValue()); - Mockito.verify(metricsService) - .getOrCreateCallbackGauge( - Mockito.eq("hystrix.threads.rolling.maxActiveThreads"), - supplierArgumentCaptor.capture(), - mapArgumentCaptor.capture()); - Assert.assertEquals(300l, supplierArgumentCaptor.getValue().get()); - Assert.assertEquals(dimensions, mapArgumentCaptor.getValue()); - Mockito.verify(metricsService) - .getOrCreateCallbackGauge( - Mockito.eq("hystrix.threads.cumulative.rejected"), - supplierArgumentCaptor.capture(), - mapArgumentCaptor.capture()); - Assert.assertEquals(400l, supplierArgumentCaptor.getValue().get()); - Assert.assertEquals(dimensions, mapArgumentCaptor.getValue()); - Mockito.verify(metricsService) - .getOrCreateCallbackGauge( - Mockito.eq("hystrix.threads.cumulative.executed"), - supplierArgumentCaptor.capture(), - mapArgumentCaptor.capture()); - Assert.assertEquals(500l, supplierArgumentCaptor.getValue().get()); - Assert.assertEquals(dimensions, mapArgumentCaptor.getValue()); - Mockito.verify(metricsService) - .getOrCreateCallbackGauge( - Mockito.eq("hystrix.threads.activeThreads"), - supplierArgumentCaptor.capture(), - mapArgumentCaptor.capture()); - Assert.assertEquals(30, supplierArgumentCaptor.getValue().get()); - Assert.assertEquals(dimensions, mapArgumentCaptor.getValue()); - Mockito.verify(metricsService) - .getOrCreateCallbackGauge( - Mockito.eq("hystrix.threads.queuedTasks"), - supplierArgumentCaptor.capture(), - mapArgumentCaptor.capture()); - Assert.assertEquals(10, supplierArgumentCaptor.getValue().get()); - Assert.assertEquals(dimensions, mapArgumentCaptor.getValue()); - Mockito.verify(metricsService) - .getOrCreateCallbackGauge( - Mockito.eq("hystrix.threads.completedTasks"), - supplierArgumentCaptor.capture(), - mapArgumentCaptor.capture()); - Assert.assertEquals(40l, supplierArgumentCaptor.getValue().get()); - Assert.assertEquals(dimensions, mapArgumentCaptor.getValue()); - } - - @Test - public void testGetHystrixCommandMetrics() { - Assert.assertTrue(hystrixMetricsProcessingJob.getHystrixCommandMetrics().isEmpty()); - } - - @Test - public void testGetHystrixThreadPoolMetrics() { - Assert.assertTrue(hystrixMetricsProcessingJob.getHystrixThreadPoolMetrics().isEmpty()); - } -}