Merge pull request #661 from scireum/feature/es-memory-pressure-metric

Add a metric for ES memory pressure

src/main/java/sirius/db/es/ElasticMetricsProvider.java

@@ -8,18 +8,25 @@
 
 package sirius.db.es;
 
+import com.fasterxml.jackson.core.JsonPointer;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import sirius.kernel.di.std.Part;
 import sirius.kernel.di.std.Register;
+import sirius.kernel.health.metrics.Metric;
 import sirius.kernel.health.metrics.MetricProvider;
 import sirius.kernel.health.metrics.MetricsCollector;
 
+import java.util.Map;
+
 /**
  * Provides metrics for Elasticsearch (if configured).
  */
-@Register
+@Register(classes = {ElasticMetricsProvider.class, MetricProvider.class})
 public class ElasticMetricsProvider implements MetricProvider {
 
+    private static final JsonPointer OLD_GEN_STATS_POINTER = JsonPointer.compile("/jvm/mem/pools/old");
+
     @Part
     private Elastic elastic;
 
@@ -43,6 +50,44 @@ public void gather(MetricsCollector collector) {
                              "ES Unassigned Shards",
                              health.path("unassigned_shards").asInt(),
                              null);
+            collector.metric("es_memory_pressure",
+                             "es-memory-pressure",
+                             "Elasticsearch Memory Pressure",
+                             getCurrentMaxMemoryPressure(),
+                             Metric.UNIT_PERCENT);
         }
     }
+
+    /**
+     * Determines the current maximum memory pressure of all ES nodes.
+     * <p>
+     * The memory pressure is defined as the usage in percent of the old gen memory pool.
+     *
+     * @return the current maximum memory pressure
+     * @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/high-jvm-memory-pressure.html">
+     * ElasticSearch reference page for JVM memory pressure</a>
+     */
+    public int getCurrentMaxMemoryPressure() {
+        return elastic.getLowLevelClient()
+                      .memoryStats()
+                      .path("nodes")
+                      .properties()
+                      .stream()
+                      .map(Map.Entry::getValue)
+                      .mapToInt(this::calculateMemoryPressure)
+                      .max()
+                      .orElse(0);
+    }
+
+    private int calculateMemoryPressure(JsonNode memoryStats) {
+        JsonNode oldGenStats = memoryStats.at(OLD_GEN_STATS_POINTER);
+        int usedInBytes = oldGenStats.path("used_in_bytes").asInt();
+        int maxInBytes = oldGenStats.path("max_in_bytes").asInt();
+
+        if (maxInBytes == 0) {
+            return 0;
+        }
+
+        return (int) (100f * usedInBytes / maxInBytes);
+    }
 }

src/main/java/sirius/db/es/LowLevelClient.java

@@ -50,6 +50,7 @@ public class LowLevelClient {
     private static final String API_REFRESH = "/_refresh";
     private static final String API_SETTINGS = "/_settings";
     private static final String API_CLUSTER_HEALTH = "/_cluster/health";
+    private static final String API_JVM_MEMORY_STATS = "/_nodes/stats?pretty&filter_path=nodes.*.jvm.mem";
     private static final String API_STATS = "/_stats";
     private static final String API_MAPPING = "/_mapping";
     private static final String API_BULK = "_bulk";
@@ -579,6 +580,15 @@ public ObjectNode clusterHealth() {
         return performGet().execute(API_CLUSTER_HEALTH).response();
     }
 
+    /**
+     * Fetches the JVM memory statistics.
+     *
+     * @return a JSON object as returned by <tt>/_nodes/stats?pretty&filter_path=nodes.*.jvm.mem</tt>
+     */
+    public ObjectNode memoryStats() {
+        return performGet().execute(API_JVM_MEMORY_STATS).response();
+    }
+
     /**
      * Fetches statistics for all indices.
      *

src/main/resources/component-060-db.conf

@@ -112,6 +112,11 @@ health {
         es-unassigned-shards.warning = 0
         es-unassigned-shards.error = 1
 
+        # Warns about old-gen memory pressure in Elasticsearch
+        es-memory-pressure.gray = 50
+        es-memory-pressure.warning = 75
+        es-memory-pressure.error = 85
+
         # Number of calls against qdrant
         qdrant-calls.gray = 10
         qdrant-calls.warning = 0