SOLR-17587: (9x backport) wt=prometheus fix duplicate TYPE information

solr/CHANGES.txt

@@ -27,7 +27,8 @@ Optimizations
 
 Bug Fixes
 ---------------------
-* PR#2680: Improve reliablity of NpmTasks finding needed files/commands.  (Tyler Bertrand via Eric Pugh)
+* SOLR-17587: Metrics: wt=prometheus fix for non-compliant exposition format containing duplicate TYPE lines.
+  (Matthew Biscocho via David Smiley)
 
 Dependency Upgrades
 ---------------------
@@ -41,6 +42,8 @@ Other Changes
 * GITHUB#2869: SolrTestCase now supports @LogLevel annotations (as SolrTestCaseJ4 has).  Added LogLevelTestRule
   for encapsulation and reuse. (David Smiley)
 
+* GITHUB#2680: Improve reliablity of NpmTasks finding needed files/commands.  (Tyler Bertrand via Eric Pugh)
+
 ==================  9.8.0 ==================
 New Features
 ---------------------

solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java

@@ -186,10 +186,20 @@ private NamedList<Object> handlePrometheusExport(SolrParams params) {
     List<MetricType> metricTypes = parseMetricTypes(params);
     List<MetricFilter> metricFilters =
         metricTypes.stream().map(MetricType::asMetricFilter).collect(Collectors.toList());
+
     Set<String> requestedRegistries = parseRegistries(params);
+    MetricRegistry mergedCoreRegistries = new MetricRegistry();
 
     for (String registryName : requestedRegistries) {
       MetricRegistry dropwizardRegistry = metricManager.registry(registryName);
+
+      // Merge all core registries into a single registry and
+      // append the core name to the metric to avoid duplicate metrics name
+      if (registryName.startsWith("solr.core")) {
+        mergedCoreRegistries.registerAll(getCoreNameFromRegistry(registryName), dropwizardRegistry);
+        continue;
+      }
+
       PrometheusResponseWriter.toPrometheus(
           dropwizardRegistry,
           registryName,
@@ -203,6 +213,22 @@ private NamedList<Object> handlePrometheusExport(SolrParams params) {
             response.add(registryName, registry);
           });
     }
+
+    if (!mergedCoreRegistries.getMetrics().isEmpty()) {
+      PrometheusResponseWriter.toPrometheus(
+          mergedCoreRegistries,
+          "solr.core",
+          metricFilters,
+          mustMatchFilter,
+          propertyFilter,
+          false,
+          false,
+          true,
+          (registry) -> {
+            response.add("solr.core", registry);
+          });
+    }
+
     return response;
   }
 
@@ -523,6 +549,11 @@ private List<MetricType> parseMetricTypes(SolrParams params) {
     return metricTypes;
   }
 
+  private String getCoreNameFromRegistry(String registryName) {
+    String coreName = registryName.substring(registryName.indexOf('.') + 1);
+    return coreName.replace(".", "_");
+  }
+
   @Override
   public String getDescription() {
     return "A handler to return all the metrics gathered by Solr";

solr/core/src/java/org/apache/solr/metrics/prometheus/core/PrometheusCoreFormatterInfo.java

@@ -16,8 +16,6 @@
  */
 package org.apache.solr.metrics.prometheus.core;
 
-import java.util.regex.Pattern;
-
 public interface PrometheusCoreFormatterInfo {
   /** Category of prefix Solr Core dropwizard handler metric names */
   enum CoreCategory {
@@ -32,6 +30,4 @@ enum CoreCategory {
     INDEX,
     CORE
   }
-
-  Pattern CLOUD_CORE_PATTERN = Pattern.compile("^core_(.*)_(shard[0-9]+)_(replica_.[0-9]+)$");
 }

solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreCacheMetric.java

@@ -24,9 +24,8 @@
 public class SolrCoreCacheMetric extends SolrCoreMetric {
   public static final String CORE_CACHE_SEARCHER_METRICS = "solr_metrics_core_cache";
 
-  public SolrCoreCacheMetric(
-      Metric dropwizardMetric, String coreName, String metricName, boolean cloudMode) {
-    super(dropwizardMetric, coreName, metricName, cloudMode);
+  public SolrCoreCacheMetric(Metric dropwizardMetric, String metricName) {
+    super(dropwizardMetric, metricName);
   }
 
   /*

solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreHandlerMetric.java

@@ -30,9 +30,8 @@ public class SolrCoreHandlerMetric extends SolrCoreMetric {
   public static final String CORE_REQUESTS_TOTAL_TIME = "solr_metrics_core_requests_time";
   public static final String CORE_REQUEST_TIMES = "solr_metrics_core_average_request_time";
 
-  public SolrCoreHandlerMetric(
-      Metric dropwizardMetric, String coreName, String metricName, boolean cloudMode) {
-    super(dropwizardMetric, coreName, metricName, cloudMode);
+  public SolrCoreHandlerMetric(Metric dropwizardMetric, String metricName) {
+    super(dropwizardMetric, metricName);
   }
 
   /*

solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreHighlighterMetric.java

@@ -24,9 +24,8 @@
 public class SolrCoreHighlighterMetric extends SolrCoreMetric {
   public static final String CORE_HIGHLIGHER_METRICS = "solr_metrics_core_highlighter_requests";
 
-  public SolrCoreHighlighterMetric(
-      Metric dropwizardMetric, String coreName, String metricName, boolean cloudMode) {
-    super(dropwizardMetric, coreName, metricName, cloudMode);
+  public SolrCoreHighlighterMetric(Metric dropwizardMetric, String metricName) {
+    super(dropwizardMetric, metricName);
   }
 
   /*

solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreIndexMetric.java

@@ -24,9 +24,8 @@
 public class SolrCoreIndexMetric extends SolrCoreMetric {
   public static final String CORE_INDEX_METRICS = "solr_metrics_core_index_size_bytes";
 
-  public SolrCoreIndexMetric(
-      Metric dropwizardMetric, String coreName, String metricName, boolean cloudMode) {
-    super(dropwizardMetric, coreName, metricName, cloudMode);
+  public SolrCoreIndexMetric(Metric dropwizardMetric, String metricName) {
+    super(dropwizardMetric, metricName);
   }
 
   /*

solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreMetric.java

@@ -16,37 +16,45 @@
  */
 package org.apache.solr.metrics.prometheus.core;
 
-import static org.apache.solr.metrics.prometheus.core.PrometheusCoreFormatterInfo.CLOUD_CORE_PATTERN;
-
 import com.codahale.metrics.Metric;
+import java.util.List;
 import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.metrics.prometheus.SolrMetric;
 
 /** Base class is a wrapper to export a solr.core {@link com.codahale.metrics.Metric} */
 public abstract class SolrCoreMetric extends SolrMetric {
-  public String coreName;
 
-  public SolrCoreMetric(
-      Metric dropwizardMetric, String coreName, String metricName, boolean cloudMode) {
-    super(dropwizardMetric, metricName);
-    this.coreName = coreName;
-    labels.put("core", coreName);
-    if (cloudMode) {
-      appendCloudModeLabels();
-    }
-  }
+  public static Pattern CLOUD_CORE_PATTERN =
+      Pattern.compile(
+          "(?<core>^core_(?<collection>.*)_(?<shard>shard[0-9]+)_(?<replica>replica_.[0-9]+)).(.*)$");
+  public static Pattern STANDALONE_CORE_PATTERN = Pattern.compile("^core_(?<core>.*?)\\.(.*)$");
+  public static List<String> CLOUD_LABEL_KEYS = List.of("core", "collection", "shard", "replica");
+  public static List<String> STANDALONE_LABEL_KEYS = List.of("core");
+  public String coreName;
 
-  private void appendCloudModeLabels() {
-    Matcher m = CLOUD_CORE_PATTERN.matcher(coreName);
-    if (m.find()) {
-      labels.put("collection", m.group(1));
-      labels.put("shard", m.group(2));
-      labels.put("replica", m.group(3));
+  public SolrCoreMetric(Metric dropwizardMetric, String metricName) {
+    // Remove Core Name prefix from metric as it is no longer needed for tag parsing from this point
+    super(dropwizardMetric, metricName.substring(metricName.indexOf(".") + 1));
+    Matcher cloudPattern = CLOUD_CORE_PATTERN.matcher(metricName);
+    Matcher standalonePattern = STANDALONE_CORE_PATTERN.matcher(metricName);
+    if (cloudPattern.find()) {
+      coreName = cloudPattern.group("core");
+      CLOUD_LABEL_KEYS.forEach(
+          (key) -> {
+            labels.put(key, cloudPattern.group(key));
+          });
+    } else if (standalonePattern.find()) {
+      coreName = standalonePattern.group("core");
+      STANDALONE_LABEL_KEYS.forEach(
+          (key) -> {
+            labels.put(key, standalonePattern.group(key));
+          });
     } else {
       throw new SolrException(
           SolrException.ErrorCode.SERVER_ERROR,
-          "Core name does not match pattern for parsing " + coreName);
+          "Core name does not match pattern for parsing in metric " + metricName);
     }
   }
 }

solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreSearcherMetric.java

@@ -29,9 +29,8 @@ public class SolrCoreSearcherMetric extends SolrCoreMetric {
   public static final String CORE_SEARCHER_METRICS = "solr_metrics_core_searcher_documents";
   public static final String CORE_SEARCHER_TIMES = "solr_metrics_core_average_searcher_warmup_time";
 
-  public SolrCoreSearcherMetric(
-      Metric dropwizardMetric, String coreName, String metricName, boolean cloudMode) {
-    super(dropwizardMetric, coreName, metricName, cloudMode);
+  public SolrCoreSearcherMetric(Metric dropwizardMetric, String metricName) {
+    super(dropwizardMetric, metricName);
   }
 
   /*

solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreTlogMetric.java

@@ -24,9 +24,8 @@
 public class SolrCoreTlogMetric extends SolrCoreMetric {
   public static final String CORE_TLOG_METRICS = "solr_metrics_core_tlog";
 
-  public SolrCoreTlogMetric(
-      Metric dropwizardMetric, String coreName, String metricName, boolean cloudMode) {
-    super(dropwizardMetric, coreName, metricName, cloudMode);
+  public SolrCoreTlogMetric(Metric dropwizardMetric, String metricName) {
+    super(dropwizardMetric, metricName);
   }
 
   /*

solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrPrometheusCoreFormatter.java

@@ -28,13 +28,9 @@
  */
 public class SolrPrometheusCoreFormatter extends SolrPrometheusFormatter
     implements PrometheusCoreFormatterInfo {
-  public final String coreName;
-  public final boolean cloudMode;
 
-  public SolrPrometheusCoreFormatter(String coreName, boolean cloudMode) {
+  public SolrPrometheusCoreFormatter() {
     super();
-    this.coreName = coreName;
-    this.cloudMode = cloudMode;
   }
 
   @Override
@@ -45,7 +41,7 @@ public void exportDropwizardMetric(Metric dropwizardMetric, String metricName) {
 
   @Override
   public SolrMetric categorizeMetric(Metric dropwizardMetric, String metricName) {
-    String metricCategory = metricName.split("\\.", 2)[0];
+    String metricCategory = metricName.split("\\.", 3)[1];
     if (!Enums.getIfPresent(PrometheusCoreFormatterInfo.CoreCategory.class, metricCategory)
         .isPresent()) {
       return new SolrNoOpMetric();
@@ -55,17 +51,17 @@ public SolrMetric categorizeMetric(Metric dropwizardMetric, String metricName) {
       case QUERY:
       case UPDATE:
       case REPLICATION:
-        return new SolrCoreHandlerMetric(dropwizardMetric, coreName, metricName, cloudMode);
+        return new SolrCoreHandlerMetric(dropwizardMetric, metricName);
       case TLOG:
-        return new SolrCoreTlogMetric(dropwizardMetric, coreName, metricName, cloudMode);
+        return new SolrCoreTlogMetric(dropwizardMetric, metricName);
       case CACHE:
-        return new SolrCoreCacheMetric(dropwizardMetric, coreName, metricName, cloudMode);
+        return new SolrCoreCacheMetric(dropwizardMetric, metricName);
       case SEARCHER:
-        return new SolrCoreSearcherMetric(dropwizardMetric, coreName, metricName, cloudMode);
+        return new SolrCoreSearcherMetric(dropwizardMetric, metricName);
       case HIGHLIGHTER:
-        return new SolrCoreHighlighterMetric(dropwizardMetric, coreName, metricName, cloudMode);
+        return new SolrCoreHighlighterMetric(dropwizardMetric, metricName);
       case INDEX:
-        return new SolrCoreIndexMetric(dropwizardMetric, coreName, metricName, cloudMode);
+        return new SolrCoreIndexMetric(dropwizardMetric, metricName);
       case CORE:
       default:
         return new SolrNoOpMetric();

solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java

@@ -25,12 +25,11 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.lang.invoke.MethodHandles;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
-import java.util.stream.Collectors;
+import org.apache.solr.common.SolrException;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.metrics.AggregateMetric;
 import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter;
@@ -115,30 +114,22 @@ public static void toPrometheus(
             Metric dropwizardMetric = dropwizardMetrics.get(metricName);
             formatter.exportDropwizardMetric(dropwizardMetric, metricName);
           } catch (Exception e) {
-            // Do not fail entirely for metrics exporting. Log and try to export next metric
-            log.warn("Error occurred exporting Dropwizard Metric to Prometheus", e);
+            throw new SolrException(
+                SolrException.ErrorCode.SERVER_ERROR,
+                "Error occurred exporting Dropwizard Metric to Prometheus",
+                e);
           }
         });
 
     consumer.accept(formatter);
   }
 
   public static SolrPrometheusFormatter getFormatterType(String registryName) {
-    String coreName;
-    boolean cloudMode = false;
     String[] parsedRegistry = registryName.split("\\.");
 
     switch (parsedRegistry[1]) {
       case "core":
-        if (parsedRegistry.length == 3) {
-          coreName = parsedRegistry[2];
-        } else if (parsedRegistry.length == 5) {
-          coreName = Arrays.stream(parsedRegistry).skip(1).collect(Collectors.joining("_"));
-          cloudMode = true;
-        } else {
-          coreName = registryName;
-        }
-        return new SolrPrometheusCoreFormatter(coreName, cloudMode);
+        return new SolrPrometheusCoreFormatter();
       case "jvm":
         return new SolrPrometheusJvmFormatter();
       case "jetty":

solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java

@@ -725,8 +725,7 @@ public void testPrometheusMetricsCore() throws Exception {
     NamedList<?> values = resp.getValues();
     assertNotNull(values.get("metrics"));
     values = (NamedList<?>) values.get("metrics");
-    SolrPrometheusFormatter formatter =
-        (SolrPrometheusFormatter) values.get("solr.core.collection1");
+    SolrPrometheusFormatter formatter = (SolrPrometheusFormatter) values.get("solr.core");
     assertNotNull(formatter);
     MetricSnapshots actualSnapshots = formatter.collect();
     assertNotNull(actualSnapshots);
@@ -1009,8 +1008,7 @@ public void testPrometheusMetricsFilter() throws Exception {
     NamedList<?> values = resp.getValues();
     assertNotNull(values.get("metrics"));
     values = (NamedList<?>) values.get("metrics");
-    SolrPrometheusFormatter formatter =
-        (SolrPrometheusFormatter) values.get("solr.core.collection1");
+    SolrPrometheusFormatter formatter = (SolrPrometheusFormatter) values.get("solr.core");
     assertNotNull(formatter);
     MetricSnapshots actualSnapshots = formatter.collect();
     assertNotNull(actualSnapshots);