Merge pull request #162 from nicolastakashi/chore/metric-usage-correl…

…ation

Chore/metric usage correlation

Dockerfile

@@ -9,7 +9,7 @@ COPY ./ui .
 RUN npm install
 RUN npm run build
 
-FROM golang:1.23.0 AS gobuild
+FROM golang:1.23.1 AS gobuild
 
 WORKDIR /go/src/github.com/nicolastakashi/prom-analytics-proxy
 

api/routes/routes.go

@@ -18,6 +18,7 @@ import (
 	"github.com/nicolastakashi/prom-analytics-proxy/api/response"
 	"github.com/nicolastakashi/prom-analytics-proxy/internal/db"
 	"github.com/nicolastakashi/prom-analytics-proxy/internal/ingester"
+	metricsUsageV1 "github.com/perses/metrics-usage/pkg/api/v1"
 	"github.com/prometheus/client_golang/api"
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/prometheus/client_golang/prometheus"
@@ -69,6 +70,10 @@ func WithHandlers(uiFS fs.FS, registry *prometheus.Registry, isTracingEnabled bo
 		mux.Handle("/api/v1/seriesMetadata", http.HandlerFunc(r.seriesMetadata))
 		mux.Handle("/api/v1/serieMetadata/{name}", http.HandlerFunc(r.serieMetadata))
 		mux.Handle("/api/v1/serieExpressions/{name}", http.HandlerFunc(r.serieExpressions))
+		mux.Handle("/api/v1/serieUsage/{name}", http.HandlerFunc(r.GetSerieUsage))
+
+		// endpoint for perses metrics usage push from the client
+		mux.Handle("/api/v1/metrics", http.HandlerFunc(r.PushMetricsUsage))
 		r.mux = mux
 	}
 }
@@ -377,3 +382,86 @@ func (r *routes) ui(uiFS fs.FS) http.HandlerFunc {
 
 	return uiHandler.ServeHTTP
 }
+
+var usage = make(map[string]*metricsUsageV1.MetricUsage)
+
+func (r *routes) PushMetricsUsage(w http.ResponseWriter, req *http.Request) {
+	if err := json.NewDecoder(req.Body).Decode(&usage); err != nil {
+		slog.Error("unable to decode request body", "err", err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	for name, metricUsage := range usage {
+		rulesUsage := make([]db.RulesUsage, 0, len(metricUsage.AlertRules)+len(metricUsage.RecordingRules))
+		for usage, _ := range metricUsage.AlertRules {
+			rulesUsage = append(rulesUsage, db.RulesUsage{
+				Serie:      name,
+				GroupName:  usage.GroupName,
+				Name:       usage.Name,
+				Expression: usage.Expression,
+				Kind:       string(db.RuleUsageKindAlert),
+				// Labels:    usage.Labels,
+			})
+		}
+
+		for usage, _ := range metricUsage.RecordingRules {
+			rulesUsage = append(rulesUsage, db.RulesUsage{
+				Serie:      name,
+				GroupName:  usage.GroupName,
+				Name:       usage.Name,
+				Expression: usage.Expression,
+				Kind:       string(db.RuleUsageKindRecord),
+				// Labels:    usage.Labels,
+			})
+		}
+
+		if err := r.dbProvider.InsertRulesUsage(req.Context(), rulesUsage); err != nil {
+			slog.Error("unable to insert rules usage", "err", err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+func (r *routes) GetSerieUsage(w http.ResponseWriter, req *http.Request) {
+	name := req.PathValue("name")
+	if name == "" {
+		http.Error(w, "missing name parameter", http.StatusBadRequest)
+		return
+	}
+
+	kind := req.URL.Query().Get("kind")
+	if kind == "" {
+		http.Error(w, "missing kind parameter", http.StatusBadRequest)
+		return
+	}
+
+	page, err := getQueryParamAsInt(req, "page", 1)
+	if err != nil {
+		slog.Error("unable to parse page parameter", "err", err)
+		http.Error(w, "unable to parse page parameter", http.StatusBadRequest)
+		return
+	}
+
+	pageSize, err := getQueryParamAsInt(req, "pageSize", 1)
+	if err != nil {
+		slog.Error("unable to parse pageSize parameter", "err", err)
+		http.Error(w, "unable to parse pageSize parameter", http.StatusBadRequest)
+		return
+	}
+
+	if kind == "dashboard" {
+		// TODO: implement dashboard usage
+		return
+	}
+
+	alerts, err := r.dbProvider.GetRulesUsage(req.Context(), name, kind, page, pageSize)
+	if err != nil {
+		slog.Error("unable to retrieve series expressions", "err", err)
+		http.Error(w, "unable to retrieve series expressions", http.StatusInternalServerError)
+		return
+	}
+
+	writeJSONResponse(w, alerts)
+}

config.yaml

@@ -0,0 +1,18 @@
+upstream:
+  url: https://demo.promlabs.com
+  # url: https://ng-api-http.eu2.coralogix.com/metrics
+  include_query_stats: true
+
+database:
+  provider: clickhouse
+  clickhouse:
+    auth:
+      database: default
+  sqlite:
+    database_path: ./prom-analytics-proxy.db
+# tracing:
+#   client_type: "grpc"
+#   service_name: "prom-analytics-proxy"
+#   endpoint: "localhost:4317"
+#   insecure: true
+#   sampler_type: "alwayssample"

go.mod

@@ -1,6 +1,8 @@
 module github.com/nicolastakashi/prom-analytics-proxy
 
-go 1.23.0
+go 1.23.1
+
+toolchain go1.23.3
 
 require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.30.0
@@ -38,11 +40,13 @@ require (
 	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/paulmach/orb v0.11.1 // indirect
+	github.com/perses/perses v0.49.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/propagators/autoprop v0.54.0 // indirect
@@ -82,6 +86,7 @@ require (
 	github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/metalmatze/signal v0.0.0-20210307161603-1c9aa721a97a
+	github.com/perses/metrics-usage v0.5.0
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/prometheus/client_golang v1.20.5
 	github.com/prometheus/client_model v0.6.1 // indirect

go.sum

@@ -77,6 +77,7 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
 github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -164,6 +165,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
 github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/nexucis/lamenv v0.5.2 h1:tK/u3XGhCq9qIoVNcXsK9LZb8fKopm0A5weqSRvHd7M=
+github.com/nexucis/lamenv v0.5.2/go.mod h1:HusJm6ltmmT7FMG8A750mOLuME6SHCsr2iFYxp5fFi0=
 github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
 github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
@@ -173,6 +176,10 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr
 github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU=
 github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
 github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
+github.com/perses/metrics-usage v0.5.0 h1:FEGRuPSFCU7ORAcpLRzF0SctcvlbmF+ZOfYvM0bhs+0=
+github.com/perses/metrics-usage v0.5.0/go.mod h1:KTN4ww4f7hDHR3oIUzyJSqdiTJz35IeAklbOnvmpLu8=
+github.com/perses/perses v0.49.0 h1:bqVTWR9x8Vg5+ezaiX/63iT+BEwEQmjYQm4yr0LqhzU=
+github.com/perses/perses v0.49.0/go.mod h1:W2SUOoLh57MU8d7fTT7vIhYycv6ZvbAKz6ZVahBskP4=
 github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
 github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
@@ -219,6 +226,8 @@ github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp
 github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
@@ -228,6 +237,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/thanos-io/thanos v0.37.2 h1:eCv5JfqM5BQ+9+hZXZPeNgsaJRxjW2uMhstacNso8ok=
@@ -326,6 +336,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=