Merge pull request #40 from zapier/combine-steps

Add some production metrics

Earthfile

@@ -50,11 +50,12 @@ build-binary:
 build-docker:
     FROM alpine:${ALPINE_VERSION}
     COPY +build-binary/prom-aggregation-gateway .
+    ENV GIN_MODE=release
     ENTRYPOINT ["/prom-aggregation-gateway"]
     SAVE IMAGE --push ${image_name}:${version}
 
 continuous-deploy:
-    BUILD +release-helm
+    BUILD +build-helm
 
 build-binaries:
     FROM golang:${GOLANG_VERSION}
@@ -162,21 +163,13 @@ test-helm:
 build-helm:
     FROM quay.io/helmpack/chart-releaser:v${CHART_RELEASER_VERSION}
 
-    WORKDIR /src
-    COPY . /src
-
-    RUN cr --config .github/cr.yaml package charts/*
-    SAVE ARTIFACT .cr-release-packages/ AS LOCAL ./dist
-
-release-helm:
-    FROM quay.io/helmpack/chart-releaser:v${CHART_RELEASER_VERSION}
-
     ARG token
 
     WORKDIR /src
     COPY . /src
 
     RUN cr --config .github/cr.yaml package charts/*
+    SAVE ARTIFACT .cr-release-packages/ AS LOCAL ./dist
 
     RUN mkdir -p .cr-index
     RUN git config --global user.email "[email protected]"

Earthfile.md

@@ -1,26 +1,17 @@
 ```mermaid
 graph TD
-    lint-golang --> test
-    test-golang --> test
-    test-helm --> test
-    
-    build-binary --> build
-    build-docker --> build
-    build-helm --> build
-    
-    build-binary --> build-docker
-    release-binary --> release
-    release-binary -.create release.-> github
-    release-docker --> release
-    release-docker -.push package.-> github
-    
-    build-binary --> release-binary
-    build-docker --> release-docker
-    
-    release-helm --> continuous-deploy
-    release-helm -.push to gh-pages.-> github
-    
-    go-deps --> build-binary
-    go-deps --> lint-golang
-    go-deps --> test-golang
+        build --> build-docker
+        build --> build-helm
+        build-docker --> build-binary
+        continuous-deploy --> build-helm
+        lint-golang --> go-deps
+        test-golang --> go-deps
+        test --> ci-golang
+        release --> build-docker
+        release --> release-binaries
+        build-binary --> go-deps
+        ci-golang --> lint-golang
+        ci-golang --> test-golang
+        ci-helm --> test-helm
+        release-binaries --> build-binaries
 ```

aggregate.go

@@ -78,6 +78,13 @@ func (ao *aggregateOptions) formatIgnoredLabels() {
 	sort.Strings(ao.ignoredLabels)
 }
 
+func (a *aggregate) Len() int {
+	a.familiesLock.RLock()
+	count := len(a.families)
+	a.familiesLock.RUnlock()
+	return count
+}
+
 // setFamilyOrGetExistingFamily either sets a new family or returns an existing family
 func (a *aggregate) setFamilyOrGetExistingFamily(familyName string, family *dto.MetricFamily) *metricFamily {
 	a.familiesLock.Lock()
@@ -126,8 +133,12 @@ func (a *aggregate) parseAndMerge(r io.Reader, job string) error {
 			return err
 		}
 
+		MetricCountByFamily.WithLabelValues(name).Set(float64(len(family.Metric)))
+
 	}
 
+	TotalFamiliesGauge.Set(float64(a.Len()))
+
 	return nil
 }
 
@@ -140,9 +151,18 @@ func (a *aggregate) handleRender(c *gin.Context) {
 	defer a.familiesLock.RUnlock()
 
 	metricNames := []string{}
-	for name := range a.families {
+	metricTypeCounts := make(map[string]int)
+	for name, family := range a.families {
 		metricNames = append(metricNames, name)
+		var typeName string
+		if family.Type == nil {
+			typeName = "unknown"
+		} else {
+			typeName = dto.MetricType_name[int32(*family.Type)]
+		}
+		metricTypeCounts[typeName]++
 	}
+
 	sort.Strings(metricNames)
 
 	for _, name := range metricNames {
@@ -151,6 +171,11 @@ func (a *aggregate) handleRender(c *gin.Context) {
 		}
 	}
 
+	MetricCountByType.Reset()
+	for typeName, count := range metricTypeCounts {
+		MetricCountByType.WithLabelValues(typeName).Set(float64(count))
+	}
+
 	// TODO reset gauges
 }
 
@@ -180,4 +205,6 @@ func (a *aggregate) handleInsert(c *gin.Context) {
 		http.Error(c.Writer, err.Error(), http.StatusBadRequest)
 		return
 	}
+
+	MetricPushes.WithLabelValues(job).Inc()
 }

aggregate_test.go

@@ -8,6 +8,8 @@ import (
 	"testing"
 
 	"github.com/pmezard/go-difflib/difflib"
+	"github.com/prometheus/client_golang/prometheus"
+	metrics "github.com/slok/go-http-metrics/metrics/prometheus"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
 )
@@ -172,7 +174,7 @@ func TestAggregate(t *testing.T) {
 	} {
 		t.Run(c.testName, func(t *testing.T) {
 			agg := newAggregate(AddIgnoredLabels(c.ignoredLabels...))
-			router := setupAPIRouter("*", agg)
+			router := setupAPIRouter("*", agg, metrics.Config{ Registry: prometheus.NewRegistry()})
 
 			err := agg.parseAndMerge(strings.NewReader(c.a), "test")
 			require.NoError(t, err)

main.go

@@ -9,6 +9,7 @@ import (
 	"syscall"
 
 	"github.com/gin-gonic/gin"
+	metrics "github.com/slok/go-http-metrics/metrics/prometheus"
 )
 
 var (
@@ -39,10 +40,14 @@ func runServers(corsDomain string, apiListen string, lifecycleListen string) {
 
 	agg := newAggregate()
 
-	apiRouter := setupAPIRouter(corsDomain, agg)
+	promMetricsConfig := metrics.Config{
+		Registry: promRegistry,
+	}
+
+	apiRouter := setupAPIRouter(corsDomain, agg, promMetricsConfig)
 	go runServer("api", apiRouter, apiListen)
 
-	lifecycleRouter := setupLifecycleRouter()
+	lifecycleRouter := setupLifecycleRouter(promRegistry)
 	go runServer("lifecycle", lifecycleRouter, lifecycleListen)
 
 	// Block until an interrupt or term signal is sent

metrics.go

@@ -0,0 +1,56 @@
+package main
+
+import "github.com/prometheus/client_golang/prometheus"
+
+const MetricsNamespace = "prom_agg_gateway"
+
+var promRegistry = prometheus.NewRegistry()
+
+func init() {
+	promRegistry.MustRegister(
+		TotalFamiliesGauge,
+		MetricCountByFamily,
+		MetricPushes,
+	)
+}
+
+var TotalFamiliesGauge = prometheus.NewGauge(
+	prometheus.GaugeOpts{
+		Namespace: MetricsNamespace,
+		Name:      "total_families",
+		Help:      "Total number of metric families",
+	},
+)
+
+var MetricCountByFamily = prometheus.NewGaugeVec(
+	prometheus.GaugeOpts{
+		Namespace: MetricsNamespace,
+		Name:      "metrics_by_family",
+		Help:      "Metric count by family",
+	},
+	[]string{
+		"family",
+	},
+)
+
+var MetricCountByType = prometheus.NewGaugeVec(
+	prometheus.GaugeOpts{
+		Namespace: MetricsNamespace,
+		Name: "metrics_by_type",
+		Help: "Metric count by type",
+	},
+	[]string{
+		"metric_type",
+	},
+);
+
+var MetricPushes = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Namespace: MetricsNamespace,
+		Name:      "metric_pushes",
+		Help:      "Total number of metric push requests, per job",
+	},
+	[]string{
+		"push_job",
+	},
+)

privateRouter.go

@@ -4,15 +4,21 @@ import (
 	"net/http"
 
 	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
-func setupLifecycleRouter() *gin.Engine {
+func setupLifecycleRouter(promRegistry *prometheus.Registry) *gin.Engine {
 	r := gin.New()
 
+	metricsHandler := promhttp.InstrumentMetricHandler(
+		promRegistry,
+		promhttp.HandlerFor(promRegistry, promhttp.HandlerOpts{}),
+	)
+
 	r.GET("/healthy", handleHealthCheck)
 	r.GET("/ready", handleHealthCheck)
-	r.GET("/metrics", convertHandler(promhttp.Handler()))
+	r.GET("/metrics", convertHandler(metricsHandler))
 
 	return r
 }

publicRouter.go

@@ -3,24 +3,21 @@ package main
 import (
 	"github.com/gin-contrib/cors"
 	"github.com/gin-gonic/gin"
-	"github.com/prometheus/client_golang/prometheus"
 	metrics "github.com/slok/go-http-metrics/metrics/prometheus"
 	"github.com/slok/go-http-metrics/middleware"
 	mGin "github.com/slok/go-http-metrics/middleware/gin"
 )
 
-func setupAPIRouter(corsDomain string, agg *aggregate) *gin.Engine {
+func setupAPIRouter(corsDomain string, agg *aggregate, promConfig metrics.Config) *gin.Engine {
 	corsConfig := cors.Config{}
 	if corsDomain != "*" {
 		corsConfig.AllowOrigins = []string{corsDomain}
 	} else {
 		corsConfig.AllowAllOrigins = true
 	}
 
-	cfg := new(metrics.Config)
-	cfg.Registry = prometheus.NewRegistry()
 	metricsMiddleware := middleware.New(middleware.Config{
-		Recorder: metrics.NewRecorder(*cfg),
+		Recorder: metrics.NewRecorder(promConfig),
 	})
 
 	r := gin.New()

skaffold.yaml

@@ -5,9 +5,9 @@ metadata:
   name: prom-aggregation-gateway
 build:
   artifacts:
-    - image: ghcr.io/zapier/prom-aggregation-gateway
+    - image: prom-aggregation-gateway
       custom:
-        buildCommand: earthly +build-docker --version=dev --image_name=ghcr.io/zapier/prom-aggregation-gateway
+        buildCommand: earthly +build-docker --version=$IMAGE_TAG --image_name=$IMAGE_REPO
   tagPolicy:
     customTemplate:
       template: dev
@@ -18,6 +18,7 @@ manifests:
         chartPath: ./charts/prom-aggregation-gateway
         createNamespace: true
         setValues:
-          image.tag: dev
+          controller.image.repository: prom-aggregation-gateway
+          controller.image.tag: dev
           podMonitor.create: false
           serviceMonitor.create: false