diff --git a/app/kuma-dp/pkg/dataplane/metrics/server.go b/app/kuma-dp/pkg/dataplane/metrics/server.go index 1ca35f50534b..01b02012ae0f 100644 --- a/app/kuma-dp/pkg/dataplane/metrics/server.go +++ b/app/kuma-dp/pkg/dataplane/metrics/server.go @@ -5,11 +5,15 @@ import ( "context" "fmt" "io" + "math" + "mime" "net" "net/http" "net/url" "os" + "regexp" "strconv" + "strings" "sync" "time" @@ -30,6 +34,23 @@ var ( var ( prometheusRequestHeaders = []string{"accept", "accept-encoding", "user-agent", "x-prometheus-scrape-timeout-seconds"} logger = core.Log.WithName("metrics-hijacker") + + // holds prometheus content types in order of priority. + prometheusPriorityContentType = []expfmt.Format{ + expfmt.FmtOpenMetrics_1_0_0, + expfmt.FmtOpenMetrics_0_0_1, + expfmt.FmtText, + expfmt.FmtUnknown, + } + + // Reverse mapping of prometheusPriorityContentType for faster lookup. + prometheusPriorityContentTypeLookup = func(expformats []expfmt.Format) map[expfmt.Format]int32 { + reverseMapping := map[expfmt.Format]int32{} + for priority, format := range expformats { + reverseMapping[format] = int32(priority) + } + return reverseMapping + }(prometheusPriorityContentType) ) var _ component.Component = &Hijacker{} @@ -153,19 +174,27 @@ func rewriteMetricsURL(address string, port uint32, path string, queryModifier Q func (s *Hijacker) ServeHTTP(writer http.ResponseWriter, req *http.Request) { ctx := req.Context() out := make(chan []byte, len(s.applicationsToScrape)) + contentTypes := make(chan expfmt.Format, len(s.applicationsToScrape)) var wg sync.WaitGroup done := make(chan []byte) wg.Add(len(s.applicationsToScrape)) go func() { wg.Wait() - close(done) close(out) + close(contentTypes) + close(done) }() for _, app := range s.applicationsToScrape { go func(app ApplicationToScrape) { defer wg.Done() - out <- s.getStats(ctx, req, app) + content, contentType := s.getStats(ctx, req, app) + out <- content + + // It's possible to track the highest priority content type seen, + // but that would require mutex. + // I would prefer to calculate it later at one go + contentTypes <- contentType }(app) } @@ -173,24 +202,118 @@ func (s *Hijacker) ServeHTTP(writer http.ResponseWriter, req *http.Request) { case <-ctx.Done(): return case <-done: - // default format returned by prometheus - writer.Header().Set("content-type", string(expfmt.FmtText)) - for resp := range out { - if _, err := writer.Write(resp); err != nil { - logger.Error(err, "error while writing the response") - } - if _, err := writer.Write([]byte("\n")); err != nil { - logger.Error(err, "error while writing the response") - } + selectedCt := selectContentType(contentTypes, req.Header) + writer.Header().Set(hdrContentType, string(selectedCt)) + + // aggregate metrics of target applications and attempt to make them + // compatible with FmtOpenMetrics if it is the selected content type. + metrics := processMetrics(out, selectedCt) + if _, err := writer.Write(metrics); err != nil { + logger.Error(err, "error while writing the response") } } } -func (s *Hijacker) getStats(ctx context.Context, initReq *http.Request, app ApplicationToScrape) []byte { +func processMetrics(metrices <-chan []byte, contentType expfmt.Format) []byte { + buf := new(bytes.Buffer) + + for metrics := range metrices { + // remove the EOF marker from the metrics, because we are + // merging multiple metrics into one response. + metrics = bytes.ReplaceAll(metrics, []byte("# EOF"), []byte("")) + + if _, err := buf.Write(metrics); err != nil { + logger.Error(err, "error while writing the response to temporary buffer") + } + if _, err := buf.Write([]byte("\n")); err != nil { + logger.Error(err, "error while writing the response") + } + } + + str := processNewlineChars(buf.String(), true, true) + str = fmt.Sprintf("%s\n", str) + + // if the content type is not OpenMetrics, we don't need to add EOF marker + if !(contentType == expfmt.FmtOpenMetrics_1_0_0 || contentType == expfmt.FmtOpenMetrics_0_0_1) { + return []byte(str) + } + + // make metrics OpenMetrics compliant + fmt.Printf("%q", fmt.Sprintf("%s# EOF\n", str)) + return []byte(fmt.Sprintf("%s# EOF\n", str)) +} + +// processNewlineChars processes the newline characters in the text. +// If dedup is true, it replaces multiple newline characters with a single newline character. +// If trim is true, it trims the leading and trailing newline characters. +func processNewlineChars(text string, dedup, trim bool) string { + if dedup { + // Create a regular expression to match multiple newline characters. + reg, err := regexp.Compile(`(\r\n?|\n){2,}`) + if err != nil { + return text + } + + // Replace all the matches with a single newline character. + text = reg.ReplaceAllString(text, "\n") + } + + if trim { + // Trim the leading and trailing newline characters. + text = strings.TrimFunc(text, func(r rune) bool { + return r == '\n' + }) + } + + return text +} + +// selectContentType selects the highest priority content type supported by the applications. +// If no valid content type is returned by the applications, it negotiates content type based +// on Accept header of the scraper. +func selectContentType(contentTypes <-chan expfmt.Format, reqHeader http.Header) expfmt.Format { + // Tracks highest negotiated content type priority. + // Lower number means higher priority + // + // We can not simply use the highest priority content type i.e. `application/openmetrics-text` + // and try to mutate the metrics to make it compatible with this type, + // because: + // - if the application is not supporting this type, + // custom metrics might not be compatible (more prone to failure). + // - the user might be using older prom scraper. + // + // So it's better to choose the highest negotiated content type between the + // target apps and the scraper. + var ctPriority int32 = math.MaxInt32 + ct := expfmt.FmtUnknown + for contentType := range contentTypes { + priority, valid := prometheusPriorityContentTypeLookup[contentType] + if !valid { + continue + } + if priority < ctPriority { + ctPriority = priority + ct = contentType + } + } + + // If no valid content type is returned by the target applications, + // negotitate content type based on Accept header of the scraper. + // + // Note: NegotiateIncludingOpenMetrics is not yet fully supported. + // So invoke expfmt.Negotiate() instead + if ct == expfmt.FmtUnknown { + ct = expfmt.Negotiate(reqHeader) + } + + return ct +} + +func (s *Hijacker) getStats(ctx context.Context, initReq *http.Request, app ApplicationToScrape) ([]byte, expfmt.Format) { req, err := http.NewRequest("GET", rewriteMetricsURL(app.Address, app.Port, app.Path, app.QueryModifier, initReq.URL), nil) if err != nil { logger.Error(err, "failed to create request") - return nil + return nil, "" } s.passRequestHeaders(req.Header, initReq.Header) req = req.WithContext(ctx) @@ -208,25 +331,27 @@ func (s *Hijacker) getStats(ctx context.Context, initReq *http.Request, app Appl } if err != nil { logger.Error(err, "failed call", "name", app.Name, "path", app.Path, "port", app.Port) - return nil + return nil, "" } + respContentType := responseFormat(resp.Header) + var bodyBytes []byte if app.Mutator != nil { buf := new(bytes.Buffer) if err := app.Mutator(resp.Body, buf); err != nil { logger.Error(err, "failed while mutating data", "name", app.Name, "path", app.Path, "port", app.Port) - return nil + return nil, "" } bodyBytes = buf.Bytes() } else { bodyBytes, err = io.ReadAll(resp.Body) if err != nil { logger.Error(err, "failed while writing", "name", app.Name, "path", app.Path, "port", app.Port) - return nil + return nil, "" } } - return bodyBytes + return bodyBytes, respContentType } func (s *Hijacker) passRequestHeaders(into http.Header, from http.Header) { @@ -243,3 +368,53 @@ func (s *Hijacker) passRequestHeaders(into http.Header, from http.Header) { func (s *Hijacker) NeedLeaderElection() bool { return false } + +const ( + hdrContentType = "Content-Type" + textType = "text/plain" + textVersion = "0.0.4" + openmetricsVersion_1_0_0 = "1.0.0" + openmetricsVersion_0_0_1 = "0.0.1" + protoType = `application/vnd.google.protobuf` + protoProtocol = `io.prometheus.client.MetricFamily` +) + +// responseFormat extracts the correct format from a HTTP response header. +// If no matching format can be found FormatUnknown is returned. +func responseFormat(h http.Header) expfmt.Format { + ct := h.Get(hdrContentType) + + mediatype, params, err := mime.ParseMediaType(ct) + if err != nil { + return expfmt.FmtUnknown + } + + version := params["version"] + + switch mediatype { + case protoType: + p := params["proto"] + e := params["encoding"] + // only delimited encoding is supported by prometheus scraper + if p == protoProtocol && e == "delimited" { + return expfmt.FmtProtoDelim + } + + // if mediatype is `text/plain`, return Prometheus text format + // without checking the version, as there are few exporters + // which don't set the version param in the content-type header. ex: Envoy + case textType: + return expfmt.FmtText + + // TODO: if version does not match or set, should we return FmtUnknown? + case expfmt.OpenMetricsType: + if version == openmetricsVersion_0_0_1 { + return expfmt.FmtOpenMetrics_0_0_1 + } + if version == openmetricsVersion_1_0_0 { + return expfmt.FmtOpenMetrics_1_0_0 + } + } + + return expfmt.FmtUnknown +} diff --git a/app/kuma-dp/pkg/dataplane/metrics/server_test.go b/app/kuma-dp/pkg/dataplane/metrics/server_test.go index 5b4aa1664a52..bea6e0de7ab5 100644 --- a/app/kuma-dp/pkg/dataplane/metrics/server_test.go +++ b/app/kuma-dp/pkg/dataplane/metrics/server_test.go @@ -1,10 +1,15 @@ package metrics import ( + "io" + "net/http" "net/url" + "os" + "path" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/prometheus/common/expfmt" ) var _ = Describe("Rewriting the metrics URL", func() { @@ -44,3 +49,165 @@ var _ = Describe("Rewriting the metrics URL", func() { }), ) }) + +var _ = Describe("Select Content Type", func() { + var reqHeader http.Header + BeforeEach(func() { + reqHeader = make(http.Header) + }) + + It("should honor app content-type", func() { + contentTypes := make(chan expfmt.Format, 3) + contentTypes <- expfmt.FmtOpenMetrics_0_0_1 + contentTypes <- expfmt.Format("") + contentTypes <- expfmt.FmtText + close(contentTypes) + reqHeader.Add("Accept", "application/openmetrics-text;version=1.0.0,application/openmetrics-text;version=0.0.1;q=0.75,text/plain;version=0.0.4;q=0.5,*/*;q=0.1") + + actualContentType := selectContentType(contentTypes, reqHeader) + Expect(actualContentType).To(Equal(expfmt.FmtOpenMetrics_0_0_1)) + }) + + It("should negotiate content-type based on Accept header", func() { + contentTypes := make(chan expfmt.Format, 1) + contentTypes <- expfmt.Format("invalid_content_type") + close(contentTypes) + reqHeader.Add("Accept", "application/openmetrics-text;version=1.0.0,application/openmetrics-text;version=0.0.1;q=0.75,text/plain;version=0.0.4;q=0.5,*/*;q=0.1") + + actualContentType := selectContentType(contentTypes, reqHeader) + Expect(actualContentType).To(Equal(expfmt.Negotiate(reqHeader))) + }) + + It("should negotiate content-type based on Accept header", func() { + contentTypes := make(chan expfmt.Format, 1) + contentTypes <- expfmt.Format("invalid_content_type") + close(contentTypes) + reqHeader.Add("Accept", "*/*") + + actualContentType := selectContentType(contentTypes, reqHeader) + Expect(actualContentType).To(Equal(expfmt.Negotiate(reqHeader))) + }) +}) + +var _ = Describe("Response Format", func() { + type testCase struct { + contentType string + expectedFormat expfmt.Format + } + DescribeTable("should", + func(given testCase) { + h := make(http.Header) + h.Set(hdrContentType, given.contentType) + Expect(responseFormat(h)).To(Equal(given.expectedFormat)) + }, + Entry("return FmtProtoDelim for a 'delimited protobuf content type' response", testCase{ + contentType: "application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited", + expectedFormat: expfmt.FmtProtoDelim, + }), + Entry("return FmtUnknown for a 'text protobuf content type' response", testCase{ + contentType: "application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=text", + expectedFormat: expfmt.FmtUnknown, + }), + Entry("return FmtText for a 'text plain content type' response", testCase{ + contentType: "text/plain; charset=UTF-8", + expectedFormat: expfmt.FmtText, + }), + Entry("return FmtOpenMetrics_1_0_0 for a 'openmetrics v1.0.0 content type' response", testCase{ + contentType: "application/openmetrics-text; version=1.0.0", + expectedFormat: expfmt.FmtOpenMetrics_1_0_0, + }), + Entry("return FmtOpenMetrics_0_0_1 for a 'openmetrics v0.0.1 content type' response", testCase{ + contentType: "application/openmetrics-text; version=0.0.1", + expectedFormat: expfmt.FmtOpenMetrics_0_0_1, + }), + Entry("return FmtUnknown for a 'invalid content type' response", testCase{ + contentType: "application/invalid", + expectedFormat: expfmt.FmtUnknown, + }), + ) +}) + +var _ = Describe("Process Metrics", func() { + type testCase struct { + input []string // input files containing metrics + contentType expfmt.Format + expected string // expected output file + } + DescribeTable("should", + func(given testCase) { + inputMetrics := make(chan []byte, len(given.input)) + for _, input := range given.input { + fo, err := os.Open(path.Join("testdata", input)) + Expect(err).ToNot(HaveOccurred()) + byteData, err := io.ReadAll(fo) + Expect(err).ToNot(HaveOccurred()) + inputMetrics <- byteData + } + close(inputMetrics) + + fo, err := os.Open(path.Join("testdata", given.expected)) + Expect(err).ToNot(HaveOccurred()) + expected, err := io.ReadAll(fo) + Expect(err).ToNot(HaveOccurred()) + + actual := processMetrics(inputMetrics, given.contentType) + os.WriteFile("actual.txt", actual, 0644) + Expect(string(actual)).To(Equal(string(expected))) + }, + Entry("return OpenMetrics compliant metrics", testCase{ + input: []string{"openmetrics_0_1_1.in", "counter.out"}, + contentType: expfmt.FmtOpenMetrics_0_0_1, + expected: "openmetrics_0_0_1-counter.out", + }), + Entry("handle multiple # EOF", testCase{ + input: []string{"openmetrics_0_1_1.in", "openmetrics_0_1_1.in", "counter.out"}, + contentType: expfmt.FmtOpenMetrics_0_0_1, + expected: "multi-openmetrics-counter.out", + }), + Entry("return Prometheus text compliant metrics", testCase{ + input: []string{"prom-text.in", "counter.out"}, + contentType: expfmt.FmtText, + expected: "prom-text-counter.out", + }), + ) +}) + +var _ = Describe("ProcessNewlineChars", func() { + type testCase struct { + input string + deduplicate bool + trim bool + expected string + } + + DescribeTable("should", + func(given testCase) { + actual := processNewlineChars(given.input, given.deduplicate, given.trim) + Expect(actual).To(Equal(given.expected)) + }, + Entry("should not deduplicate or trim newline characters", testCase{ + input: "This is a test.\n\nThis is another test.\n", + deduplicate: false, + trim: false, + expected: "This is a test.\n\nThis is another test.\n", + }), + Entry("should deduplicate newline characters", testCase{ + input: "This is a test.\n\n\nThis is another test.\n", + deduplicate: true, + trim: false, + expected: "This is a test.\nThis is another test.\n", + }), + Entry("should trim leading and trailing newline characters", testCase{ + input: "\nThis is a test.\n\nThis is another test\n\n", + deduplicate: false, + trim: true, + expected: "This is a test.\n\nThis is another test", + }), + Entry("should deduplicate and trim newline characters", testCase{ + input: "\nThis is a test.\n\n\nThis is another test\n", + deduplicate: true, + trim: true, + expected: "This is a test.\nThis is another test", + }), + ) +}) diff --git a/app/kuma-dp/pkg/dataplane/metrics/testdata/multi-openmetrics-counter.out b/app/kuma-dp/pkg/dataplane/metrics/testdata/multi-openmetrics-counter.out new file mode 100644 index 000000000000..1d764373a729 --- /dev/null +++ b/app/kuma-dp/pkg/dataplane/metrics/testdata/multi-openmetrics-counter.out @@ -0,0 +1,61 @@ +# HELP python_gc_objects_collected Objects collected during gc +# TYPE python_gc_objects_collected counter +python_gc_objects_collected_total{generation="0"} 309.0 +python_gc_objects_collected_total{generation="1"} 53.0 +python_gc_objects_collected_total{generation="2"} 0.0 +# HELP python_gc_objects_uncollectable Uncollectable objects found during GC +# TYPE python_gc_objects_uncollectable counter +python_gc_objects_uncollectable_total{generation="0"} 0.0 +python_gc_objects_uncollectable_total{generation="1"} 0.0 +python_gc_objects_uncollectable_total{generation="2"} 0.0 +# HELP python_gc_collections Number of times this generation was collected +# TYPE python_gc_collections counter +python_gc_collections_total{generation="0"} 77.0 +python_gc_collections_total{generation="1"} 7.0 +python_gc_collections_total{generation="2"} 0.0 +# HELP python_info Python platform information +# TYPE python_info gauge +python_info{implementation="CPython",major="3",minor="10",patchlevel="8",version="3.10.8"} 1.0 +# HELP python_gc_objects_collected Objects collected during gc +# TYPE python_gc_objects_collected counter +python_gc_objects_collected_total{generation="0"} 309.0 +python_gc_objects_collected_total{generation="1"} 53.0 +python_gc_objects_collected_total{generation="2"} 0.0 +# HELP python_gc_objects_uncollectable Uncollectable objects found during GC +# TYPE python_gc_objects_uncollectable counter +python_gc_objects_uncollectable_total{generation="0"} 0.0 +python_gc_objects_uncollectable_total{generation="1"} 0.0 +python_gc_objects_uncollectable_total{generation="2"} 0.0 +# HELP python_gc_collections Number of times this generation was collected +# TYPE python_gc_collections counter +python_gc_collections_total{generation="0"} 77.0 +python_gc_collections_total{generation="1"} 7.0 +python_gc_collections_total{generation="2"} 0.0 +# HELP python_info Python platform information +# TYPE python_info gauge +python_info{implementation="CPython",major="3",minor="10",patchlevel="8",version="3.10.8"} 1.0 +# TYPE envoy_cluster_assignment_stale counter +envoy_cluster_assignment_stale{envoy_cluster_name="access_log_sink"} 11 +envoy_cluster_assignment_stale{envoy_cluster_name="ads_cluster"} 12 +envoy_cluster_assignment_stale{envoy_cluster_name="demo-client",kuma_io_mesh_traffic="true"} 13 +envoy_cluster_assignment_stale{envoy_cluster_name="echo-server_kuma-test_svc_8080",kuma_io_mesh_traffic="true"} 306 +envoy_cluster_assignment_stale{envoy_cluster_name="inbound_passthrough_ipv4",kuma_io_mesh_traffic="true"} 14 +envoy_cluster_assignment_stale{envoy_cluster_name="inbound_passthrough_ipv6",kuma_io_mesh_traffic="true"} 15 +envoy_cluster_assignment_stale{envoy_cluster_name="kuma_envoy_admin"} 16 +envoy_cluster_assignment_stale{envoy_cluster_name="localhost_3000",kuma_io_mesh_traffic="true"} 17 +envoy_cluster_assignment_stale{envoy_cluster_name="outbound_passthrough_ipv4",kuma_io_mesh_traffic="true"} 18 +envoy_cluster_assignment_stale{envoy_cluster_name="outbound_passthrough_ipv6",kuma_io_mesh_traffic="true"} 19 +# TYPE envoy_http_downstream_rq_total counter +envoy_http_downstream_rq_total{envoy_http_conn_manager_prefix="default_gateway",kuma_io_mesh_traffic="true"} 10 +# TYPE envoy_cluster_assignment_timeout_received counter +envoy_cluster_assignment_timeout_received{envoy_cluster_name="access_log_sink"} 1 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="ads_cluster"} 2 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="demo-client",kuma_io_mesh_traffic="true"} 3 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="echo-server_kuma-test_svc_8080",kuma_io_mesh_traffic="true"} 606 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="inbound_passthrough_ipv4",kuma_io_mesh_traffic="true"} 4 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="inbound_passthrough_ipv6",kuma_io_mesh_traffic="true"} 5 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="kuma_envoy_admin"} 6 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="localhost_3000",kuma_io_mesh_traffic="true"} 7 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="outbound_passthrough_ipv4",kuma_io_mesh_traffic="true"} 8 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="outbound_passthrough_ipv6",kuma_io_mesh_traffic="true"} 9 +# EOF diff --git a/app/kuma-dp/pkg/dataplane/metrics/testdata/openmetrics_0_0_1-counter.out b/app/kuma-dp/pkg/dataplane/metrics/testdata/openmetrics_0_0_1-counter.out new file mode 100644 index 000000000000..096c89143521 --- /dev/null +++ b/app/kuma-dp/pkg/dataplane/metrics/testdata/openmetrics_0_0_1-counter.out @@ -0,0 +1,43 @@ +# HELP python_gc_objects_collected Objects collected during gc +# TYPE python_gc_objects_collected counter +python_gc_objects_collected_total{generation="0"} 309.0 +python_gc_objects_collected_total{generation="1"} 53.0 +python_gc_objects_collected_total{generation="2"} 0.0 +# HELP python_gc_objects_uncollectable Uncollectable objects found during GC +# TYPE python_gc_objects_uncollectable counter +python_gc_objects_uncollectable_total{generation="0"} 0.0 +python_gc_objects_uncollectable_total{generation="1"} 0.0 +python_gc_objects_uncollectable_total{generation="2"} 0.0 +# HELP python_gc_collections Number of times this generation was collected +# TYPE python_gc_collections counter +python_gc_collections_total{generation="0"} 77.0 +python_gc_collections_total{generation="1"} 7.0 +python_gc_collections_total{generation="2"} 0.0 +# HELP python_info Python platform information +# TYPE python_info gauge +python_info{implementation="CPython",major="3",minor="10",patchlevel="8",version="3.10.8"} 1.0 +# TYPE envoy_cluster_assignment_stale counter +envoy_cluster_assignment_stale{envoy_cluster_name="access_log_sink"} 11 +envoy_cluster_assignment_stale{envoy_cluster_name="ads_cluster"} 12 +envoy_cluster_assignment_stale{envoy_cluster_name="demo-client",kuma_io_mesh_traffic="true"} 13 +envoy_cluster_assignment_stale{envoy_cluster_name="echo-server_kuma-test_svc_8080",kuma_io_mesh_traffic="true"} 306 +envoy_cluster_assignment_stale{envoy_cluster_name="inbound_passthrough_ipv4",kuma_io_mesh_traffic="true"} 14 +envoy_cluster_assignment_stale{envoy_cluster_name="inbound_passthrough_ipv6",kuma_io_mesh_traffic="true"} 15 +envoy_cluster_assignment_stale{envoy_cluster_name="kuma_envoy_admin"} 16 +envoy_cluster_assignment_stale{envoy_cluster_name="localhost_3000",kuma_io_mesh_traffic="true"} 17 +envoy_cluster_assignment_stale{envoy_cluster_name="outbound_passthrough_ipv4",kuma_io_mesh_traffic="true"} 18 +envoy_cluster_assignment_stale{envoy_cluster_name="outbound_passthrough_ipv6",kuma_io_mesh_traffic="true"} 19 +# TYPE envoy_http_downstream_rq_total counter +envoy_http_downstream_rq_total{envoy_http_conn_manager_prefix="default_gateway",kuma_io_mesh_traffic="true"} 10 +# TYPE envoy_cluster_assignment_timeout_received counter +envoy_cluster_assignment_timeout_received{envoy_cluster_name="access_log_sink"} 1 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="ads_cluster"} 2 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="demo-client",kuma_io_mesh_traffic="true"} 3 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="echo-server_kuma-test_svc_8080",kuma_io_mesh_traffic="true"} 606 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="inbound_passthrough_ipv4",kuma_io_mesh_traffic="true"} 4 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="inbound_passthrough_ipv6",kuma_io_mesh_traffic="true"} 5 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="kuma_envoy_admin"} 6 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="localhost_3000",kuma_io_mesh_traffic="true"} 7 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="outbound_passthrough_ipv4",kuma_io_mesh_traffic="true"} 8 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="outbound_passthrough_ipv6",kuma_io_mesh_traffic="true"} 9 +# EOF diff --git a/app/kuma-dp/pkg/dataplane/metrics/testdata/openmetrics_0_1_1.in b/app/kuma-dp/pkg/dataplane/metrics/testdata/openmetrics_0_1_1.in new file mode 100644 index 000000000000..45c664e5fdab --- /dev/null +++ b/app/kuma-dp/pkg/dataplane/metrics/testdata/openmetrics_0_1_1.in @@ -0,0 +1,19 @@ +# HELP python_gc_objects_collected Objects collected during gc +# TYPE python_gc_objects_collected counter +python_gc_objects_collected_total{generation="0"} 309.0 +python_gc_objects_collected_total{generation="1"} 53.0 +python_gc_objects_collected_total{generation="2"} 0.0 +# HELP python_gc_objects_uncollectable Uncollectable objects found during GC +# TYPE python_gc_objects_uncollectable counter +python_gc_objects_uncollectable_total{generation="0"} 0.0 +python_gc_objects_uncollectable_total{generation="1"} 0.0 +python_gc_objects_uncollectable_total{generation="2"} 0.0 +# HELP python_gc_collections Number of times this generation was collected +# TYPE python_gc_collections counter +python_gc_collections_total{generation="0"} 77.0 +python_gc_collections_total{generation="1"} 7.0 +python_gc_collections_total{generation="2"} 0.0 +# HELP python_info Python platform information +# TYPE python_info gauge +python_info{implementation="CPython",major="3",minor="10",patchlevel="8",version="3.10.8"} 1.0 +# EOF diff --git a/app/kuma-dp/pkg/dataplane/metrics/testdata/prom-text-counter.out b/app/kuma-dp/pkg/dataplane/metrics/testdata/prom-text-counter.out new file mode 100644 index 000000000000..5875acade51a --- /dev/null +++ b/app/kuma-dp/pkg/dataplane/metrics/testdata/prom-text-counter.out @@ -0,0 +1,135 @@ +# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} 6.1185e-05 +go_gc_duration_seconds{quantile="0.25"} 6.5599e-05 +go_gc_duration_seconds{quantile="0.5"} 0.000171854 +go_gc_duration_seconds{quantile="0.75"} 0.000200275 +go_gc_duration_seconds{quantile="1"} 0.000257027 +go_gc_duration_seconds_sum 0.000902316 +go_gc_duration_seconds_count 6 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 28 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{version="go1.19.2"} 1 +# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes 5.105112e+06 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total 1.2656064e+07 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes 6791 +# HELP go_memstats_frees_total Total number of frees. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total 67648 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes 9.831256e+06 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes 5.105112e+06 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes 1.0928128e+07 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes 8.732672e+06 +# HELP go_memstats_heap_objects Number of allocated objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects 30811 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes 7.241728e+06 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes 1.96608e+07 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds 1.6876005560881631e+09 +# HELP go_memstats_lookups_total Total number of pointer lookups. +# TYPE go_memstats_lookups_total counter +go_memstats_lookups_total 0 +# HELP go_memstats_mallocs_total Total number of mallocs. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total 98459 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes 19200 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes 31200 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes 202640 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes 244800 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes 1.038824e+07 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes 2.975761e+06 +# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes 1.31072e+06 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes 1.31072e+06 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes 3.4061328e+07 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads 21 +# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served. +# TYPE promhttp_metric_handler_requests_in_flight gauge +promhttp_metric_handler_requests_in_flight 1 +# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code. +# TYPE promhttp_metric_handler_requests_total counter +promhttp_metric_handler_requests_total{code="200"} 0 +promhttp_metric_handler_requests_total{code="500"} 0 +promhttp_metric_handler_requests_total{code="503"} 0 +# HELP sm_api_http_req_process_duration Duration of HTTP requests. +# TYPE sm_api_http_req_process_duration histogram +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="0.005"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="0.01"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="0.025"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="0.05"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="0.1"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="0.25"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="0.5"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="1"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="2.5"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="5"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="10"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="+Inf"} 0 +sm_api_http_req_process_duration_sum{method="GET",path="/api/metrics",route_name="Metrics"} 0 +sm_api_http_req_process_duration_count{method="GET",path="/api/metrics",route_name="Metrics"} 0 +# TYPE envoy_cluster_assignment_stale counter +envoy_cluster_assignment_stale{envoy_cluster_name="access_log_sink"} 11 +envoy_cluster_assignment_stale{envoy_cluster_name="ads_cluster"} 12 +envoy_cluster_assignment_stale{envoy_cluster_name="demo-client",kuma_io_mesh_traffic="true"} 13 +envoy_cluster_assignment_stale{envoy_cluster_name="echo-server_kuma-test_svc_8080",kuma_io_mesh_traffic="true"} 306 +envoy_cluster_assignment_stale{envoy_cluster_name="inbound_passthrough_ipv4",kuma_io_mesh_traffic="true"} 14 +envoy_cluster_assignment_stale{envoy_cluster_name="inbound_passthrough_ipv6",kuma_io_mesh_traffic="true"} 15 +envoy_cluster_assignment_stale{envoy_cluster_name="kuma_envoy_admin"} 16 +envoy_cluster_assignment_stale{envoy_cluster_name="localhost_3000",kuma_io_mesh_traffic="true"} 17 +envoy_cluster_assignment_stale{envoy_cluster_name="outbound_passthrough_ipv4",kuma_io_mesh_traffic="true"} 18 +envoy_cluster_assignment_stale{envoy_cluster_name="outbound_passthrough_ipv6",kuma_io_mesh_traffic="true"} 19 +# TYPE envoy_http_downstream_rq_total counter +envoy_http_downstream_rq_total{envoy_http_conn_manager_prefix="default_gateway",kuma_io_mesh_traffic="true"} 10 +# TYPE envoy_cluster_assignment_timeout_received counter +envoy_cluster_assignment_timeout_received{envoy_cluster_name="access_log_sink"} 1 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="ads_cluster"} 2 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="demo-client",kuma_io_mesh_traffic="true"} 3 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="echo-server_kuma-test_svc_8080",kuma_io_mesh_traffic="true"} 606 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="inbound_passthrough_ipv4",kuma_io_mesh_traffic="true"} 4 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="inbound_passthrough_ipv6",kuma_io_mesh_traffic="true"} 5 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="kuma_envoy_admin"} 6 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="localhost_3000",kuma_io_mesh_traffic="true"} 7 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="outbound_passthrough_ipv4",kuma_io_mesh_traffic="true"} 8 +envoy_cluster_assignment_timeout_received{envoy_cluster_name="outbound_passthrough_ipv6",kuma_io_mesh_traffic="true"} 9 diff --git a/app/kuma-dp/pkg/dataplane/metrics/testdata/prom-text.in b/app/kuma-dp/pkg/dataplane/metrics/testdata/prom-text.in new file mode 100644 index 000000000000..e2b922601803 --- /dev/null +++ b/app/kuma-dp/pkg/dataplane/metrics/testdata/prom-text.in @@ -0,0 +1,111 @@ +# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} 6.1185e-05 +go_gc_duration_seconds{quantile="0.25"} 6.5599e-05 +go_gc_duration_seconds{quantile="0.5"} 0.000171854 +go_gc_duration_seconds{quantile="0.75"} 0.000200275 +go_gc_duration_seconds{quantile="1"} 0.000257027 +go_gc_duration_seconds_sum 0.000902316 +go_gc_duration_seconds_count 6 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 28 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{version="go1.19.2"} 1 +# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes 5.105112e+06 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total 1.2656064e+07 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes 6791 +# HELP go_memstats_frees_total Total number of frees. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total 67648 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes 9.831256e+06 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes 5.105112e+06 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes 1.0928128e+07 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes 8.732672e+06 +# HELP go_memstats_heap_objects Number of allocated objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects 30811 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes 7.241728e+06 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes 1.96608e+07 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds 1.6876005560881631e+09 +# HELP go_memstats_lookups_total Total number of pointer lookups. +# TYPE go_memstats_lookups_total counter +go_memstats_lookups_total 0 +# HELP go_memstats_mallocs_total Total number of mallocs. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total 98459 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes 19200 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes 31200 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes 202640 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes 244800 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes 1.038824e+07 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes 2.975761e+06 +# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes 1.31072e+06 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes 1.31072e+06 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes 3.4061328e+07 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads 21 +# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served. +# TYPE promhttp_metric_handler_requests_in_flight gauge +promhttp_metric_handler_requests_in_flight 1 +# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code. +# TYPE promhttp_metric_handler_requests_total counter +promhttp_metric_handler_requests_total{code="200"} 0 +promhttp_metric_handler_requests_total{code="500"} 0 +promhttp_metric_handler_requests_total{code="503"} 0 +# HELP sm_api_http_req_process_duration Duration of HTTP requests. +# TYPE sm_api_http_req_process_duration histogram +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="0.005"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="0.01"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="0.025"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="0.05"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="0.1"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="0.25"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="0.5"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="1"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="2.5"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="5"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="10"} 0 +sm_api_http_req_process_duration_bucket{method="GET",path="/api/metrics",route_name="Metrics",le="+Inf"} 0 +sm_api_http_req_process_duration_sum{method="GET",path="/api/metrics",route_name="Metrics"} 0 +sm_api_http_req_process_duration_count{method="GET",path="/api/metrics",route_name="Metrics"} 0