diff --git a/charts/langstream/grafana-dashboards/apps.json b/charts/langstream/grafana-dashboards/apps.json index 51fd5c7..467ac5a 100644 --- a/charts/langstream/grafana-dashboards/apps.json +++ b/charts/langstream/grafana-dashboards/apps.json @@ -1,1146 +1,2037 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "Apps related metrics", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 55, - "iteration": 1694590572005, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 109, - "panels": [], - "title": "Global", - "type": "row" - }, + "annotations": { + "list": [ { + "builtIn": 1, "datasource": { - "type": "prometheus", - "uid": "prometheus" + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "type": "dashboard" + } + ] + }, + "description": "Apps related metrics", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 131, + "panels": [], + "repeat": "agent", + "repeatDirection": "h", + "title": "OpenAI Comsumption", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 1 + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 132, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "id": 87, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (agent_id) (openai_embeddings_num_calls_total)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Http requests - {{agent_id}}", + "metric": "jvm_memory_bytes_committed", + "range": true, + "refId": "A", + "step": 20 }, - "pluginVersion": "8.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(kube_pod_status_ready{namespace=~\"ls-.*\", condition=\"true\"})", - "legendFormat": "Running", - "range": true, - "refId": "A" + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(kube_pod_status_ready{namespace=~\"ls-.*\", condition=\"false\"})", - "hide": false, - "legendFormat": "Failing", - "range": true, - "refId": "B" + "editorMode": "code", + "expr": "sum by (agent_id) (openai_embeddings_num_errors_total)", + "hide": false, + "instant": false, + "legendFormat": "Errors - {{agent_id}}", + "range": true, + "refId": "B" + } + ], + "title": "OpenAI Embeddings requests", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "editorMode": "code", - "expr": "sum(kube_pod_status_ready{namespace=~\"ls-.*\"})", - "hide": false, - "legendFormat": "Total", - "range": true, - "refId": "C" - } - ], - "title": "Executors ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 1 - }, - "id": 106, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(kube_pod_container_resource_limits{namespace=~\"ls-$tenant\", pod=~\"$application-$agent-.*\", resource=\"cpu\"}) by (namespace)", - "legendFormat": "{{namespace}}", - "range": true, - "refId": "A" - } - ], - "title": "CPU limits", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "thresholdsStyle": { - "mode": "off" + { + "color": "red", + "value": 80 } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" + ] }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 1 + "unit": "none" }, - "id": 107, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(kube_pod_container_resource_limits{namespace=~\"ls-$tenant\", pod=~\"$application-$agent-.*\", resource=\"memory\"}) by (namespace)", - "legendFormat": "{{namespace}}", - "range": true, - "refId": "A" - } - ], - "title": "Memory limits", - "type": "timeseries" + "overrides": [] }, - { - "collapsed": false, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 41, - "panels": [], - "repeat": "tenant", - "title": "Tenant: $tenant", - "type": "row" + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 1 }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "id": 133, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (agent_id) (openai_embeddings_num_texts_total)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Texts - {{agent_id}}", + "metric": "jvm_memory_bytes_committed", + "range": true, + "refId": "A", + "step": 20 + } + ], + "title": "OpenAI Embeddings Texts", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "red", - "value": 100 - } - ] + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" }, - "unit": "percent" + "thresholdsStyle": { + "mode": "off" + } }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 9 - }, - "id": 4, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "irate(process_cpu_seconds_total{namespace=~\"ls-$tenant\", pod=~\"$application-$agent-.*\"}[60s]) * 100\n\n", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "metric": "process_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 20 - } - ], - "title": "CPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "thresholdsStyle": { - "mode": "off" + { + "color": "red", + "value": 80 } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" + ] }, - "overrides": [] + "unit": "none" }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 9 + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 134, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "id": 1, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (agent_id) (openai_embeddings_prompt_tokens_total) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Prompt tokens - {{agent_id}}", + "metric": "jvm_memory_bytes_committed", + "range": true, + "refId": "A", + "step": 20 }, - "pluginVersion": "8.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (pod) (jvm_memory_bytes_used{namespace=~\"ls-$tenant\",pod=~\"$application-$agent-.*\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "Used - {{pod}}", - "metric": "jvm_memory_bytes_committed", - "range": true, - "refId": "A", - "step": 20 + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (pod) (jvm_memory_bytes_max{namespace=~\"ls-$tenant\",pod=~\"$application-$agent-.*\"})", - "hide": false, - "legendFormat": "Max: {{pod}}", - "range": true, - "refId": "B" + "editorMode": "code", + "expr": "sum by (agent_id) (openai_embeddings_total_tokens_total)", + "hide": false, + "instant": false, + "legendFormat": "Total tokens - {{agent_id}}", + "range": true, + "refId": "B" + } + ], + "title": "OpenAI Embeddings Tokens", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "editorMode": "code", - "expr": "sum by (pod) (jvm_memory_bytes_committed{namespace=~\"ls-$tenant\",pod=~\"$application-$agent-.*\"})", - "hide": false, - "legendFormat": "Committed: {{pod}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "editorMode": "code", - "expr": "kube_pod_container_resource_limits{namespace=~\"ls-$tenant\", pod=~\"$application-$agent-.*\", resource=\"memory\"}", - "hide": false, - "legendFormat": "Kube limit: {{pod}}", - "range": true, - "refId": "D" - } - ], - "title": "Heap Memory", - "transformations": [], - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "thresholdsStyle": { - "mode": "off" + { + "color": "red", + "value": 80 } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" + ] }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "old count" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.axisPlacement", - "value": "hidden" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "young count" - }, - "properties": [ - { - "id": "unit", - "value": "short" - }, - { - "id": "custom.axisPlacement", - "value": "hidden" - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 9 - }, - "id": 3, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "unit": "none" }, - "pluginVersion": "8.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "increase(jvm_gc_collection_seconds_sum{namespace=~\"ls-$tenant\",pod=~\"$application-$agent-.*\"}[60s])", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "metric": "jvm_gc_collection_seconds_count", - "range": true, - "refId": "B", - "step": 20 - } - ], - "title": "GC Pauses", - "type": "timeseries" + "overrides": [] }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 135, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - } - ] - }, - "unit": "string" + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 15 - }, - "id": 118, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "editorMode": "code", + "exemplar": false, + "expr": "sum by (agent_id) (openai_chat_completions_num_calls_total)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Http requests - {{agent_id}}", + "metric": "jvm_memory_bytes_committed", + "range": true, + "refId": "A", + "step": 20 }, - "pluginVersion": "8.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(kube_pod_status_ready{namespace=~\"ls-$tenant\", condition=\"true\", pod=~\"$application-$agent-.*\"}) by (pod)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "metric": "process_cpu_seconds_total", - "range": true, - "refId": "A", - "step": 20 - } - ], - "title": "Running executors", - "type": "timeseries" + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "code", + "expr": "sum by (agent_id) (openai_chat_completions_num_errors_total)", + "hide": false, + "instant": false, + "legendFormat": "Errors - {{agent_id}}", + "range": true, + "refId": "B" + } + ], + "title": "OpenAI Chat Completions requests", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - } - ] + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" }, - "unit": "string" + "thresholdsStyle": { + "mode": "off" + } }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 15 - }, - "id": 129, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(kube_pod_status_ready{namespace=~\"ls-$tenant\", condition=\"false\", pod=~\"$application-$agent-.*\"}) by (pod)", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "B" - } - ], - "title": "Failing executors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" + { + "color": "red", + "value": 80 } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - } - ] - }, - "unit": "string" + ] }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 15 + "unit": "none" }, - "id": 130, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 7 + }, + "id": 137, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "pluginVersion": "8.5.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "\nsum(label_replace(\n kube_pod_status_phase{namespace=~\"ls-$tenant\", phase=~\"Failed|Unknown\", pod=~\"langstream-runtime-deployer-.*\"},\n \"application1\",\"$1\",\"pod\", \"(.*)-(.*)\"\n )\n)by(application1)", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "B" - } - ], - "title": "Failed deployments", - "transformations": [], - "type": "timeseries" - } - ], - "refresh": "", - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [ + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ { - "allValue": "", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "jvm_memory_bytes_used{namespace=~\".+\"}", - "hide": 0, - "includeAll": true, - "label": "Tenant", - "multi": false, - "name": "tenant", - "options": [], - "query": { - "query": "jvm_memory_bytes_used{namespace=~\".+\"}", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "/.*[^_]namespace=\\\"ls-([^\\\"]+)\\\".*/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "editorMode": "code", + "exemplar": false, + "expr": "sum by (agent_id) (openai_chat_completions_num_calls_total)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Http requests - {{agent_id}}", + "metric": "jvm_memory_bytes_committed", + "range": true, + "refId": "A", + "step": 20 }, { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "P1809F7CD0C75ACF3" }, - "definition": "jvm_memory_bytes_used{namespace=~\"ls-($tenant|All)\",langstream_application=~\".+\"}", - "hide": 0, - "includeAll": true, - "label": "Application", - "multi": true, - "name": "application", - "options": [], - "query": { - "query": "jvm_memory_bytes_used{namespace=~\"ls-($tenant|All)\",langstream_application=~\".+\"}", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "/.*[^_]langstream_application=\\\"([^\\\"]+)\\\".*/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "editorMode": "code", + "expr": "sum by (agent_id) (openai_chat_completions_num_errors_total)", + "hide": false, + "instant": false, + "legendFormat": "Errors - {{agent_id}}", + "range": true, + "refId": "B" + } + ], + "title": "OpenAI Chat Completions requests", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 13 + }, + "id": 136, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (agent_id) (openai_text_completions_prompt_tokens_total) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Prompt tokens - {{agent_id}}", + "metric": "jvm_memory_bytes_committed", + "range": true, + "refId": "A", + "step": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "code", + "expr": "sum by (agent_id) (openai_text_completions_total_tokens_total)", + "hide": false, + "instant": false, + "legendFormat": "Completions tokens - {{agent_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "code", + "expr": "sum by (agent_id) (openai_text_completions_completions_tokens_total)", + "hide": false, + "instant": false, + "legendFormat": "Total tokens - {{agent_id}}", + "range": true, + "refId": "C" + } + ], + "title": "OpenAI Text Completions Tokens", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 13 + }, + "id": 138, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (agent_id) (openai_text_completions_num_calls_total)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Http requests - {{agent_id}}", + "metric": "jvm_memory_bytes_committed", + "range": true, + "refId": "A", + "step": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "code", + "expr": "sum by (agent_id) (openai_text_completions_num_errors_total)", + "hide": false, + "instant": false, + "legendFormat": "Errors - {{agent_id}}", + "range": true, + "refId": "B" + } + ], + "title": "OpenAI Text Completions requests", + "transformations": [], + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 41, + "panels": [], + "repeat": "tenant", + "title": "Tenant: $tenant", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 100 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 4, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(process_cpu_seconds_total{namespace=~\"ls-$tenant\", pod=~\"$application-$agent-.*\"}[60s]) * 100\n\n", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "metric": "process_cpu_seconds_total", + "range": true, + "refId": "A", + "step": 20 + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 1, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (pod) (jvm_memory_bytes_used{namespace=~\"ls-$tenant\",pod=~\"$application-$agent-.*\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Used - {{pod}}", + "metric": "jvm_memory_bytes_committed", + "range": true, + "refId": "A", + "step": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (pod) (jvm_memory_bytes_max{namespace=~\"ls-$tenant\",pod=~\"$application-$agent-.*\"})", + "hide": false, + "legendFormat": "Max: {{pod}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (pod) (jvm_memory_bytes_committed{namespace=~\"ls-$tenant\",pod=~\"$application-$agent-.*\"})", + "hide": false, + "legendFormat": "Committed: {{pod}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kube_pod_container_resource_limits{namespace=~\"ls-$tenant\", pod=~\"$application-$agent-.*\", resource=\"memory\"}", + "hide": false, + "legendFormat": "Kube limit: {{pod}}", + "range": true, + "refId": "D" + } + ], + "title": "Heap Memory", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "old count" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "hidden" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "young count" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "hidden" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 3, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "increase(jvm_gc_collection_seconds_sum{namespace=~\"ls-$tenant\",pod=~\"$application-$agent-.*\"}[60s])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "metric": "jvm_gc_collection_seconds_count", + "range": true, + "refId": "B", + "step": 20 + } + ], + "title": "GC Pauses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 26 + }, + "id": 118, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_status_ready{namespace=~\"ls-$tenant\", condition=\"true\", pod=~\"$application-$agent-.*\"}) by (pod)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "metric": "process_cpu_seconds_total", + "range": true, + "refId": "A", + "step": 20 + } + ], + "title": "Running executors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 26 + }, + "id": 129, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_status_ready{namespace=~\"ls-$tenant\", condition=\"false\", pod=~\"$application-$agent-.*\"}) by (pod)", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Failing executors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } ] }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 26 + }, + "id": 130, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "jvm_memory_bytes_used{namespace=~\"ls-($tenant|All)\",langstream_application=~\"($application|All)\",langstream_agent=~\".+\"}", - "hide": 0, - "includeAll": true, - "label": "Agent", - "multi": true, - "name": "agent", - "options": [], - "query": { - "query": "jvm_memory_bytes_used{namespace=~\"ls-($tenant|All)\",langstream_application=~\"($application|All)\",langstream_agent=~\".+\"}", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "/.*[^_]langstream_agent=\\\"([^\\\"]+)\\\".*/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "editorMode": "code", + "expr": "\nsum(label_replace(\n kube_pod_status_phase{namespace=~\"ls-$tenant\", phase=~\"Failed|Unknown\", pod=~\"langstream-runtime-deployer-.*\"},\n \"application1\",\"$1\",\"pod\", \"(.*)-(.*)\"\n )\n)by(application1)", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" } - ] + ], + "title": "Failed deployments", + "transformations": [], + "type": "timeseries" }, - "time": { - "from": "now-1h", - "to": "now" + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 109, + "panels": [], + "title": "Global", + "type": "row" }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 33 + }, + "id": 87, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_status_ready{namespace=~\"ls-.*\", condition=\"true\"})", + "legendFormat": "Running", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_status_ready{namespace=~\"ls-.*\", condition=\"false\"})", + "hide": false, + "legendFormat": "Failing", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_status_ready{namespace=~\"ls-.*\"})", + "hide": false, + "legendFormat": "Total", + "range": true, + "refId": "C" + } + ], + "title": "Executors ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 33 + }, + "id": 106, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_limits{namespace=~\"ls-$tenant\", pod=~\"$application-$agent-.*\", resource=\"cpu\"}) by (namespace)", + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] + "title": "CPU limits", + "type": "timeseries" }, - "timezone": "browser", - "title": "Langstream / Apps", - "uid": "rGuiWUzSk", - "version": 70, - "weekStart": "" - } \ No newline at end of file + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 33 + }, + "id": 107, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_limits{namespace=~\"ls-$tenant\", pod=~\"$application-$agent-.*\", resource=\"memory\"}) by (namespace)", + "legendFormat": "{{namespace}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory limits", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 38, + "tags": [], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "jvm_memory_bytes_used{namespace=~\".+\"}", + "hide": 0, + "includeAll": true, + "label": "Tenant", + "multi": false, + "name": "tenant", + "options": [], + "query": { + "query": "jvm_memory_bytes_used{namespace=~\".+\"}", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "/.*[^_]namespace=\\\"ls-([^\\\"]+)\\\".*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "jvm_memory_bytes_used{namespace=~\"ls-($tenant|All)\",langstream_application=~\".+\"}", + "hide": 0, + "includeAll": true, + "label": "Application", + "multi": true, + "name": "application", + "options": [], + "query": { + "query": "jvm_memory_bytes_used{namespace=~\"ls-($tenant|All)\",langstream_application=~\".+\"}", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "/.*[^_]langstream_application=\\\"([^\\\"]+)\\\".*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "jvm_memory_bytes_used{namespace=~\"ls-($tenant|All)\",langstream_application=~\"($application|All)\",langstream_agent=~\".+\"}", + "hide": 0, + "includeAll": true, + "label": "Agent", + "multi": true, + "name": "agent", + "options": [], + "query": { + "query": "jvm_memory_bytes_used{namespace=~\"ls-($tenant|All)\",langstream_application=~\"($application|All)\",langstream_agent=~\".+\"}", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "/.*[^_]langstream_agent=\\\"([^\\\"]+)\\\".*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Langstream / Apps", + "uid": "rGuiWUzSk", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/docker/prometheus.yml b/docker/prometheus.yml new file mode 100644 index 0000000..eae1307 --- /dev/null +++ b/docker/prometheus.yml @@ -0,0 +1,12 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'langstream' + static_configs: + - targets: ['host.docker.internal:8790'] + labels: + namespace: 'ls-docker' + pod: 'app-docker-local' + langstream_application: 'app' + langstream_agent: 'docker' diff --git a/docker/provisioning/dashboards/dashboards.yml b/docker/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..d80757c --- /dev/null +++ b/docker/provisioning/dashboards/dashboards.yml @@ -0,0 +1,14 @@ +apiVersion: 1 + +providers: + - name: 'langstream' + orgId: 1 + folder: '' + folderUid: '123123' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: true diff --git a/docker/provisioning/datasources/prometheus-datasources.yml b/docker/provisioning/datasources/prometheus-datasources.yml new file mode 100644 index 0000000..d6ea2b8 --- /dev/null +++ b/docker/provisioning/datasources/prometheus-datasources.yml @@ -0,0 +1,7 @@ +apiVersion: 1 +datasources: + - name: prometheus + type: prometheus + access: proxy + url: http://host.docker.internal:9090 + isDefault: true diff --git a/docker/run-local-grafana.sh b/docker/run-local-grafana.sh new file mode 100755 index 0000000..81b1ee1 --- /dev/null +++ b/docker/run-local-grafana.sh @@ -0,0 +1,13 @@ +#/bin/bash + +HERE=$(dirname $0) +docker rm -f prometheus +docker run -d -p 9090:9090 --name prometheus -v $HERE/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus +docker rm -f grafana + +docker run -d -p 3000:3000 --name=grafana \ + -e "GF_SECURITY_ADMIN_USER=admin" \ + -e "GF_SECURITY_ADMIN_PASSWORD=admin" \ + -v $HERE/provisioning:/etc/grafana/provisioning \ + -v $HERE/../charts/langstream/grafana-dashboards:/var/lib/grafana/dashboards \ + grafana/grafana