{ "__inputs": [ { "name": "DS_PROM", "label": "Your Prometheus data source", "description": "A Prometheus data source you use for ingesting all OTel collector metrics, such as a Prometheus server, Cortex, Thanos, or managed offerings like Amazon Managed Service for Prometheus.", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__elements": [], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "8.4.7" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "iteration": 1656668277256, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 11, "panels": [], "title": "Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "dark-green", "value": 80 } ] }, "unit": "dtdurations" }, "overrides": [ { "matcher": { "id": "byName", "options": "online" }, "properties": [ { "id": "unit", "value": "bool" } ] } ] }, "gridPos": { "h": 6, "w": 3, "x": 0, "y": 1 }, "id": 2, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "value" }, "pluginVersion": "8.4.7", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "up{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"$distro-collector-.*\"}", "hide": false, "interval": "", "legendFormat": "online", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "otelcol_process_uptime{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"$distro-collector-.*\"}", "hide": false, "interval": "", "legendFormat": "uptime", "refId": "A" } ], "title": "overall health", "transformations": [], "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepBefore", "lineStyle": { "fill": "solid" }, "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [ { "matcher": { "id": "byName", "options": "Total CPU user and system time in seconds" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Total CPU user and system time in seconds (adot)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Total CPU user and system time in seconds (otel)" }, "properties": [ { "id": "color", "value": { "fixedColor": "light-blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 6, "w": 5, "x": 3, "y": 1 }, "id": 6, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "7.5.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "rate(otelcol_process_cpu_seconds{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"adot-collector-.*\"}[5m])*100", "interval": "", "legendFormat": "Total CPU user and system time in seconds (adot)", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "rate(otelcol_process_cpu_seconds{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"otel-collector-.*\"}[5m])*100", "hide": false, "interval": "", "legendFormat": "Total CPU user and system time in seconds (otel)", "queryType": "randomWalk", "refId": "B" } ], "title": "[resource usage] CPU seconds", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepBefore", "lineStyle": { "fill": "solid" }, "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [ { "matcher": { "id": "byName", "options": "Total CPU user and system time in seconds" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Total CPU user and system time in seconds (adot)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Total CPU user and system time in seconds (otel)" }, "properties": [ { "id": "color", "value": { "fixedColor": "light-blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 6, "w": 5, "x": 8, "y": 1 }, "id": 24, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "7.5.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "rate(otelcol_process_cpu_seconds{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"$distro-collector-.*\"}[5m])*100", "interval": "", "legendFormat": "Total CPU user and system time in seconds ($distro)", "queryType": "randomWalk", "refId": "A" } ], "title": "[resource usage] CPU seconds", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "description": "Memory usage from operating system (OS) and Go runtime", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisGridShow": true, "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 22, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepBefore", "lineStyle": { "fill": "solid" }, "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "orange", "value": null } ] }, "unit": "deckbytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Total physical memory (adot)" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Total physical memory (otel)" }, "properties": [ { "id": "color", "value": { "fixedColor": "light-blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 6, "w": 5, "x": 13, "y": 1 }, "id": 7, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "7.5.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "rate(otelcol_process_memory_rss{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"adot-collector-.*\"}[5m])", "interval": "", "legendFormat": "Total physical memory (adot)", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "rate(otelcol_process_memory_rss{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"otel-collector-.*\"}[5m])", "hide": false, "interval": "", "legendFormat": "Total physical memory (otel)", "queryType": "randomWalk", "refId": "B" } ], "title": "[resource usage] Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "description": "Memory usage from operating system (OS) and Go runtime", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisGridShow": true, "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 22, "gradientMode": "opacity", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepBefore", "lineStyle": { "fill": "solid" }, "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "orange", "value": null } ] }, "unit": "deckbytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Cumulative bytes allocated for heap objects (adot)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Total bytes of memory obtained from OS (adot)" }, "properties": [ { "id": "color", "value": { "fixedColor": "super-light-orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Cumulative bytes allocated for heap objects (otel)" }, "properties": [ { "id": "color", "value": { "fixedColor": "dark-blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Total bytes of memory obtained from OS (otel)" }, "properties": [ { "id": "color", "value": { "fixedColor": "super-light-blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 1 }, "id": 25, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "list", "placement": "right" }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "7.5.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "rate(otelcol_process_runtime_total_alloc_bytes{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"$distro-collector-.*\"}[5m])", "hide": false, "interval": "", "legendFormat": "Cumulative bytes allocated for heap objects ($distro)", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "rate(otelcol_process_runtime_total_sys_memory_bytes{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"$distro-collector-.*\"}[5m])", "hide": false, "interval": "", "legendFormat": "Total bytes of memory obtained from OS ($distro)", "refId": "C" } ], "title": "[resource usage] Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [ { "matcher": { "id": "byName", "options": "overall CPU" }, "properties": [ { "id": "color", "value": { "fixedColor": "#000000", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 6, "w": 5, "x": 0, "y": 7 }, "id": 27, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "sum(rate(system_cpu_time{state=\"user\"}[5m])) + sum(rate(system_cpu_time{state=\"system\"}[5m]))", "interval": "", "legendFormat": "overall CPU", "refId": "A" } ], "title": "[hostmetrics] CPU", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "overall CPU (user)" }, "properties": [ { "id": "color", "value": { "fixedColor": "#000000", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 6, "w": 4, "x": 5, "y": 7 }, "id": 28, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "system_memory_usage{state=\"free\"}", "interval": "", "legendFormat": "overall CPU (user)", "refId": "A" } ], "title": "[hostmetrics] Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "overall I/O" }, "properties": [ { "id": "color", "value": { "fixedColor": "#000000", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 6, "w": 5, "x": 9, "y": 7 }, "id": 29, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "sum(rate(system_network_io[5m]))", "interval": "", "legendFormat": "overall I/O", "refId": "A" } ], "title": "[hostmetrics] Network", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "overall I/O" }, "properties": [ { "id": "color", "value": { "fixedColor": "#000000", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 6, "w": 5, "x": 14, "y": 7 }, "id": 30, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "sum(rate(system_disk_io[5m]))", "interval": "", "legendFormat": "overall I/O", "refId": "A" } ], "title": "[hostmetrics] Disk", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "overall I/O" }, "properties": [ { "id": "color", "value": { "fixedColor": "#000000", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "system_processes_count{status=\"running\"}" }, "properties": [ { "id": "color", "value": { "fixedColor": "#000000", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 6, "w": 5, "x": 19, "y": 7 }, "id": 31, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "8.4.7", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": true, "expr": "system_processes_count{status=\"running\"}", "hide": false, "interval": "", "legendFormat": "", "refId": "B" } ], "title": "[hostmetrics] Filesystem", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, "id": 13, "panels": [], "title": "Receivers", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepBefore", "lineStyle": { "fill": "solid" }, "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 14 }, "id": 33, "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": true, "expr": "sum without(instance,job,k8s_namespace) (scrape_series_added{k8s_namespace=\"$k8s_namespace\"})", "interval": "", "legendFormat": "series added from {{k8s_pod}}", "refId": "A" } ], "title": "scrape stats", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 4, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "prometheus" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] }, { "matcher": { "id": "byName", "options": "prometheus" }, "properties": [ { "id": "color", "value": { "fixedColor": "light-purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 14 }, "id": 9, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "list", "placement": "right" }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "7.5.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "rate(otelcol_receiver_accepted_metric_points{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"$distro-collector-.*\"}[5m])", "interval": "", "legendFormat": "{{receiver}}", "refId": "A" } ], "title": "metrics points received", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "description": "Number of metric points that could not be pushed into the pipeline\n", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 4, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "prometheus" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] }, { "matcher": { "id": "byName", "options": "prometheus" }, "properties": [ { "id": "color", "value": { "fixedColor": "light-purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 14 }, "id": 23, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "list", "placement": "right" }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "7.5.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "rate(otelcol_receiver_refused_metric_points{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"$distro-collector-.*\"}[5m])", "interval": "", "legendFormat": "{{receiver}}", "refId": "A" } ], "title": "metrics points receive failed", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, "id": 15, "panels": [], "title": "Processors", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "description": "number of units in batch", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "p90" }, "properties": [ { "id": "color", "value": { "fixedColor": "light-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "p50" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "p50 for batch/metrics " }, "properties": [ { "id": "color", "value": { "fixedColor": "light-blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "p90 for batch/metrics " }, "properties": [ { "id": "color", "value": { "fixedColor": "light-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 22 }, "id": 19, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right" }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "histogram_quantile(0.5, sum(rate(otelcol_processor_batch_batch_send_size_bucket{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"$distro-collector-.*\"}[5m])) by (le,processor))", "interval": "", "legendFormat": "p50 for {{processor}} ", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "histogram_quantile(0.9, sum(rate(otelcol_processor_batch_batch_send_size_bucket{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"$distro-collector-.*\"}[5m])) by (le,processor))", "hide": false, "interval": "", "legendFormat": "p90 for {{processor}} ", "refId": "B" } ], "title": "send size performance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "description": "number of bytes in batch that was sent", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepBefore", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "deckbytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "p90" }, "properties": [ { "id": "color", "value": { "fixedColor": "light-red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "p50" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "p50 for batch/metrics " }, "properties": [ { "id": "color", "value": { "fixedColor": "light-blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "p90 for batch/metrics " }, "properties": [ { "id": "color", "value": { "fixedColor": "light-red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 22 }, "id": 20, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right" }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "histogram_quantile(0.5, sum(rate(otelcol_processor_batch_batch_send_size_bytes_bucket{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"$distro-collector-.*\"}[5m])) by (le,processor))", "interval": "", "legendFormat": "p50 for {{processor}} ", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "histogram_quantile(0.9, sum(rate(otelcol_processor_batch_batch_send_size_bytes_bucket{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"$distro-collector-.*\"}[5m])) by (le,processor))", "hide": false, "interval": "", "legendFormat": "p90 for {{processor}} ", "refId": "B" } ], "title": "send size memory performance", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 29 }, "id": 17, "panels": [], "title": "Exporters", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "description": "current size of the retry queue (in batches)\n", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 4, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 30 }, "id": 8, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "hidden", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "7.5.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "otelcol_exporter_queue_size{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"$distro-collector-.*\"}", "interval": "", "legendFormat": "", "refId": "A" } ], "title": "exporter queue size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "description": "number of metric points successfully sent to destination\n", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepBefore", "lineWidth": 1, "pointSize": 4, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "awsprometheusremotewrite" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-purple", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "logging" }, "properties": [ { "id": "color", "value": { "fixedColor": "super-light-yellow", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 7, "w": 9, "x": 6, "y": 30 }, "id": 21, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "7.5.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "rate(otelcol_exporter_sent_metric_points{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"$distro-collector-.*\"}[5m])", "interval": "", "legendFormat": "{{exporter}}", "refId": "A" } ], "title": "metrics points sent", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "description": "number of metric points failed to be added to the sending queue\n", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepBefore", "lineWidth": 1, "pointSize": 4, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "awsprometheusremotewrite" }, "properties": [ { "id": "color", "value": { "fixedColor": "semi-dark-purple", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "logging" }, "properties": [ { "id": "color", "value": { "fixedColor": "super-light-yellow", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 7, "w": 9, "x": 15, "y": 30 }, "id": 22, "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "7.5.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROM}" }, "exemplar": false, "expr": "rate(otelcol_exporter_enqueue_failed_metric_points{k8s_namespace=\"$k8s_namespace\",k8s_pod=~\"$distro-collector-.*\"}[5m])", "interval": "", "legendFormat": "{{exporter}}", "refId": "A" } ], "title": "metrics points send fails", "type": "timeseries" } ], "refresh": "", "schemaVersion": 35, "style": "dark", "tags": [], "templating": { "list": [ { "current": { "selected": false, "text": "oto", "value": "oto" }, "hide": 0, "label": "Kubernetes namespace", "name": "k8s_namespace", "options": [ { "selected": true, "text": "", "value": "" } ], "query": "oto", "skipUrlSync": false, "type": "textbox" }, { "current": { "selected": true, "text": "otel", "value": "otel" }, "hide": 0, "includeAll": false, "multi": false, "name": "distro", "options": [ { "selected": false, "text": "adot", "value": "adot" }, { "selected": true, "text": "otel", "value": "otel" } ], "query": "adot,otel", "queryValue": "", "skipUrlSync": false, "type": "custom" } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "OpenTelemetry Collector", "uid": "rwMn-Ni7k", "version": 38, "weekStart": "" }