{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "description": "Provides information about the status of the OpenTelemetry Collector", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": 12, "links": [], "liveNow": true, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 55, "panels": [], "title": "Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "OTel Collectors. Derived from the `otelcol.process.uptime` metric.", "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "footer": { "reducers": [] }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Uptime" }, "properties": [ { "id": "unit", "value": "dtdurations" }, { "id": "custom.width", "value": 84 } ] }, { "matcher": { "id": "byName", "options": "Namespace" }, "properties": [ { "id": "custom.width", "value": 110 } ] }, { "matcher": { "id": "byName", "options": "Version" }, "properties": [ { "id": "custom.width", "value": 85 } ] } ] }, "gridPos": { "h": 4, "w": 17, "x": 0, "y": 1 }, "id": 57, "options": { "cellHeight": "sm", "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Service Name" } ] }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "otelcol_process_uptime_seconds_total{deployment_environment_name=~\"$deployment_environment_name\", k8s_cluster_name=~\"$k8s_cluster_name\", k8s_node_name=~\"$k8s_node_name\", host_name=~\"$host_name\", service_instance_id=~\"$service_instance_id\", service_name=~\"$service_name\"}", "format": "table", "instant": true, "legendFormat": "{{service_name}} {{service_instance_id}}", "queryType": "instant", "range": false, "refId": "A" } ], "title": "", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "Value": false, "__name__": true, "instance": true, "job": true }, "includeByName": {}, "indexByName": { "Time": 2, "Value": 12, "__name__": 3, "host_name": 4, "instance": 5, "job": 6, "k8s_cluster_name": 7, "k8s_namespace_name": 8, "k8s_node_name": 9, "k8s_pod_name": 10, "service_instance_id": 11, "service_name": 0, "service_version": 1 }, "orderByMode": "manual", "renameByName": { "Value": "Uptime", "host_name": "Host", "k8s_cluster_name": "K8s cluster", "k8s_namespace_name": "Namespace", "k8s_node_name": "Node", "k8s_pod_name": "Pod", "service_instance_id": "Service Instance Id", "service_name": "Service Name", "service_version": "Version" } } } ], "type": "table" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 17, "y": 1 }, "id": 56, "options": { "alertInstanceLabelFilter": "deployment_environment_name=~\"$deployment_environment_name\", k8s_cluster_name=~\"$k8s_cluster_name\", k8s_node_name=~\"$k8s_node_name\", host_name=~\"$host_name\", service_instance_id=~\"$service_instance_id\", service_name=~\"$service_name\"", "alertName": "OpenTelemetryCollector", "dashboardAlerts": false, "groupBy": [], "groupMode": "default", "maxItems": 17, "showInactiveAlerts": true, "sortOrder": 1, "stateFilter": { "error": true, "firing": true, "noData": true, "normal": false, "pending": true, "recovering": true }, "viewMode": "list" }, "pluginVersion": "12.3.1", "title": "Alerts", "type": "alertlist" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 0, "y": 5 }, "id": 63, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "Receivers", "mode": "markdown" }, "pluginVersion": "12.3.1", "title": "", "type": "text" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "Span Rate" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 2, "y": 5 }, "id": 64, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum(rate(otelcol_receiver_accepted_spans_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))\n+\nsum(rate(otelcol_receiver_refused_spans_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))", "instant": false, "legendFormat": "Span Rate", "queryType": "range", "range": true, "refId": "spans_rate" } ], "title": "Spans", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Span Errors" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 5, "y": 5 }, "id": 65, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "1\n- \nsum(\n rate(\n otelcol_receiver_accepted_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n)\n/\n(\n sum(\n rate(\n otelcol_receiver_accepted_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_receiver_refused_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_receiver_failed_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n)", "instant": false, "legendFormat": "Span Errors", "queryType": "range", "range": true, "refId": "span_errors" } ], "title": "Span Errors", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Aggregate ingestion rate across all receivers by telemetry type", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 7, "y": 5 }, "id": 66, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum(rate(otelcol_receiver_accepted_log_records_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))\n+\nsum(rate(otelcol_receiver_refused_log_records_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))", "instant": false, "legendFormat": "Logs", "queryType": "range", "range": true, "refId": "logs_rate" } ], "title": "Logs", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 10, "y": 5 }, "id": 67, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "1\n-\nsum(\n rate(\n otelcol_receiver_accepted_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n)\n/\n(\n sum(\n rate(\n otelcol_receiver_accepted_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_receiver_refused_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_receiver_failed_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n)", "instant": false, "legendFormat": "Logs", "queryType": "range", "range": true, "refId": "logs_error" } ], "title": "Log Errors", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Aggregate ingestion rate across all receivers by telemetry type", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 12, "y": 5 }, "id": 68, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum(rate(otelcol_receiver_accepted_metric_points_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))\n+\nsum(rate(otelcol_receiver_refused_metric_points_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))", "instant": false, "legendFormat": "Metrics", "queryType": "range", "range": true, "refId": "metrics_rate" } ], "title": "Metrics", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 15, "y": 5 }, "id": 69, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "1\n-\nsum(\n rate(\n otelcol_receiver_accepted_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n)\n/\n(\n sum(\n rate(\n otelcol_receiver_accepted_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_receiver_refused_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_receiver_failed_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n)", "instant": false, "legendFormat": "Metrics", "queryType": "range", "range": true, "refId": "metrics_errors" } ], "title": "Metric Errors", "type": "gauge" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 0, "y": 8 }, "id": 80, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "Processors", "mode": "markdown" }, "pluginVersion": "12.3.1", "title": "", "type": "text" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Span processors inbound rate, aggregated across all processors", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "traces" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 2, "y": 8 }, "id": 88, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"traces\"\n }[$__rate_interval]\n )\n)", "instant": false, "legendFormat": "__auto", "queryType": "range", "range": true, "refId": "rate" } ], "title": "Spans", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Drop Rate" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 5, "y": 8 }, "id": 89, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "(\n sum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"traces\"\n }[$__rate_interval]\n )\n )\n-\n sum by (otel_signal) (\n rate(\n otelcol_processor_outgoing_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"traces\"\n }[$__rate_interval]\n )\n )\n)\n/\nsum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"traces\"\n }[$__rate_interval]\n )\n)", "instant": false, "legendFormat": "Drop Rate", "queryType": "range", "range": true, "refId": "drop_rate" } ], "title": "Drop Rate", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Logs processors inbound rate, aggregated across all processors", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 7, "y": 8 }, "id": 90, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"logs\"\n }[$__rate_interval]\n )\n)", "instant": false, "legendFormat": "__auto", "queryType": "range", "range": true, "refId": "rate" } ], "title": "Logs", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Drop Rate" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 10, "y": 8 }, "id": 91, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "(\n sum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"logs\"\n }[$__rate_interval]\n )\n )\n-\n sum by (otel_signal) (\n rate(\n otelcol_processor_outgoing_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"logs\"\n }[$__rate_interval]\n )\n )\n)\n/\nsum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"logs\"\n }[$__rate_interval]\n )\n)", "instant": false, "legendFormat": "Drop Rate", "queryType": "range", "range": true, "refId": "drop_rate" } ], "title": "Drop Rate", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Metrics processors inbound rate, aggregated across all processors", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 12, "y": 8 }, "id": 92, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"metrics\"\n }[$__rate_interval]\n )\n)", "instant": false, "legendFormat": "__auto", "queryType": "range", "range": true, "refId": "rate" } ], "title": "Metrics", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Drop Rate" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 15, "y": 8 }, "id": 93, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "(\n sum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"metrics\"\n }[$__rate_interval]\n )\n )\n-\n sum by (otel_signal) (\n rate(\n otelcol_processor_outgoing_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"metrics\"\n }[$__rate_interval]\n )\n )\n)\n/\nsum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"metrics\"\n }[$__rate_interval]\n )\n)", "instant": false, "legendFormat": "Drop Rate", "queryType": "range", "range": true, "refId": "drop_rate" } ], "title": "Drop Rate", "type": "gauge" }, { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 0, "y": 11 }, "id": 87, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "Exporters", "mode": "markdown" }, "pluginVersion": "12.3.1", "title": "", "type": "text" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Aggregated spans export rate across all exporters", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 2, "y": 11 }, "id": 74, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum(rate(otelcol_exporter_sent_spans_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))", "instant": false, "legendFormat": "Spans", "queryType": "range", "range": true, "refId": "A" } ], "title": "Spans", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Span Errors" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 5, "y": 11 }, "id": 82, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "1\n- \nsum(\n rate(\n otelcol_receiver_accepted_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n)\n/\n(\n sum(\n rate(\n otelcol_receiver_accepted_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_receiver_refused_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_receiver_failed_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n)", "instant": false, "legendFormat": "Span Errors", "queryType": "range", "range": true, "refId": "span_errors" } ], "title": "Span Errors", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Aggregated logs export rate across all exporters", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 7, "y": 11 }, "id": 76, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum(rate(otelcol_exporter_sent_log_records_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))", "instant": false, "legendFormat": "Logs", "queryType": "range", "range": true, "refId": "B" } ], "title": "Logs", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 10, "y": 11 }, "id": 77, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "1\n-\nsum(\n rate(\n otelcol_exporter_sent_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n)\n/\n(\n sum(\n rate(\n otelcol_exporter_sent_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_exporter_send_failed_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n)", "instant": false, "legendFormat": "Logs", "queryType": "range", "range": true, "refId": "logs_error" } ], "title": "Logs Errors", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Aggregate metrics export rate across all exporters", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 12, "y": 11 }, "id": 78, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum(rate(otelcol_exporter_sent_metric_points_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))", "instant": false, "legendFormat": "Metrics", "queryType": "range", "range": true, "refId": "C" } ], "title": "Metrics", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 15, "y": 11 }, "id": 79, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "1\n-\nsum(\n rate(\n otelcol_exporter_sent_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n)\n/\n(\n sum(\n rate(\n otelcol_exporter_sent_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_exporter_send_failed_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n)", "instant": false, "legendFormat": "Metrics", "queryType": "range", "range": true, "refId": "metrics_errors" } ], "title": "Metrics Errors", "type": "gauge" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "id": 53, "panels": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "OTel Collectors. Derived from the `otelcol.process.uptime` metric.", "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "footer": { "reducers": [] }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Uptime" }, "properties": [ { "id": "unit", "value": "dtdurations" } ] }, { "matcher": { "id": "byName", "options": "Namespace" }, "properties": [ { "id": "custom.width", "value": 110 } ] }, { "matcher": { "id": "byName", "options": "Version" }, "properties": [ { "id": "custom.width", "value": 85 } ] } ] }, "gridPos": { "h": 3, "w": 24, "x": 0, "y": 16 }, "id": 54, "options": { "cellHeight": "sm", "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Service Name" } ] }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "otelcol_process_uptime_seconds_total{deployment_environment_name=~\"$deployment_environment_name\", k8s_cluster_name=~\"$k8s_cluster_name\", k8s_node_name=~\"$k8s_node_name\", host_name=~\"$host_name\", service_instance_id=~\"$service_instance_id\", service_name=~\"$service_name\"}", "format": "table", "instant": true, "legendFormat": "__auto", "queryType": "instant", "range": false, "refId": "A" } ], "title": "", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "Value": false, "__name__": true, "instance": true, "job": true }, "includeByName": {}, "indexByName": { "Time": 2, "Value": 12, "__name__": 3, "host_name": 4, "instance": 5, "job": 6, "k8s_cluster_name": 7, "k8s_namespace_name": 8, "k8s_node_name": 9, "k8s_pod_name": 10, "service_instance_id": 11, "service_name": 0, "service_version": 1 }, "orderByMode": "manual", "renameByName": { "Value": "Uptime", "host_name": "Host", "k8s_cluster_name": "K8s cluster", "k8s_namespace_name": "Namespace", "k8s_node_name": "Node", "k8s_pod_name": "Pod", "service_instance_id": "Service Instance Id", "service_name": "Service Name", "service_version": "Version" } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "CPU consumption rate of the OpenTelemetry collector process", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 0, "y": 39 }, "id": 50, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "rate(otelcol_process_cpu_seconds_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval])", "instant": false, "legendFormat": "{{service_name}} {{service_instance_id}}", "queryType": "range", "range": true, "refId": "A" } ], "title": "CPU Usage", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Resident memory consumed by the OpenTelemetry collector process", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 3, "w": 5, "x": 5, "y": 39 }, "id": 52, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "otelcol_process_memory_rss_bytes{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}", "instant": false, "legendFormat": "{{service_name}} {{service_instance_id}}", "queryType": "range", "range": true, "refId": "A" } ], "title": "Memory Usage (RSS)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Total uptime of the OpenTelemetry collector process", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": 0 }, { "color": "yellow", "value": 300 }, { "color": "green", "value": 3600 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 3, "w": 2, "x": 10, "y": 39 }, "id": 51, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "otelcol_process_uptime_seconds_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}", "instant": false, "legendFormat": "{{service_name}} {{service_instance_id}}", "queryType": "range", "range": true, "refId": "A" } ], "title": "Uptime", "type": "stat" } ], "title": "Collector", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }, "id": 23, "panels": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "Span Rate" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 0, "y": 15 }, "id": 39, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum(rate(otelcol_receiver_accepted_spans_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))\n+\nsum(rate(otelcol_receiver_refused_spans_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))", "instant": false, "legendFormat": "Span Rate", "queryType": "range", "range": true, "refId": "spans_rate" } ], "title": "Span Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Span Errors" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 3, "y": 15 }, "id": 40, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "1\n- \nsum(\n rate(\n otelcol_receiver_accepted_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n)\n/\n(\n sum(\n rate(\n otelcol_receiver_accepted_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_receiver_refused_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_receiver_failed_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n)", "instant": false, "legendFormat": "Span Errors", "queryType": "range", "range": true, "refId": "span_errors" } ], "title": "Span Errors", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Aggregate ingestion rate across all receivers by telemetry type", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 5, "y": 15 }, "id": 59, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum(rate(otelcol_receiver_accepted_log_records_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))\n+\nsum(rate(otelcol_receiver_refused_log_records_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))", "instant": false, "legendFormat": "Logs", "queryType": "range", "range": true, "refId": "logs_rate" } ], "title": "Logs", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 8, "y": 15 }, "id": 61, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "1\n-\nsum(\n rate(\n otelcol_receiver_accepted_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n)\n/\n(\n sum(\n rate(\n otelcol_receiver_accepted_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_receiver_refused_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_receiver_failed_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n)", "instant": false, "legendFormat": "Logs", "queryType": "range", "range": true, "refId": "logs_error" } ], "title": "Log Errors", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Aggregate ingestion rate across all receivers by telemetry type", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 10, "y": 15 }, "id": 60, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum(rate(otelcol_receiver_accepted_metric_points_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))\n+\nsum(rate(otelcol_receiver_refused_metric_points_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))", "instant": false, "legendFormat": "Metrics", "queryType": "range", "range": true, "refId": "metrics_rate" } ], "title": "Metrics", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 13, "y": 15 }, "id": 62, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "1\n-\nsum(\n rate(\n otelcol_receiver_accepted_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n)\n/\n(\n sum(\n rate(\n otelcol_receiver_accepted_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_receiver_refused_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_receiver_failed_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n)", "instant": false, "legendFormat": "Metrics", "queryType": "range", "range": true, "refId": "metrics_errors" } ], "title": "Metric Errors", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Detailed breakdown by receiver and telemetry type", "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": false, "footer": { "reducers": [] }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*Rate.*" }, "properties": [ { "id": "custom.cellOptions", "value": { "type": "sparkline" } }, { "id": "unit", "value": "cps" } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\%" }, "properties": [ { "id": "unit", "value": "percentunit" } ] } ] }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 54 }, "id": 41, "options": { "cellHeight": "sm", "showHeader": true }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (signal_type_receiver) (\n label_join(\n label_replace(\n sum by (receiver) (\n rate(\n otelcol_receiver_accepted_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_receiver_refused_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n ),\n \"signal_type\",\n \"traces\",\n \"\",\n \"\"\n )\n or\n label_replace(\n sum by (receiver) (\n rate(\n otelcol_receiver_accepted_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_receiver_refused_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n ),\n \"signal_type\",\n \"logs\",\n \"\",\n \"\"\n )\n or\n label_replace(\n sum by (receiver) (\n rate(\n otelcol_receiver_accepted_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_receiver_refused_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n ),\n \"signal_type\",\n \"metrics\",\n \"\",\n \"\"\n )\n ,\n \"signal_type_receiver\",\n \"-\",\n \"signal_type\",\n \"receiver\"\n )\n)", "instant": false, "legendFormat": "__auto", "queryType": "range", "range": true, "refId": "accepted_rate" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (signal_type_receiver) (\n label_join(\n label_replace(\n sum by (receiver) (\n rate(\n otelcol_receiver_refused_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n ),\n \"signal_type\",\n \"traces\",\n \"\",\n \"\"\n )\n or\n label_replace(\n sum by (receiver) (\n rate(\n otelcol_receiver_refused_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n ),\n \"signal_type\",\n \"logs\",\n \"\",\n \"\"\n )\n or\n label_replace(\n sum by (receiver) (\n rate(\n otelcol_receiver_refused_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n ),\n \"signal_type\",\n \"metrics\",\n \"\",\n \"\"\n )\n ,\n \"signal_type_receiver\",\n \"-\",\n \"signal_type\",\n \"receiver\"\n )\n)", "instant": false, "legendFormat": "__auto", "queryType": "range", "range": true, "refId": "refused_rate" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (signal_type_receiver) (\n label_join(\n label_replace(\n sum by (receiver) (\n rate(\n otelcol_receiver_failed_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n ),\n \"signal_type\",\n \"traces\",\n \"\",\n \"\"\n )\n or\n label_replace(\n sum by (receiver) (\n rate(\n otelcol_receiver_failed_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n ),\n \"signal_type\",\n \"logs\",\n \"\",\n \"\"\n )\n or\n label_replace(\n sum by (receiver) (\n rate(\n otelcol_receiver_failed_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n ),\n \"signal_type\",\n \"metrics\",\n \"\",\n \"\"\n )\n ,\n \"signal_type_receiver\",\n \"-\",\n \"signal_type\",\n \"receiver\"\n )\n)", "instant": false, "legendFormat": "__auto", "queryType": "range", "range": true, "refId": "failed_rate" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (signal_type_receiver) (\n label_join(\n label_replace(\n (\n sum by (receiver) (\n rate(\n otelcol_receiver_refused_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_receiver_failed_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n /\n sum by (receiver) (\n rate(\n otelcol_receiver_accepted_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_receiver_refused_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_receiver_failed_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n ) * 100,\n \"signal_type\",\n \"traces\",\n \"\",\n \"\"\n )\n or\n label_replace(\n (\n sum by (receiver) (\n rate(\n otelcol_receiver_refused_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_receiver_failed_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n /\n sum by (receiver) (\n rate(\n otelcol_receiver_accepted_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_receiver_refused_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_receiver_failed_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n ) * 100,\n \"signal_type\",\n \"logs\",\n \"\",\n \"\"\n )\n or\n label_replace(\n (\n sum by (receiver) (\n rate(\n otelcol_receiver_refused_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_receiver_failed_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n /\n sum by (receiver) (\n rate(\n otelcol_receiver_accepted_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_receiver_refused_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_receiver_failed_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n ) * 100,\n \"signal_type\",\n \"metrics\",\n \"\",\n \"\"\n )\n ,\n \"signal_type_receiver\",\n \"-\",\n \"signal_type\",\n \"receiver\"\n )\n)", "instant": false, "legendFormat": "__auto", "queryType": "range", "range": true, "refId": "error_pct" } ], "title": "Receivers", "transformations": [ { "id": "timeSeriesTable", "options": { "accepted_rate": { "timeField": "Time" }, "error_pct": { "timeField": "Time" }, "errors": { "timeField": "Time" }, "failed_pct": { "timeField": "Time" }, "failed_rate": { "timeField": "Time" }, "refused_rate": { "timeField": "Time" }, "rps": { "timeField": "Time" }, "spans_errors": { "timeField": "Time" }, "spans_rps": { "timeField": "Time" } } }, { "id": "joinByField", "options": { "byField": "signal_type_receiver", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": { "Time": true }, "includeByName": {}, "indexByName": { "Trend #accepted_rate": 1, "Trend #failed_pct": 4, "Trend #failed_rate": 3, "Trend #refused_rate": 2, "signal_type_receiver": 0 }, "renameByName": { "Trend #accepted_rate": "Accepted Rate", "Trend #error_pct": "Error %", "Trend #errors": "Error %", "Trend #failed_pct": "Error %", "Trend #failed_rate": "Failed Rate", "Trend #refused_rate": "Refused Rate", "Trend #rps": "Accepted Rate", "Trend #spans_errors": "Error", "Trend #spans_rps": "Rate", "Value #A": "Spans Rate", "Value #B": "Spans Failure %", "Value #C": "Logs Rate", "Value #D": "Logs Failure %", "Value #E": "Metrics Rate", "Value #F": "Metrics Failure %", "receiver": "Receiver", "signal_type_receiver": "Receiver" } } } ], "type": "table" } ], "title": "Receivers", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 }, "id": 48, "panels": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Span processors inbound rate, aggregated across all processors", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "traces" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 0, "y": 95 }, "id": 81, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"traces\"\n }[$__rate_interval]\n )\n)", "instant": false, "legendFormat": "__auto", "queryType": "range", "range": true, "refId": "rate" } ], "title": "Spans", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Drop Rate" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 3, "y": 95 }, "id": 75, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "(\n sum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"traces\"\n }[$__rate_interval]\n )\n )\n-\n sum by (otel_signal) (\n rate(\n otelcol_processor_outgoing_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"traces\"\n }[$__rate_interval]\n )\n )\n)\n/\nsum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"traces\"\n }[$__rate_interval]\n )\n)", "instant": false, "legendFormat": "Drop Rate", "queryType": "range", "range": true, "refId": "drop_rate" } ], "title": "Drop Rate", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Logs processors inbound rate, aggregated across all processors", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 5, "y": 95 }, "id": 83, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"logs\"\n }[$__rate_interval]\n )\n)", "instant": false, "legendFormat": "__auto", "queryType": "range", "range": true, "refId": "rate" } ], "title": "Logs", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Drop Rate" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 8, "y": 95 }, "id": 84, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "(\n sum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"logs\"\n }[$__rate_interval]\n )\n )\n-\n sum by (otel_signal) (\n rate(\n otelcol_processor_outgoing_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"logs\"\n }[$__rate_interval]\n )\n )\n)\n/\nsum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"logs\"\n }[$__rate_interval]\n )\n)", "instant": false, "legendFormat": "Drop Rate", "queryType": "range", "range": true, "refId": "drop_rate" } ], "title": "Drop Rate", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Metrics processors inbound rate, aggregated across all processors", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 10, "y": 95 }, "id": 85, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"metrics\"\n }[$__rate_interval]\n )\n)", "instant": false, "legendFormat": "__auto", "queryType": "range", "range": true, "refId": "rate" } ], "title": "Metrics", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Drop Rate" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 13, "y": 95 }, "id": 86, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "(\n sum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"metrics\"\n }[$__rate_interval]\n )\n )\n-\n sum by (otel_signal) (\n rate(\n otelcol_processor_outgoing_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"metrics\"\n }[$__rate_interval]\n )\n )\n)\n/\nsum by (otel_signal) (\n rate(\n otelcol_processor_incoming_items_total{\n deployment_environment_name =~ \"$deployment_environment_name\",\n k8s_cluster_name =~ \"$k8s_cluster_name\",\n k8s_node_name =~ \"$k8s_node_name\",\n host_name =~ \"$host_name\",\n service_instance_id =~ \"$service_instance_id\",\n service_name =~ \"$service_name\",\n otel_signal = \"metrics\"\n }[$__rate_interval]\n )\n)", "instant": false, "legendFormat": "Drop Rate", "queryType": "range", "range": true, "refId": "drop_rate" } ], "title": "Drop Rate", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "footer": { "reducers": [] }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*Rate.*" }, "properties": [ { "id": "unit", "value": "cps" } ] }, { "matcher": { "id": "byRegexp", "options": ".*\\%" }, "properties": [ { "id": "unit", "value": "percentunit" }, { "id": "decimals", "value": 1 } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 121 }, "id": 49, "options": { "cellHeight": "sm", "frameIndex": 0, "showHeader": true }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (otel_signal_processor)(\n label_join(\n (\n sum by(processor, otel_signal) (\n rate(otelcol_processor_outgoing_items_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval])\n )\n -\n sum by(processor, otel_signal) (\n rate(otelcol_processor_incoming_items_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval])\n )\n )\n /\n sum by(processor, otel_signal) (\n rate(otelcol_processor_incoming_items_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval])\n ),\n \"otel_signal_processor\",\n \"-\",\n \"otel_signal\",\n \"processor\"\n )\n)", "instant": false, "queryType": "range", "range": true, "refId": "discarded_pct" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (otel_signal_processor)(\n label_join(\n sum by(processor, otel_signal) (\n rate(otelcol_processor_incoming_items_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval])\n ),\n \"otel_signal_processor\",\n \"-\",\n \"otel_signal\",\n \"processor\"\n )\n)", "instant": false, "queryType": "range", "range": true, "refId": "inbound_rate" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "expr": "sum by (otel_signal_processor)(\n label_join(\n sum by(processor, otel_signal) (\n rate(otelcol_processor_outgoing_items_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval])\n ),\n \"otel_signal_processor\",\n \"-\",\n \"otel_signal\",\n \"processor\"\n )\n)", "instant": false, "queryType": "range", "range": true, "refId": "outbound_rate" } ], "title": "Processors", "transformations": [ { "id": "timeSeriesTable", "options": { "discarded_pct": { "timeField": "Time" }, "inbound_rate": { "timeField": "Time" } } }, { "id": "joinByField", "options": { "byField": "otel_signal_processor", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": {}, "includeByName": {}, "indexByName": { "Trend #discarded_pct": 2, "Trend #inbound_rate": 1, "Trend #outbound_rate": 3, "otel_signal_processor": 0 }, "renameByName": { "Trend #discarded_pct": "Discarded %", "Trend #inbound_rate": "Inbound Rate", "Trend #outbound_rate": "Outbound Rate", "otel_signal_processor": "Processor", "otel_signal_receiver": "" } } } ], "type": "table" } ], "title": "Processors", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }, "id": 47, "panels": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Aggregated spans export rate across all exporters", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 0, "y": 96 }, "id": 43, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum(rate(otelcol_exporter_sent_spans_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))", "instant": false, "legendFormat": "Spans", "queryType": "range", "range": true, "refId": "A" } ], "title": "Spans", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 3, "y": 96 }, "id": 44, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "1\n-\nsum(\n rate(\n otelcol_exporter_sent_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n)\n/\n(\n sum(\n rate(\n otelcol_exporter_sent_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_exporter_send_failed_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n)", "instant": false, "legendFormat": "Spans", "queryType": "range", "range": true, "refId": "spans_error" } ], "title": "Span Errors", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Aggregated logs export rate across all exporters", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 5, "y": 96 }, "id": 70, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum(rate(otelcol_exporter_sent_log_records_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))", "instant": false, "legendFormat": "Logs", "queryType": "range", "range": true, "refId": "B" } ], "title": "Logs", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 8, "y": 96 }, "id": 72, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "1\n-\nsum(\n rate(\n otelcol_exporter_sent_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n)\n/\n(\n sum(\n rate(\n otelcol_exporter_sent_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_exporter_send_failed_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n)", "instant": false, "legendFormat": "Logs", "queryType": "range", "range": true, "refId": "logs_error" } ], "title": "Logs Errors", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Aggregate metrics export rate across all exporters", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 3, "x": 10, "y": 96 }, "id": 71, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum(rate(otelcol_exporter_sent_metric_points_total{deployment_environment_name=~\"$deployment_environment_name\",k8s_cluster_name=~\"$k8s_cluster_name\",k8s_node_name=~\"$k8s_node_name\",host_name=~\"$host_name\",service_instance_id=~\"$service_instance_id\",service_name=~\"$service_name\"}[$__rate_interval]))", "instant": false, "legendFormat": "Metrics", "queryType": "range", "range": true, "refId": "C" } ], "title": "Metrics", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byName", "options": "Spans" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Logs" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Metrics" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 3, "w": 2, "x": 13, "y": 96 }, "id": 73, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "1\n-\nsum(\n rate(\n otelcol_exporter_sent_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n)\n/\n(\n sum(\n rate(\n otelcol_exporter_sent_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n +\n sum(\n rate(\n otelcol_exporter_send_failed_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n)", "instant": false, "legendFormat": "Metrics", "queryType": "range", "range": true, "refId": "metrics_errors" } ], "title": "Metrics Errors", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "description": "Detailed breakdown by exporter and telemetry type", "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": false, "footer": { "reducers": [] }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*Rate.*" }, "properties": [ { "id": "custom.cellOptions", "value": { "type": "sparkline" } }, { "id": "unit", "value": "cps" } ] }, { "matcher": { "id": "byRegexp", "options": ".*%" }, "properties": [ { "id": "unit", "value": "percentunit" } ] } ] }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 109 }, "id": 45, "options": { "cellHeight": "sm", "showHeader": true }, "pluginVersion": "12.4.0-20757165945", "targets": [ { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (signal_type_exporter) (\n label_join(\n label_replace(\n sum by (exporter) (\n rate(\n otelcol_exporter_sent_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_exporter_send_failed_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n ),\n \"signal_type\",\n \"traces\",\n \"\",\n \"\"\n )\n or\n label_replace(\n sum by (exporter) (\n rate(\n otelcol_exporter_sent_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_exporter_send_failed_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n ),\n \"signal_type\",\n \"logs\",\n \"\",\n \"\"\n )\n or\n label_replace(\n sum by (exporter) (\n rate(\n otelcol_exporter_sent_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_exporter_send_failed_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n ),\n \"signal_type\",\n \"metrics\",\n \"\",\n \"\"\n )\n ,\n \"signal_type_exporter\",\n \"-\",\n \"signal_type\",\n \"exporter\"\n )\n)", "instant": false, "legendFormat": "__auto", "queryType": "range", "range": true, "refId": "rps" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (signal_type_exporter) (\n label_join(\n label_replace(\n sum by (exporter) (\n rate(\n otelcol_exporter_send_failed_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n /\n (\n sum by (exporter) (\n rate(\n otelcol_exporter_sent_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_exporter_send_failed_spans_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n )\n * 100\n ,\n \"signal_type\",\n \"traces\",\n \"\",\n \"\"\n )\n or\n label_replace(\n sum by (exporter) (\n rate(\n otelcol_exporter_send_failed_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n /\n (\n sum by (exporter) (\n rate(\n otelcol_exporter_sent_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_exporter_send_failed_log_records_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n )\n * 100\n ,\n \"signal_type\",\n \"logs\",\n \"\",\n \"\"\n )\n or\n label_replace(\n sum by (exporter) (\n rate(\n otelcol_exporter_send_failed_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n /\n (\n sum by (exporter) (\n rate(\n otelcol_exporter_sent_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n +\n rate(\n otelcol_exporter_send_failed_metric_points_total{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }[$__rate_interval]\n )\n )\n )\n * 100\n ,\n \"signal_type\",\n \"metrics\",\n \"\",\n \"\"\n )\n ,\n \"signal_type_exporter\",\n \"-\",\n \"signal_type\",\n \"exporter\"\n )\n)", "instant": false, "legendFormat": "__auto", "queryType": "range", "range": true, "refId": "errors" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (signal_type_exporter) (\n label_join(\n sum by (exporter, data_type) (\n otelcol_exporter_queue_size{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }\n )\n ,\n \"signal_type_exporter\",\n \"-\",\n \"data_type\",\n \"exporter\"\n )\n)", "instant": false, "legendFormat": "__auto", "queryType": "range", "range": true, "refId": "queue_size" }, { "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "editorMode": "code", "expr": "sum by (signal_type_exporter) (\n label_join(\n sum by (exporter, data_type) (\n otelcol_exporter_queue_size{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }\n /\n otelcol_exporter_queue_capacity{\n deployment_environment_name=~\"$deployment_environment_name\",\n k8s_cluster_name=~\"$k8s_cluster_name\",\n k8s_node_name=~\"$k8s_node_name\",\n host_name=~\"$host_name\",\n service_instance_id=~\"$service_instance_id\",\n service_name=~\"$service_name\"\n }\n )\n ,\n \"signal_type_exporter\",\n \"-\",\n \"data_type\",\n \"exporter\"\n )\n)", "instant": false, "legendFormat": "__auto", "queryType": "range", "range": true, "refId": "queue_size_pct" } ], "title": "Exporters", "transformations": [ { "id": "timeSeriesTable", "options": { "errors": { "timeField": "Time" }, "queue_size": { "timeField": "Time" }, "rps": { "timeField": "Time" } } }, { "id": "joinByField", "options": { "byField": "signal_type_exporter", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": {}, "includeByName": {}, "indexByName": {}, "renameByName": { "Trend #errors": "Errors %", "Trend #queue_size": "Queue size", "Trend #queue_size_pct": "Queue saturation %", "Trend #rps": "Rate", "signal_type_exporter": "Exporter" } } } ], "type": "table" } ], "title": "Exporters", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, "id": 46, "panels": [ { "datasource": { "type": "grafana-opensearch-datasource", "uid": "${opensearch_datasource}" }, "description": "Logs from OpenTelemetry Collector filtered by dashboard variables", "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 19 }, "id": 42, "options": { "dedupStrategy": "none", "detailsMode": "sidebar", "enableInfiniteScrolling": false, "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, "showControls": false, "showLabels": false, "showLogAttributes": true, "showTime": true, "sortOrder": "Ascending", "wrapLogMessage": false }, "pluginVersion": "12.3.1", "targets": [ { "alias": "", "bucketAggs": [ { "field": "observedTimestamp", "id": "2", "settings": { "interval": "auto" }, "type": "date_histogram" } ], "datasource": { "type": "grafana-opensearch-datasource", "uid": "${opensearch_datasource}" }, "format": "table", "luceneQueryType": "Metric", "metrics": [ { "id": "1", "type": "count" } ], "query": "search source=otel-logs-* | where resource.service.name=\"$service_name\" | fields @timestamp, resource.service.name, resource.service.instance.id, resource.host.name, severity.text, body", "queryType": "PPL", "refId": "A", "timeField": "observedTimestamp" } ], "title": "OpenTelemetry Collector Logs", "type": "logs" } ], "title": "Logs", "type": "row" } ], "preload": false, "refresh": "", "schemaVersion": 42, "tags": [ "opentelemetry", "opentelemetry-collector" ], "templating": { "list": [ { "current": { "text": "Prometheus", "value": "webstore-metrics" }, "description": "OpenTelemetry metrics. \nSend metrics using the Prometheus OTLP endpoint activating `keep_identifying_resource_attributes` and resource attribute promotion (aka `promote_resource_attributes`) including `service.name`, service.namespace`, `service.instance.id`, and `deployment.environment.name`", "label": "Metrics", "name": "prometheus_datasource", "options": [], "query": "prometheus", "refresh": 1, "regex": "(?!grafanacloud-usage|grafanacloud-ml-metrics).+", "type": "datasource" }, { "allowCustomValue": false, "current": { "text": "OpenSearch", "value": "webstore-logs" }, "description": "OpenTelemetry logs.", "label": "Logs", "name": "opensearch_datasource", "options": [], "query": "grafana-opensearch-datasource", "refresh": 1, "regex": "(?!grafanacloud-.*-alert-state-history|grafanacloud-.*-usage-insights).+", "type": "datasource" }, { "allValue": ".*", "current": { "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "definition": "label_values(otelcol_process_uptime_seconds_total,deployment_environment)", "description": "Deployment environment (e.g. \"production\").\nResource attribute `deployment.environment.name` via `target_info`", "includeAll": true, "label": "Environment", "multi": true, "name": "deployment_environment_name", "options": [], "query": { "qryType": 1, "query": "label_values(otelcol_process_uptime_seconds_total,deployment_environment)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", "sort": 1, "staticOptions": [ { "text": "<>", "value": "<>" } ], "staticOptionsOrder": "after", "type": "query" }, { "allValue": ".*", "current": { "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "definition": "label_values(otelcol_process_uptime_seconds_total{deployment_environment_name=~\"$deployment_environment_name\"},k8s_cluster_name)", "description": "Resource attribute `k8s.cluster.name`", "includeAll": true, "label": "Kubernetes cluster", "multi": true, "name": "k8s_cluster_name", "options": [], "query": { "qryType": 1, "query": "label_values(otelcol_process_uptime_seconds_total{deployment_environment_name=~\"$deployment_environment_name\"},k8s_cluster_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "staticOptions": [ { "text": "<>", "value": "<>" } ], "staticOptionsOrder": "after", "type": "query" }, { "allValue": ".*", "current": { "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "definition": "label_values(otelcol_process_cpu_seconds_total{k8s_cluster_name=~\"$k8s_cluster_name\"},k8s_node_name)", "description": "`k8s.node.name`", "includeAll": true, "label": "K8s node", "multi": true, "name": "k8s_node_name", "options": [], "query": { "qryType": 1, "query": "label_values(otelcol_process_cpu_seconds_total{k8s_cluster_name=~\"$k8s_cluster_name\"},k8s_node_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "staticOptions": [ { "text": "<>", "value": "<>" } ], "staticOptionsOrder": "after", "type": "query" }, { "allValue": ".*", "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "definition": "label_values(otelcol_process_uptime_seconds_total,host_name)", "description": "`host.name`", "includeAll": true, "label": "Host", "name": "host_name", "options": [], "query": { "qryType": 1, "query": "label_values(otelcol_process_uptime_seconds_total,host_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "staticOptions": [ { "text": "<>", "value": "<>" } ], "type": "query" }, { "allValue": ".*", "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "definition": "label_values(otelcol_process_uptime_seconds_total{deployment_environment_name=~\"$deployment_environment_name\", k8s_cluster_name=~\"$k8s_cluster_name\", k8s_node_name=~\"$k8s_node_name\", host_name=~\"$host_name\"},service_instance_id)", "description": "`service.instance.id`", "includeAll": true, "label": "Instance", "name": "service_instance_id", "options": [], "query": { "qryType": 1, "query": "label_values(otelcol_process_uptime_seconds_total{deployment_environment_name=~\"$deployment_environment_name\", k8s_cluster_name=~\"$k8s_cluster_name\", k8s_node_name=~\"$k8s_node_name\", host_name=~\"$host_name\"},service_instance_id)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" }, { "allValue": ".*", "current": { "text": "otelcol-contrib", "value": "otelcol-contrib" }, "datasource": { "type": "prometheus", "uid": "${prometheus_datasource}" }, "definition": "label_values(otelcol_process_uptime_seconds_total{deployment_environment_name=~\"$deployment_environment_name\", k8s_cluster_name=~\"$k8s_cluster_name\", k8s_node_name=~\"$k8s_node_name\", host_name=~\"$host_name\"},service_name)", "description": "`service.name`", "includeAll": true, "label": "Service", "name": "service_name", "options": [], "query": { "qryType": 1, "query": "label_values(otelcol_process_uptime_seconds_total{deployment_environment_name=~\"$deployment_environment_name\", k8s_cluster_name=~\"$k8s_cluster_name\", k8s_node_name=~\"$k8s_node_name\", host_name=~\"$host_name\"},service_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" } ] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": { "refresh_intervals": [ "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ] }, "timezone": "browser", "title": "OpenTelemetry Collector", "uid": "otel-demo_otel-collector_dashboard", "version": 1 }