From 9dae14aa66dbc46c0c919877322b3b093d19bcd9 Mon Sep 17 00:00:00 2001 From: mamuthus Date: Fri, 1 May 2026 18:26:32 +0000 Subject: [PATCH 1/5] Add class-based error metrics and dashboard panels --- src/metrics.rs | 41 +- tools/grafana-dashboard.json | 11538 +++++++++++++++++++-------------- 2 files changed, 6652 insertions(+), 4927 deletions(-) diff --git a/src/metrics.rs b/src/metrics.rs index addb18f..670d0e0 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -454,6 +454,21 @@ async fn render_metrics( } ); + let _ = writeln!( + out, + "# HELP telemt_connections_bad_by_class_total Bad/rejected connections by class" + ); + let _ = writeln!(out, "# TYPE telemt_connections_bad_by_class_total counter"); + if core_enabled { + for (class, total) in stats.get_connects_bad_class_counts() { + let _ = writeln!( + out, + "telemt_connections_bad_by_class_total{{class=\"{}\"}} {}", + class, total + ); + } + } + let _ = writeln!( out, "# HELP telemt_handshake_timeouts_total Handshake timeouts" @@ -469,6 +484,24 @@ async fn render_metrics( } ); + let _ = writeln!( + out, + "# HELP telemt_handshake_failures_by_class_total Handshake failures by class" + ); + let _ = writeln!( + out, + "# TYPE telemt_handshake_failures_by_class_total counter" + ); + if core_enabled { + for (class, total) in stats.get_handshake_failure_class_counts() { + let _ = writeln!( + out, + "telemt_handshake_failures_by_class_total{{class=\"{}\"}} {}", + class, total + ); + } + } + let _ = writeln!( out, "# HELP telemt_auth_expensive_checks_total Expensive authentication candidate checks executed during handshake validation" @@ -3342,8 +3375,9 @@ mod tests { stats.increment_connects_all(); stats.increment_connects_all(); - stats.increment_connects_bad(); + stats.increment_connects_bad_with_class("tls_handshake_bad_client"); stats.increment_handshake_timeouts(); + stats.increment_handshake_failure_class("timeout"); shared_state .handshake .auth_expensive_checks_total @@ -3403,7 +3437,10 @@ mod tests { ))); assert!(output.contains("telemt_connections_total 2")); assert!(output.contains("telemt_connections_bad_total 1")); + assert!(output + .contains("telemt_connections_bad_by_class_total{class=\"tls_handshake_bad_client\"} 1")); assert!(output.contains("telemt_handshake_timeouts_total 1")); + assert!(output.contains("telemt_handshake_failures_by_class_total{class=\"timeout\"} 1")); assert!(output.contains("telemt_auth_expensive_checks_total 9")); assert!(output.contains("telemt_auth_budget_exhausted_total 2")); assert!(output.contains("telemt_upstream_connect_attempt_total 2")); @@ -3503,7 +3540,9 @@ mod tests { assert!(output.contains("# TYPE telemt_uptime_seconds gauge")); assert!(output.contains("# TYPE telemt_connections_total counter")); assert!(output.contains("# TYPE telemt_connections_bad_total counter")); + assert!(output.contains("# TYPE telemt_connections_bad_by_class_total counter")); assert!(output.contains("# TYPE telemt_handshake_timeouts_total counter")); + assert!(output.contains("# TYPE telemt_handshake_failures_by_class_total counter")); assert!(output.contains("# TYPE telemt_auth_expensive_checks_total counter")); assert!(output.contains("# TYPE telemt_auth_budget_exhausted_total counter")); assert!(output.contains("# TYPE telemt_upstream_connect_attempt_total counter")); diff --git a/tools/grafana-dashboard.json b/tools/grafana-dashboard.json index 6c16b8d..1ca898a 100644 --- a/tools/grafana-dashboard.json +++ b/tools/grafana-dashboard.json @@ -1,4964 +1,6650 @@ { - "apiVersion": "dashboard.grafana.app/v1beta1", - "kind": "DashboardWithAccessInfo", - "metadata": { - "name": "Telemt MtProto proxy" - }, - "spec": { "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "links": [ - { - "asDropdown": false, - "icon": "bolt", - "includeVars": false, - "keepTime": false, - "tags": [], - "targetBlank": true, - "title": "Official GitHub repository", - "tooltip": "Official GitHub repository", - "type": "link", - "url": "https://github.com/telemt/telemt" - } + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Official GitHub repository", + "tooltip": "Official GitHub repository", + "type": "link", + "url": "https://github.com/telemt/telemt" + } ], "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 200, + "panels": [], + "title": "General metrics", + "type": "row" }, - "id": 200, - "panels": [], - "title": "General metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Build version reported by telemt.", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Build version reported by telemt.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "none" }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] + "overrides": [] }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "id": 301, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "name", - "wideLayout": true - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "max by (version) (telemt_build_info{job=~\"$job\"})", - "instant": true, - "legendFormat": "{{version}}", - "refId": "A" - } - ], - "title": "Build version", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "telemt process uptime in seconds.", - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "id": 309, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "max(telemt_uptime_seconds{job=~\"$job\"})", - "refId": "A" - } - ], - "title": "Uptime", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Configured user count derived from per-user unique IP limit series.", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "id": 317, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "count(telemt_user_unique_ips_limit{job=~\"$job\"})", - "instant": true, - "refId": "A" - } - ], - "title": "Configured users", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Total inbound and outbound user traffic bytes since process start.", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "In total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Out total" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "id": 318, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value_and_name", - "wideLayout": true - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(telemt_user_octets_from_client{job=~\"$job\"})", - "instant": true, - "legendFormat": "In total", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(telemt_user_octets_to_client{job=~\"$job\"})", - "instant": true, - "legendFormat": "Out total", - "refId": "B" - } - ], - "title": "Total traffic", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Current number of in-use shared buffers.", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "id": 312, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "max(telemt_buffer_pool_buffers_total{kind=\"in_use\",job=~\"$job\"})", - "legendFormat": "in use", - "refId": "A" - } - ], - "title": "Buffer pool state", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "User-series export state (1 enabled, 0 suppressed).", - "fieldConfig": { - "defaults": { - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": 0 - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 311, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "max(1 - telemt_telemetry_user_series_suppressed{job=~\"$job\"})", - "instant": true, - "refId": "A" - } - ], - "title": "User series enabled", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Accepted client connections in the selected time range.", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": 0 - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 5 - }, - "id": 7, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(increase(telemt_connections_total{job=~\"$job\"}[$__range]))", - "refId": "A" - } - ], - "title": "Accepted connections", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Failed or rejected client connections in the selected time range.", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 5 - }, - "id": 8, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(increase(telemt_connections_bad_total{job=~\"$job\"}[$__range]))", - "refId": "A" - } - ], - "title": "Bad connections", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Bad connection ratio over the selected interval.", - "fieldConfig": { - "defaults": { - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 5 - }, - "id": 10, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "100 * sum(rate(telemt_connections_bad_total{job=~\"$job\"}[$__rate_interval])) / clamp_min(sum(rate(telemt_connections_total{job=~\"$job\"}[$__rate_interval])), 1e-9)", - "refId": "A" - } - ], - "title": "Bad connection ratio", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Per-second rates for total, bad, handshake-timeout, and permit-timeout connection events.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Connections/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Bad/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Timeouts/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Permit waits/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 5 - }, - "id": 11, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_connections_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Connections/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_connections_bad_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Bad/s", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_handshake_timeouts_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Timeouts/s", - "refId": "C" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_accept_permit_timeout_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Permit waits/s", - "refId": "D" - } - ], - "title": "Connection and permit rates", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Accepted connections dropped by permit-acquisition timeout in the selected time range.", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 9 - }, - "id": 302, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(increase(telemt_accept_permit_timeout_total{job=~\"$job\"}[$__range]))", - "refId": "A" - } - ], - "title": "Permit wait timeouts", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Handshake validations that exhausted the authentication candidate budget in the selected time range.", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 9 - }, - "id": 303, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(increase(telemt_auth_budget_exhausted_total{job=~\"$job\"}[$__range]))", - "refId": "A" - } - ], - "title": "Auth budget exhausted", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Handshake timeouts in the selected time range.", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 9 - }, - "id": 9, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(increase(telemt_handshake_timeouts_total{job=~\"$job\"}[$__range]))", - "refId": "A" - } - ], - "title": "Handshake timeouts", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 320, - "panels": [], - "title": "Upstream connectivity", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Upstream connect attempt/success/fail rates for Telegram/DC connectivity path.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 70, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Attempt/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Success/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Fail/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 14 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_upstream_connect_attempt_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Attempt/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_upstream_connect_success_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Success/s", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_upstream_connect_fail_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Fail/s", - "refId": "C" - } - ], - "title": "Upstream connect outcomes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Per-second rates of upstream connect-attempt buckets per request.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "1 attempt/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "2 attempts/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "3-4 attempts/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": ">4 attempts/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 14 - }, - "id": 321, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_upstream_connect_attempts_per_request{bucket=\"1\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "1 attempt/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_upstream_connect_attempts_per_request{bucket=\"2\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "2 attempts/s", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_upstream_connect_attempts_per_request{bucket=\"3_4\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "3-4 attempts/s", - "refId": "C" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_upstream_connect_attempts_per_request{bucket=\"gt_4\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": ">4 attempts/s", - "refId": "D" - } - ], - "title": "Upstream connect attempts per request", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Per-second rates of slow upstream connect buckets and hard-error failfast events.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "success >1s/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "fail >1s/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "failfast hard/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 14 - }, - "id": 322, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_upstream_connect_duration_success_total{bucket=\"gt_1000ms\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "success >1s/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_upstream_connect_duration_fail_total{bucket=\"gt_1000ms\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "fail >1s/s", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_upstream_connect_failfast_hard_error_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "failfast hard/s", - "refId": "C" - } - ], - "title": "Upstream connect duration and failfast", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 330, - "panels": [], - "title": "Authentication and security", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Per-second rates of authentication expensive checks and budget exhaustion.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Expensive checks/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Budget exhausted/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 304, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_auth_expensive_checks_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Expensive checks/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_auth_budget_exhausted_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Budget exhausted/s", - "refId": "B" - } - ], - "title": "Auth validation pressure", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Per-second rates of invalid secure padding and permit wait timeouts.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "secure padding invalid/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "permit wait timeout/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 23 - }, - "id": 331, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_secure_padding_invalid_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "secure padding invalid/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_accept_permit_timeout_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "permit wait timeout/s", - "refId": "B" - } - ], - "title": "Security rejects", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 340, - "panels": [], - "title": "Conntrack control", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Current conntrack-control state flags (enabled, available, pressure_active, rule_apply_ok).", - "fieldConfig": { - "defaults": { - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": 0 - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "pressure_active" - }, - "properties": [ - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 1 - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 0, - "y": 32 - }, - "id": 305, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": false - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "max(telemt_conntrack_control_state{flag=\"enabled\",job=~\"$job\"})", - "instant": true, - "legendFormat": "enabled", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "max(telemt_conntrack_control_state{flag=\"available\",job=~\"$job\"})", - "instant": true, - "legendFormat": "available", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "max(telemt_conntrack_control_state{flag=\"pressure_active\",job=~\"$job\"})", - "instant": true, - "legendFormat": "pressure_active", - "refId": "C" - }, - { - "editorMode": "code", - "expr": "max(telemt_conntrack_control_state{flag=\"rule_apply_ok\",job=~\"$job\"})", - "instant": true, - "legendFormat": "rule_apply_ok", - "refId": "D" - } - ], - "title": "Conntrack state flags", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Per-second rates of conntrack delete outcomes and dropped close events.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Delete attempt/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Delete success/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Delete not_found/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6B7280", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Delete error/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Close-event drop/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 16, - "x": 8, - "y": 32 - }, - "id": 306, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_conntrack_delete_total{result=\"attempt\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Delete attempt/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_conntrack_delete_total{result=\"success\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Delete success/s", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_conntrack_delete_total{result=\"not_found\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Delete not_found/s", - "refId": "C" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_conntrack_delete_total{result=\"error\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Delete error/s", - "refId": "D" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_conntrack_close_event_drop_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Close-event drop/s", - "refId": "E" - } - ], - "title": "Conntrack delete and drop rates", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Current conntrack close-event queue depth.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 0, - "y": 36 - }, - "id": 307, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "max(telemt_conntrack_event_queue_depth{job=~\"$job\"})", - "legendFormat": "Queue depth", - "refId": "A" - } - ], - "title": "Conntrack event queue depth", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 310, - "panels": [], - "title": "ME metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Current ME telemetry mode level.", - "fieldConfig": { - "defaults": { - "mappings": [ - { - "options": { - "0": { - "color": "#6B7280", - "text": "silent" - }, - "1": { - "color": "green", - "text": "normal" - }, - "2": { - "color": "orange", - "text": "debug" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#6B7280", - "value": 0 - }, - { - "color": "green", - "value": 1 - }, - { - "color": "orange", - "value": 2 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 41 - }, - "id": 316, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": false - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "max(telemt_telemetry_me_level{level=\"normal\",job=~\"$job\"}) + 2 * max(telemt_telemetry_me_level{level=\"debug\",job=~\"$job\"})", - "instant": true, - "refId": "A" - } - ], - "title": "ME telemetry mode", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "ME handshake rejects in the selected time range.", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 41 - }, - "id": 308, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(increase(telemt_me_handshake_reject_total{job=~\"$job\"}[$__range]))", - "refId": "A" - } - ], - "title": "ME handshake rejects", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Reconnect attempts/success and reader EOF event rates in Middle-End (ME) subsystem.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Reconnect attempt/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Reconnect success/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Reader EOF/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 41 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_reconnect_attempts_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Reconnect attempt/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_reconnect_success_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Reconnect success/s", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_reader_eof_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Reader EOF/s", - "refId": "C" - } - ], - "title": "ME reconnect and reader EOF rates", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Rates of Middle-End (ME) route drops by reason and total crypto desync detections.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Drop no_conn/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Drop channel_closed/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Drop queue_full/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Desync/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 41 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_route_drop_no_conn_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Drop no_conn/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_route_drop_channel_closed_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Drop channel_closed/s", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_route_drop_queue_full_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Drop queue_full/s", - "refId": "C" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_desync_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Desync/s", - "refId": "D" - } - ], - "title": "ME route drops and crypto desync", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Per-second rate of ME handshake rejects, including error-code breakdown.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "reject total/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 0, - "y": 45 - }, - "id": 353, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_handshake_reject_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "reject total/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum by (error_code) (rate(telemt_me_handshake_error_code_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "Code {{error_code}}/s", - "refId": "B" - } - ], - "title": "ME handshake reject rate by code", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Current counts of active and warm ME writers.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Warm" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6B7280", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 49 - }, - "id": 13, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(telemt_me_writers_active_current{job=~\"$job\"})", - "legendFormat": "Active", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(telemt_me_writers_warm_current{job=~\"$job\"})", - "legendFormat": "Warm", - "refId": "B" - } - ], - "title": "ME writers state", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Keepalive sent, pong, failed and timeout rates for ME.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "sent/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "pong/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "failed/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timeout/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 16, - "x": 8, - "y": 49 - }, - "id": 313, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_keepalive_sent_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "sent/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_keepalive_pong_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "pong/s", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_keepalive_failed_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "failed/s", - "refId": "C" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_keepalive_timeout_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "timeout/s", - "refId": "D" - } - ], - "title": "ME keepalive health", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Per-second rates of ME pool forced closes and refill events.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "forced close/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "refill triggered/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "refill failed/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 57 - }, - "id": 314, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_pool_force_close_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "forced close/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_refill_triggered_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "refill triggered/s", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_refill_failed_total{job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "refill failed/s", - "refId": "C" - } - ], - "title": "ME refill activity", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Current ME capacity target, active writers, and warm writers.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "target writers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "active writers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "warm writers" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6B7280", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 57 - }, - "id": 315, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "max(telemt_me_adaptive_floor_target_writers_total{job=~\"$job\"})", - "legendFormat": "target writers", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(telemt_me_writers_active_current{job=~\"$job\"})", - "legendFormat": "active writers", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "sum(telemt_me_writers_warm_current{job=~\"$job\"})", - "legendFormat": "warm writers", - "refId": "C" - } - ], - "title": "ME capacity", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 65 - }, - "id": 355, - "panels": [], - "title": "ME debug metrics", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Per-second rates of ME debug D2C batch-byte buckets.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "0-1 KiB/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "1-4 KiB/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "4-16 KiB/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "16-64 KiB/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "64-128 KiB/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F97316", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": ">128 KiB/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 66 - }, - "id": 356, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_batch_bytes_bucket_total{bucket=\"0_1k\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "0-1 KiB/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_batch_bytes_bucket_total{bucket=\"1k_4k\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "1-4 KiB/s", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_batch_bytes_bucket_total{bucket=\"4k_16k\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "4-16 KiB/s", - "refId": "C" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_batch_bytes_bucket_total{bucket=\"16k_64k\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "16-64 KiB/s", - "refId": "D" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_batch_bytes_bucket_total{bucket=\"64k_128k\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "64-128 KiB/s", - "refId": "E" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_batch_bytes_bucket_total{bucket=\"gt_128k\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": ">128 KiB/s", - "refId": "F" - } - ], - "title": "ME debug D2C batch bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Per-second rates of ME debug D2C batch-size buckets.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "1 frame/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "2-4 frames/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "5-8 frames/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "9-16 frames/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "17-32 frames/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F97316", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": ">32 frames/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 66 - }, - "id": 357, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_batch_frames_bucket_total{bucket=\"1\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "1 frame/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_batch_frames_bucket_total{bucket=\"2_4\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "2-4 frames/s", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_batch_frames_bucket_total{bucket=\"5_8\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "5-8 frames/s", - "refId": "C" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_batch_frames_bucket_total{bucket=\"9_16\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "9-16 frames/s", - "refId": "D" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_batch_frames_bucket_total{bucket=\"17_32\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "17-32 frames/s", - "refId": "E" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_batch_frames_bucket_total{bucket=\"gt_32\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": ">32 frames/s", - "refId": "F" - } - ], - "title": "ME debug D2C batch size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Per-second rates of ME debug D2C flush-duration buckets.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "0-50 us/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "51-200 us/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "201-1000 us/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "1-5 ms/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "5-20 ms/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#F97316", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": ">20 ms/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 66 - }, - "id": 358, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_flush_duration_us_bucket_total{bucket=\"0_50\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "0-50 us/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_flush_duration_us_bucket_total{bucket=\"51_200\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "51-200 us/s", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_flush_duration_us_bucket_total{bucket=\"201_1000\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "201-1000 us/s", - "refId": "C" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_flush_duration_us_bucket_total{bucket=\"1001_5000\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "1-5 ms/s", - "refId": "D" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_flush_duration_us_bucket_total{bucket=\"5001_20000\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "5-20 ms/s", - "refId": "E" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_me_d2c_flush_duration_us_bucket_total{bucket=\"gt_20000\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": ">20 ms/s", - "refId": "F" - } - ], - "title": "ME debug D2C flush duration", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 74 - }, - "id": 350, - "panels": [], - "title": "Access and limits", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Current active and recent user/IP tracker sizes and cleanup queue depth.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "users active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "users recent" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6B7280", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "IPs active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "IPs recent" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#6B7280", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "cleanup queue" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 75 - }, - "id": 351, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "max(telemt_ip_tracker_users{scope=\"active\",job=~\"$job\"})", - "legendFormat": "users active", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "max(telemt_ip_tracker_users{scope=\"recent\",job=~\"$job\"})", - "legendFormat": "users recent", - "refId": "B" - }, - { - "editorMode": "code", - "expr": "max(telemt_ip_tracker_entries{scope=\"active\",job=~\"$job\"})", - "legendFormat": "IPs active", - "refId": "C" - }, - { - "editorMode": "code", - "expr": "max(telemt_ip_tracker_entries{scope=\"recent\",job=~\"$job\"})", - "legendFormat": "IPs recent", - "refId": "D" - }, - { - "editorMode": "code", - "expr": "max(telemt_ip_tracker_cleanup_queue_len{job=~\"$job\"})", - "legendFormat": "cleanup queue", - "refId": "E" - } - ], - "title": "IP tracker state", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "IP reservation rollbacks in the selected time range.", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 75 - }, - "id": 352, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(increase(telemt_ip_reservation_rollback_total{job=~\"$job\"}[$__range]))", - "refId": "A" - } - ], - "title": "IP reservation rollbacks", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Reservation rollback rate by reason.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "tcp_limit/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "quota_limit/s" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 79 - }, - "id": 354, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum(rate(telemt_ip_reservation_rollback_total{reason=\"tcp_limit\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "tcp_limit/s", - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum(rate(telemt_ip_reservation_rollback_total{reason=\"quota_limit\",job=~\"$job\"}[$__rate_interval]))", - "legendFormat": "quota_limit/s", - "refId": "B" - } - ], - "title": "IP reservation rollback rate by reason", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Current per-user unique IP utilization percentage.", - "fieldConfig": { - "defaults": { - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 60 - }, - { - "color": "red", - "value": 85 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 83 - }, - "id": 23, - "options": { - "displayMode": "basic", - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "maxVizHeight": 300, - "minVizHeight": 16, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sort_desc(100 * max by (user) (telemt_user_unique_ips_utilization{job=~\"$job\",user=~\"$user\"}))", - "instant": true, - "legendFormat": "{{user}}", - "refId": "A" - } - ], - "title": "User unique IP utilization", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Current unique IP count per selected user.", - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "footer": { - "reducers": [] - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 83 - }, - "id": 21, - "options": { - "cellHeight": "sm", - "showHeader": true - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum by (user) (telemt_user_unique_ips_current{job=~\"$job\",user=~\"$user\"})", - "format": "table", - "instant": true, - "refId": "A" - } - ], - "title": "User unique IPs", - "transformations": [ - { - "id": "organize", + "id": 301, "options": { - "excludeByName": { - "Time": true - }, - "renameByName": { - "Value": "Unique IPs", - "user": "User" - } - } - }, - { - "id": "sortBy", + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max by (version) (telemt_build_info{job=~\"$job\"})", + "instant": true, + "legendFormat": "{{version}}", + "refId": "A" + } + ], + "title": "Build version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "telemt process uptime in seconds.", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 309, "options": { - "fields": { - "Unique IPs": true - }, - "sort": [ + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ { - "desc": true, - "field": "Unique IPs" + "editorMode": "code", + "expr": "max(telemt_uptime_seconds{job=~\"$job\"})", + "refId": "A" } - ] - } - } - ], - "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 91 + ], + "title": "Uptime", + "type": "stat" }, - "id": 300, - "panels": [], - "title": "Users", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Per-user active connections, traffic totals, message rates, and traffic rates.", - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "footer": { - "reducers": [] - }, - "inspect": false + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] + "description": "Configured user count derived from per-user unique IP limit series.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "In Total" - }, - "properties": [ - { - "id": "unit", - "value": "decbytes" - } - ] + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 }, - { - "matcher": { - "id": "byName", - "options": "Out Total" - }, - "properties": [ - { - "id": "unit", - "value": "decbytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "In msg/s" - }, - "properties": [ - { - "id": "unit", - "value": "pps" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Out msg/s" - }, - "properties": [ - { - "id": "unit", - "value": "pps" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "In traffic" - }, - "properties": [ - { - "id": "unit", - "value": "binBps" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Out traffic" - }, - "properties": [ - { - "id": "unit", - "value": "binBps" - } - ] - } - ] - }, - "gridPos": { - "h": 16, - "w": 24, - "x": 0, - "y": 92 - }, - "id": 16, - "options": { - "cellHeight": "sm", - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Active Connections" - } - ] - }, - "pluginVersion": "12.4.2", - "targets": [ - { - "editorMode": "code", - "expr": "sum by (user) (telemt_user_connections_current{job=~\"$job\",user=~\"$user\"})", - "format": "table", - "instant": true, - "refId": "A" - }, - { - "editorMode": "code", - "expr": "sum by (user) (telemt_user_octets_from_client{job=~\"$job\",user=~\"$user\"})", - "format": "table", - "instant": true, - "refId": "B" - }, - { - "editorMode": "code", - "expr": "sum by (user) (telemt_user_octets_to_client{job=~\"$job\",user=~\"$user\"})", - "format": "table", - "instant": true, - "refId": "C" - }, - { - "editorMode": "code", - "expr": "sum by (user) (rate(telemt_user_msgs_from_client{job=~\"$job\",user=~\"$user\"}[$__rate_interval]))", - "format": "table", - "instant": true, - "refId": "D" - }, - { - "editorMode": "code", - "expr": "sum by (user) (rate(telemt_user_msgs_to_client{job=~\"$job\",user=~\"$user\"}[$__rate_interval]))", - "format": "table", - "instant": true, - "refId": "E" - }, - { - "editorMode": "code", - "expr": "sum by (user) (rate(telemt_user_octets_from_client{job=~\"$job\",user=~\"$user\"}[$__rate_interval]))", - "format": "table", - "instant": true, - "refId": "F" - }, - { - "editorMode": "code", - "expr": "sum by (user) (rate(telemt_user_octets_to_client{job=~\"$job\",user=~\"$user\"}[$__rate_interval]))", - "format": "table", - "instant": true, - "refId": "G" - } - ], - "title": "User connections, traffic, and message rates", - "transformations": [ - { - "id": "joinByField", + "id": 317, "options": { - "byField": "user", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time #A": true, - "Time #B": true, - "Time #C": true, - "Time #D": true, - "Time #E": true, - "Time #F": true, - "Time #G": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "Time 6": true - }, - "renameByName": { - "Value": "Active Connections", - "Value #A": "Active Connections", - "Value #B": "In Total", - "Value #C": "Out Total", - "Value #D": "In msg/s", - "Value #E": "Out msg/s", - "Value #F": "In traffic", - "Value #G": "Out traffic", - "Value 1": "In Total", - "Value 2": "Out Total", - "Value 3": "In msg/s", - "Value 4": "Out msg/s", - "Value 5": "In traffic", - "Value 6": "Out traffic", - "user": "User" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": { - "Active Connections": true - }, - "sort": [ + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ { - "desc": true, - "field": "Active Connections" + "editorMode": "code", + "expr": "count(telemt_user_unique_ips_limit{job=~\"$job\"})", + "instant": true, + "refId": "A" } - ] - } - } - ], - "type": "table" - } + ], + "title": "Configured users", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total inbound and outbound user traffic bytes since process start.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "In total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Out total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 318, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(telemt_user_octets_from_client{job=~\"$job\"})", + "instant": true, + "legendFormat": "In total", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(telemt_user_octets_to_client{job=~\"$job\"})", + "instant": true, + "legendFormat": "Out total", + "refId": "B" + } + ], + "title": "Total traffic", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current number of in-use shared buffers.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 312, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max(telemt_buffer_pool_buffers_total{kind=\"in_use\",job=~\"$job\"})", + "legendFormat": "in use", + "refId": "A" + } + ], + "title": "Buffer pool state", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "User-series export state (1 enabled, 0 suppressed).", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 311, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max(1 - telemt_telemetry_user_series_suppressed{job=~\"$job\"})", + "instant": true, + "refId": "A" + } + ], + "title": "User series enabled", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Accepted client connections in the selected time range.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 7, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(increase(telemt_connections_total{job=~\"$job\"}[$__range]))", + "refId": "A" + } + ], + "title": "Accepted connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Failed or rejected client connections in the selected time range.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 8, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(increase(telemt_connections_bad_total{job=~\"$job\"}[$__range]))", + "refId": "A" + } + ], + "title": "Bad connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Bad connection ratio over the selected interval.", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 5 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "100 * sum(rate(telemt_connections_bad_total{job=~\"$job\"}[$__rate_interval])) / clamp_min(sum(rate(telemt_connections_total{job=~\"$job\"}[$__rate_interval])), 1e-9)", + "refId": "A" + } + ], + "title": "Bad connection ratio", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rates for total, bad, handshake-timeout, and permit-timeout connection events.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Connections/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Bad/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Timeouts/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Permit waits/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_connections_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Connections/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_connections_bad_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Bad/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_handshake_timeouts_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Timeouts/s", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_accept_permit_timeout_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Permit waits/s", + "refId": "D" + } + ], + "title": "Connection and permit rates", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Accepted connections dropped by permit-acquisition timeout in the selected time range.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 302, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(increase(telemt_accept_permit_timeout_total{job=~\"$job\"}[$__range]))", + "refId": "A" + } + ], + "title": "Permit wait timeouts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Handshake validations that exhausted the authentication candidate budget in the selected time range.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 9 + }, + "id": 303, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(increase(telemt_auth_budget_exhausted_total{job=~\"$job\"}[$__range]))", + "refId": "A" + } + ], + "title": "Auth budget exhausted", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Handshake timeouts in the selected time range.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 9 + }, + "id": 9, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(increase(telemt_handshake_timeouts_total{job=~\"$job\"}[$__range]))", + "refId": "A" + } + ], + "title": "Handshake timeouts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rates of bad/rejected connections split by failure class.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 9 + }, + "id": 437, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (class) (rate(telemt_connections_bad_by_class_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "{{class}}/s", + "refId": "A" + } + ], + "title": "Bad connections by class", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rates of handshake failures split by failure class.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 9 + }, + "id": 438, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (class) (rate(telemt_handshake_failures_by_class_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "{{class}}/s", + "refId": "A" + } + ], + "title": "Handshake failures by class", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 320, + "panels": [], + "title": "Upstream connectivity", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Upstream connect attempt/success/fail rates for Telegram/DC connectivity path.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 70, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Attempt/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Success/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Fail/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_upstream_connect_attempt_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Attempt/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_upstream_connect_success_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Success/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_upstream_connect_fail_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Fail/s", + "refId": "C" + } + ], + "title": "Upstream connect outcomes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rates of upstream connect-attempt buckets per request.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1 attempt/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2 attempts/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3-4 attempts/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": ">4 attempts/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 14 + }, + "id": 321, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_upstream_connect_attempts_per_request{bucket=\"1\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "1 attempt/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_upstream_connect_attempts_per_request{bucket=\"2\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "2 attempts/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_upstream_connect_attempts_per_request{bucket=\"3_4\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "3-4 attempts/s", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_upstream_connect_attempts_per_request{bucket=\"gt_4\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": ">4 attempts/s", + "refId": "D" + } + ], + "title": "Upstream connect attempts per request", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rates of slow upstream connect buckets and hard-error failfast events.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "success >1s/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "fail >1s/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failfast hard/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 322, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_upstream_connect_duration_success_total{bucket=\"gt_1000ms\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "success >1s/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_upstream_connect_duration_fail_total{bucket=\"gt_1000ms\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "fail >1s/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_upstream_connect_failfast_hard_error_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "failfast hard/s", + "refId": "C" + } + ], + "title": "Upstream connect duration and failfast", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 330, + "panels": [], + "title": "Authentication and security", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rates of authentication expensive checks and budget exhaustion.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Expensive checks/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Budget exhausted/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 304, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_auth_expensive_checks_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Expensive checks/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_auth_budget_exhausted_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Budget exhausted/s", + "refId": "B" + } + ], + "title": "Auth validation pressure", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rates of invalid secure padding and permit wait timeouts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "secure padding invalid/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "permit wait timeout/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 331, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_secure_padding_invalid_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "secure padding invalid/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_accept_permit_timeout_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "permit wait timeout/s", + "refId": "B" + } + ], + "title": "Security rejects", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 340, + "panels": [], + "title": "Conntrack control", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current conntrack-control state flags (enabled, available, pressure_active, rule_apply_ok).", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "pressure_active" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 32 + }, + "id": 305, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": false + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max(telemt_conntrack_control_state{flag=\"enabled\",job=~\"$job\"})", + "instant": true, + "legendFormat": "enabled", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "max(telemt_conntrack_control_state{flag=\"available\",job=~\"$job\"})", + "instant": true, + "legendFormat": "available", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "max(telemt_conntrack_control_state{flag=\"pressure_active\",job=~\"$job\"})", + "instant": true, + "legendFormat": "pressure_active", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "max(telemt_conntrack_control_state{flag=\"rule_apply_ok\",job=~\"$job\"})", + "instant": true, + "legendFormat": "rule_apply_ok", + "refId": "D" + } + ], + "title": "Conntrack state flags", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rates of conntrack delete outcomes and dropped close events.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Delete attempt/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Delete success/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Delete not_found/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6B7280", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Delete error/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Close-event drop/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 32 + }, + "id": 306, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_conntrack_delete_total{result=\"attempt\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Delete attempt/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_conntrack_delete_total{result=\"success\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Delete success/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_conntrack_delete_total{result=\"not_found\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Delete not_found/s", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_conntrack_delete_total{result=\"error\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Delete error/s", + "refId": "D" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_conntrack_close_event_drop_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Close-event drop/s", + "refId": "E" + } + ], + "title": "Conntrack delete and drop rates", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current conntrack close-event queue depth.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 36 + }, + "id": 307, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max(telemt_conntrack_event_queue_depth{job=~\"$job\"})", + "legendFormat": "Queue depth", + "refId": "A" + } + ], + "title": "Conntrack event queue depth", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 310, + "panels": [], + "title": "ME metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current ME telemetry mode level.", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "0": { + "color": "#6B7280", + "text": "silent" + }, + "1": { + "color": "green", + "text": "normal" + }, + "2": { + "color": "orange", + "text": "debug" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#6B7280", + "value": 0 + }, + { + "color": "green", + "value": 1 + }, + { + "color": "orange", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 41 + }, + "id": 316, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max(telemt_telemetry_me_level{level=\"normal\",job=~\"$job\"}) + 2 * max(telemt_telemetry_me_level{level=\"debug\",job=~\"$job\"})", + "instant": true, + "refId": "A" + } + ], + "title": "ME telemetry mode", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "ME handshake rejects in the selected time range.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 41 + }, + "id": 308, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(increase(telemt_me_handshake_reject_total{job=~\"$job\"}[$__range]))", + "refId": "A" + } + ], + "title": "ME handshake rejects", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Reconnect attempts/success and reader EOF event rates in Middle-End (ME) subsystem.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Reconnect attempt/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Reconnect success/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Reader EOF/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 41 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_reconnect_attempts_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Reconnect attempt/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_reconnect_success_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Reconnect success/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_reader_eof_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Reader EOF/s", + "refId": "C" + } + ], + "title": "ME reconnect and reader EOF rates", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rates of Middle-End (ME) route drops by reason and total crypto desync detections.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Drop no_conn/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Drop channel_closed/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Drop queue_full/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Desync/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 41 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_route_drop_no_conn_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Drop no_conn/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_route_drop_channel_closed_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Drop channel_closed/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_route_drop_queue_full_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Drop queue_full/s", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_desync_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Desync/s", + "refId": "D" + } + ], + "title": "ME route drops and crypto desync", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rate of ME handshake rejects, including error-code breakdown.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "reject total/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 45 + }, + "id": 353, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_handshake_reject_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "reject total/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum by (error_code) (rate(telemt_me_handshake_error_code_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "Code {{error_code}}/s", + "refId": "B" + } + ], + "title": "ME handshake reject rate by code", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current counts of active and warm ME writers.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Warm" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6B7280", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 49 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(telemt_me_writers_active_current{job=~\"$job\"})", + "legendFormat": "Active", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(telemt_me_writers_warm_current{job=~\"$job\"})", + "legendFormat": "Warm", + "refId": "B" + } + ], + "title": "ME writers state", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Keepalive sent, pong, failed and timeout rates for ME.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "sent/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "pong/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timeout/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 49 + }, + "id": 313, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_keepalive_sent_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "sent/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_keepalive_pong_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "pong/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_keepalive_failed_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "failed/s", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_keepalive_timeout_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "timeout/s", + "refId": "D" + } + ], + "title": "ME keepalive health", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rates of ME pool forced closes and refill events.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "forced close/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "refill triggered/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "refill failed/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 314, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_pool_force_close_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "forced close/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_refill_triggered_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "refill triggered/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_refill_failed_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "refill failed/s", + "refId": "C" + } + ], + "title": "ME refill activity", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current ME capacity target, active writers, and warm writers.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "target writers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "active writers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "warm writers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6B7280", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 315, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max(telemt_me_adaptive_floor_target_writers_total{job=~\"$job\"})", + "legendFormat": "target writers", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(telemt_me_writers_active_current{job=~\"$job\"})", + "legendFormat": "active writers", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(telemt_me_writers_warm_current{job=~\"$job\"})", + "legendFormat": "warm writers", + "refId": "C" + } + ], + "title": "ME capacity", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 355, + "panels": [], + "title": "ME debug metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rates of ME debug D2C batch-byte buckets.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "0-1 KiB/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "1-4 KiB/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4-16 KiB/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "16-64 KiB/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "64-128 KiB/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F97316", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": ">128 KiB/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 66 + }, + "id": 356, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_batch_bytes_bucket_total{bucket=\"0_1k\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "0-1 KiB/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_batch_bytes_bucket_total{bucket=\"1k_4k\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "1-4 KiB/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_batch_bytes_bucket_total{bucket=\"4k_16k\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "4-16 KiB/s", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_batch_bytes_bucket_total{bucket=\"16k_64k\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "16-64 KiB/s", + "refId": "D" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_batch_bytes_bucket_total{bucket=\"64k_128k\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "64-128 KiB/s", + "refId": "E" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_batch_bytes_bucket_total{bucket=\"gt_128k\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": ">128 KiB/s", + "refId": "F" + } + ], + "title": "ME debug D2C batch bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rates of ME debug D2C batch-size buckets.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1 frame/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2-4 frames/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5-8 frames/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "9-16 frames/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "17-32 frames/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F97316", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": ">32 frames/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 66 + }, + "id": 357, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_batch_frames_bucket_total{bucket=\"1\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "1 frame/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_batch_frames_bucket_total{bucket=\"2_4\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "2-4 frames/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_batch_frames_bucket_total{bucket=\"5_8\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "5-8 frames/s", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_batch_frames_bucket_total{bucket=\"9_16\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "9-16 frames/s", + "refId": "D" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_batch_frames_bucket_total{bucket=\"17_32\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "17-32 frames/s", + "refId": "E" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_batch_frames_bucket_total{bucket=\"gt_32\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": ">32 frames/s", + "refId": "F" + } + ], + "title": "ME debug D2C batch size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rates of ME debug D2C flush-duration buckets.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "0-50 us/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "51-200 us/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "201-1000 us/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "1-5 ms/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5-20 ms/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F97316", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": ">20 ms/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 66 + }, + "id": 358, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_flush_duration_us_bucket_total{bucket=\"0_50\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "0-50 us/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_flush_duration_us_bucket_total{bucket=\"51_200\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "51-200 us/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_flush_duration_us_bucket_total{bucket=\"201_1000\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "201-1000 us/s", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_flush_duration_us_bucket_total{bucket=\"1001_5000\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "1-5 ms/s", + "refId": "D" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_flush_duration_us_bucket_total{bucket=\"5001_20000\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "5-20 ms/s", + "refId": "E" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_d2c_flush_duration_us_bucket_total{bucket=\"gt_20000\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": ">20 ms/s", + "refId": "F" + } + ], + "title": "ME debug D2C flush duration", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 350, + "panels": [], + "title": "Access and limits", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current active and recent user/IP tracker sizes and cleanup queue depth.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "users active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "users recent" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6B7280", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "IPs active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "IPs recent" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6B7280", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cleanup queue" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 351, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max(telemt_ip_tracker_users{scope=\"active\",job=~\"$job\"})", + "legendFormat": "users active", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "max(telemt_ip_tracker_users{scope=\"recent\",job=~\"$job\"})", + "legendFormat": "users recent", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "max(telemt_ip_tracker_entries{scope=\"active\",job=~\"$job\"})", + "legendFormat": "IPs active", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "max(telemt_ip_tracker_entries{scope=\"recent\",job=~\"$job\"})", + "legendFormat": "IPs recent", + "refId": "D" + }, + { + "editorMode": "code", + "expr": "max(telemt_ip_tracker_cleanup_queue_len{job=~\"$job\"})", + "legendFormat": "cleanup queue", + "refId": "E" + } + ], + "title": "IP tracker state", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "IP reservation rollbacks in the selected time range.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 75 + }, + "id": 352, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(increase(telemt_ip_reservation_rollback_total{job=~\"$job\"}[$__range]))", + "refId": "A" + } + ], + "title": "IP reservation rollbacks", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Reservation rollback rate by reason.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "tcp_limit/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "quota_limit/s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 79 + }, + "id": 354, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_ip_reservation_rollback_total{reason=\"tcp_limit\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "tcp_limit/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_ip_reservation_rollback_total{reason=\"quota_limit\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "quota_limit/s", + "refId": "B" + } + ], + "title": "IP reservation rollback rate by reason", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current per-user unique IP utilization percentage.", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "red", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 23, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sort_desc(100 * max by (user) (telemt_user_unique_ips_utilization{job=~\"$job\",user=~\"$user\"}))", + "instant": true, + "legendFormat": "{{user}}", + "refId": "A" + } + ], + "title": "User unique IP utilization", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current unique IP count per selected user.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 21, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (user) (telemt_user_unique_ips_current{job=~\"$job\",user=~\"$user\"})", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "title": "User unique IPs", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "renameByName": { + "Value": "Unique IPs", + "user": "User" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + "Unique IPs": true + }, + "sort": [ + { + "desc": true, + "field": "Unique IPs" + } + ] + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 91 + }, + "id": 300, + "panels": [], + "title": "Users", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-user active connections, traffic totals, message rates, and traffic rates.", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "In Total" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Out Total" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "In msg/s" + }, + "properties": [ + { + "id": "unit", + "value": "pps" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Out msg/s" + }, + "properties": [ + { + "id": "unit", + "value": "pps" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "In traffic" + }, + "properties": [ + { + "id": "unit", + "value": "binBps" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Out traffic" + }, + "properties": [ + { + "id": "unit", + "value": "binBps" + } + ] + } + ] + }, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 92 + }, + "id": 16, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Active Connections" + } + ] + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (user) (telemt_user_connections_current{job=~\"$job\",user=~\"$user\"})", + "format": "table", + "instant": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum by (user) (telemt_user_octets_from_client{job=~\"$job\",user=~\"$user\"})", + "format": "table", + "instant": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum by (user) (telemt_user_octets_to_client{job=~\"$job\",user=~\"$user\"})", + "format": "table", + "instant": true, + "refId": "C" + }, + { + "editorMode": "code", + "expr": "sum by (user) (rate(telemt_user_msgs_from_client{job=~\"$job\",user=~\"$user\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "refId": "D" + }, + { + "editorMode": "code", + "expr": "sum by (user) (rate(telemt_user_msgs_to_client{job=~\"$job\",user=~\"$user\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "refId": "E" + }, + { + "editorMode": "code", + "expr": "sum by (user) (rate(telemt_user_octets_from_client{job=~\"$job\",user=~\"$user\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "refId": "F" + }, + { + "editorMode": "code", + "expr": "sum by (user) (rate(telemt_user_octets_to_client{job=~\"$job\",user=~\"$user\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "refId": "G" + } + ], + "title": "User connections, traffic, and message rates", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "user", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time #A": true, + "Time #B": true, + "Time #C": true, + "Time #D": true, + "Time #E": true, + "Time #F": true, + "Time #G": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true + }, + "renameByName": { + "Value": "Active Connections", + "Value #A": "Active Connections", + "Value #B": "In Total", + "Value #C": "Out Total", + "Value #D": "In msg/s", + "Value #E": "Out msg/s", + "Value #F": "In traffic", + "Value #G": "Out traffic", + "Value 1": "In Total", + "Value 2": "Out Total", + "Value 3": "In msg/s", + "Value 4": "Out msg/s", + "Value 5": "In traffic", + "Value 6": "Out traffic", + "user": "User" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": { + "Active Connections": true + }, + "sort": [ + { + "desc": true, + "field": "Active Connections" + } + ] + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 108 + }, + "id": 401, + "panels": [], + "title": "Traffic control", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current active relay leases under rate limiting, split by user and CIDR scope.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 109 + }, + "id": 402, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(telemt_rate_limiter_active_leases{scope=\"user\",job=~\"$job\"})", + "instant": true, + "legendFormat": "user leases", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(telemt_rate_limiter_active_leases{scope=\"cidr\",job=~\"$job\"})", + "instant": true, + "legendFormat": "cidr leases", + "refId": "B" + } + ], + "title": "Active rate-limited leases", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Configured active rate-limit policy entries, split by user and CIDR scope.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 109 + }, + "id": 403, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(telemt_rate_limiter_policy_entries{scope=\"user\",job=~\"$job\"})", + "instant": true, + "legendFormat": "user policies", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(telemt_rate_limiter_policy_entries{scope=\"cidr\",job=~\"$job\"})", + "instant": true, + "legendFormat": "cidr policies", + "refId": "B" + } + ], + "title": "Rate-limit policy entries", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second throttle events produced by the traffic limiter, split by scope and direction.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 109 + }, + "id": 404, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_rate_limiter_throttle_total{scope=\"user\",direction=\"up\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "user up/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_rate_limiter_throttle_total{scope=\"user\",direction=\"down\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "user down/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_rate_limiter_throttle_total{scope=\"cidr\",direction=\"up\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "cidr up/s", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_rate_limiter_throttle_total{scope=\"cidr\",direction=\"down\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "cidr down/s", + "refId": "D" + } + ], + "title": "Throttle events", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Accumulated wait time introduced by rate limiting, split by scope and direction.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 113 + }, + "id": 405, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_rate_limiter_wait_ms_total{scope=\"user\",direction=\"up\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "user up ms/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_rate_limiter_wait_ms_total{scope=\"user\",direction=\"down\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "user down ms/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_rate_limiter_wait_ms_total{scope=\"cidr\",direction=\"up\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "cidr up ms/s", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_rate_limiter_wait_ms_total{scope=\"cidr\",direction=\"down\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "cidr down ms/s", + "refId": "D" + } + ], + "title": "Throttling wait accumulation", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 117 + }, + "id": 410, + "panels": [], + "title": "ME flow control", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Worker-local fairness pressure state gauge.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 122 + }, + "id": 411, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": false + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max(telemt_me_fair_pressure_state{job=~\"$job\"})", + "instant": true, + "refId": "A" + } + ], + "title": "Fair pressure state", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current number of active flows in the ME fair scheduler.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 122 + }, + "id": 412, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max(telemt_me_fair_active_flows{job=~\"$job\"})", + "instant": true, + "refId": "A" + } + ], + "title": "Fair active flows", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current amount of queued data buffered by the ME fair scheduler.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 126 + }, + "id": 413, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max(telemt_me_fair_queued_bytes{job=~\"$job\"})", + "instant": true, + "refId": "A" + } + ], + "title": "Fair queued bytes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current counts of standing versus backpressured ME fair-scheduler flows.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 126 + }, + "id": 414, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max(telemt_me_fair_flow_state_gauge{class=\"standing\",job=~\"$job\"})", + "instant": true, + "legendFormat": "standing", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "max(telemt_me_fair_flow_state_gauge{class=\"backpressured\",job=~\"$job\"})", + "instant": true, + "legendFormat": "backpressured", + "refId": "B" + } + ], + "title": "Fair flow state", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rates of high-signal fair-scheduler overload and backpressure events.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 4, + "y": 118 + }, + "id": 415, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_fair_events_total{event=\"enqueue_reject\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "enqueue reject/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_fair_events_total{event=\"shed_drop\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "shed drop/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_fair_events_total{event=\"downstream_stall\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "downstream stall/s", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_me_fair_events_total{event=\"penalty\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "penalty/s", + "refId": "D" + } + ], + "title": "Fair scheduler overload events", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rates of relay idle marking, hard idle close, pressure eviction, and protocol-desync closure events.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 118 + }, + "id": 416, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_relay_idle_soft_mark_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "soft mark/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_relay_idle_hard_close_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "hard close/s", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_relay_pressure_evict_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "pressure evict/s", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_relay_protocol_desync_close_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "desync close/s", + "refId": "D" + } + ], + "title": "Relay lifecycle and pressure", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 130 + }, + "id": 420, + "panels": [], + "title": "Access and limits expansion", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rejects caused by hitting global IP tracker capacity limits, split by active versus recent scope.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 131 + }, + "id": 421, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_ip_tracker_cap_rejects_total{scope=\"active\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "active cap reject/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_ip_tracker_cap_rejects_total{scope=\"recent\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "recent cap reject/s", + "refId": "B" + } + ], + "title": "IP tracker cap rejects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second deferred cleanup releases drained through the IP tracker cleanup queue.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 131 + }, + "id": 422, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_ip_tracker_cleanup_total{path=\"deferred\",job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "deferred cleanup/s", + "refId": "A" + } + ], + "title": "Deferred cleanup releases", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 139 + }, + "id": 430, + "panels": [], + "title": "TLS and telemetry capacity", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current adaptive TLS fetch profile-cache entry count.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 144 + }, + "id": 431, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(telemt_tls_fetch_profile_cache_entries{job=~\"$job\"})", + "instant": true, + "refId": "A" + } + ], + "title": "TLS profile cache entries", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current number of IPs tracked by the TLS full-certificate budget.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 144 + }, + "id": 432, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(telemt_tls_front_full_cert_budget_ips{job=~\"$job\"})", + "instant": true, + "refId": "A" + } + ], + "title": "TLS full-cert budget IPs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current number of retained per-user stats entries in memory.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 14, + "y": 140 + }, + "id": 433, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(telemt_stats_user_entries{job=~\"$job\"})", + "instant": true, + "refId": "A" + } + ], + "title": "Retained user stats", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Runtime core and per-user telemetry switches exported by telemt.", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 14, + "y": 144 + }, + "id": 434, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "min(telemt_telemetry_core_enabled{job=~\"$job\"})", + "instant": true, + "legendFormat": "core enabled", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "min(telemt_telemetry_user_enabled{job=~\"$job\"})", + "instant": true, + "legendFormat": "user enabled", + "refId": "B" + } + ], + "title": "Telemetry switches", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current emitted and suppressed user-labeled series counts for stats and unique-IP families.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 140 + }, + "id": 435, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(telemt_telemetry_user_series_users{family=\"stats\",status=\"emitted\",job=~\"$job\"})", + "instant": true, + "legendFormat": "stats emitted", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(telemt_telemetry_user_series_users{family=\"stats\",status=\"suppressed\",job=~\"$job\"})", + "instant": true, + "legendFormat": "stats suppressed", + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(telemt_telemetry_user_series_users{family=\"unique_ip\",status=\"emitted\",job=~\"$job\"})", + "instant": true, + "legendFormat": "IP emitted", + "refId": "C" + }, + { + "editorMode": "code", + "expr": "sum(telemt_telemetry_user_series_users{family=\"unique_ip\",status=\"suppressed\",job=~\"$job\"})", + "instant": true, + "legendFormat": "IP suppressed", + "refId": "D" + } + ], + "title": "User-labeled series export", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Per-second rates of TLS cache and full-certificate budget cap-drop events.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 4, + "y": 140 + }, + "id": 436, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(telemt_tls_fetch_profile_cache_cap_drops_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "profile cache cap drop/s", + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(rate(telemt_tls_front_full_cert_budget_cap_drops_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "full-cert budget cap drop/s", + "refId": "B" + } + ], + "title": "TLS capacity cap drops", + "type": "timeseries" + } ], "preload": false, "refresh": "30s", "schemaVersion": 42, "tags": [ - "telemt", - "mtproto", - "telegram" + "telemt", + "mtproto", + "telegram" ], "templating": { - "list": [ - { - "label": "Datasource", - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "type": "datasource" - }, - { - "allValue": ".*", - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(telemt_build_info, job)", - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "options": [], - "query": { - "query": "label_values(telemt_build_info, job)", - "refId": "VarJob" - }, - "refresh": 1, - "regexApplyTo": "value", - "type": "query" - }, - { - "allValue": ".*", - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(telemt_user_connections_total{job=~\"$job\"}, user)", - "includeAll": true, - "label": "User", - "multi": true, - "name": "user", - "options": [], - "query": { - "query": "label_values(telemt_user_connections_total{job=~\"$job\"}, user)", - "refId": "VarUser" - }, - "refresh": 1, - "regexApplyTo": "value", - "type": "query" - } - ] + "list": [ + { + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "type": "datasource" + }, + { + "allValue": ".*", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(telemt_build_info, job)", + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": { + "query": "label_values(telemt_build_info, job)", + "refId": "VarJob" + }, + "refresh": 1, + "regexApplyTo": "value", + "type": "query" + }, + { + "allValue": ".*", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(telemt_user_connections_total{job=~\"$job\"}, user)", + "includeAll": true, + "label": "User", + "multi": true, + "name": "user", + "options": [], + "query": { + "query": "label_values(telemt_user_connections_total{job=~\"$job\"}, user)", + "refId": "VarUser" + }, + "refresh": 1, + "regexApplyTo": "value", + "type": "query" + } + ] }, "time": { - "from": "now-1h", - "to": "now" + "from": "now-1h", + "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Telemt MtProto proxy", "weekStart": "" - } -} \ No newline at end of file +} From a7a2f4ab2715ed1ac6789cf43a18989d2a066583 Mon Sep 17 00:00:00 2001 From: mamuthus Date: Fri, 1 May 2026 19:19:00 +0000 Subject: [PATCH 2/5] Adjust General metrics dashboard layout --- tools/grafana-dashboard.json | 44 ++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tools/grafana-dashboard.json b/tools/grafana-dashboard.json index 1ca898a..60c7659 100644 --- a/tools/grafana-dashboard.json +++ b/tools/grafana-dashboard.json @@ -767,7 +767,7 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 4, "x": 12, "y": 5 }, @@ -777,12 +777,12 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "hideZeros": true, + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "12.4.2", @@ -812,7 +812,7 @@ "refId": "D" } ], - "title": "Connection and permit rates", + "title": "Connection / permit rates", "type": "timeseries" }, { @@ -1070,10 +1070,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 9 + "h": 8, + "w": 4, + "x": 16, + "y": 5 }, "id": 437, "options": { @@ -1081,12 +1081,12 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "hideZeros": true, + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "12.4.2", @@ -1165,10 +1165,10 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 9 + "h": 8, + "w": 4, + "x": 20, + "y": 5 }, "id": 438, "options": { @@ -1176,12 +1176,12 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "hideZeros": true, + "mode": "multi", + "sort": "desc" } }, "pluginVersion": "12.4.2", From 9f9ca9f2702f28bb726d081805d0ff249e2b9795 Mon Sep 17 00:00:00 2001 From: astronaut808 <38975427+astronaut808@users.noreply.github.com> Date: Sun, 3 May 2026 18:06:59 +0500 Subject: [PATCH 3/5] Add TLS front profile health metrics --- .../TLS_FRONT_PROFILE_FIDELITY.en.md | 49 +++- .../TLS_FRONT_PROFILE_FIDELITY.ru.md | 51 +++- src/maestro/mod.rs | 1 + src/maestro/runtime_tasks.rs | 4 + src/metrics.rs | 255 +++++++++++++++++- src/tls_front/cache.rs | 71 ++++- 6 files changed, 416 insertions(+), 15 deletions(-) diff --git a/docs/Architecture/Fronting-splitting/TLS_FRONT_PROFILE_FIDELITY.en.md b/docs/Architecture/Fronting-splitting/TLS_FRONT_PROFILE_FIDELITY.en.md index 21d6558..4c445ad 100644 --- a/docs/Architecture/Fronting-splitting/TLS_FRONT_PROFILE_FIDELITY.en.md +++ b/docs/Architecture/Fronting-splitting/TLS_FRONT_PROFILE_FIDELITY.en.md @@ -128,7 +128,48 @@ Recommended for cleaner testing: Persisted cache artifacts are useful, but they are not required if packet captures already demonstrate the runtime result. -### 4. Capture a direct-origin trace +### 4. Check TLS-front profile health metrics + +If the metrics endpoint is enabled, check the TLS-front profile health before packet-capture validation: + +```bash +curl -s http://127.0.0.1:9999/metrics | grep -E 'telemt_tls_front_profile|telemt_tls_fetch_profile_cache|telemt_tls_front_full_cert' +``` + +The profile-health metrics expose the runtime state of configured TLS front domains: + +- `telemt_tls_front_profile_domains` shows configured, emitted, and suppressed domain series. +- `telemt_tls_front_profile_info` shows profile source and feature flags per domain. +- `telemt_tls_front_profile_age_seconds` shows cached profile age. +- `telemt_tls_front_profile_app_data_records` shows cached AppData record count. +- `telemt_tls_front_profile_ticket_records` shows cached ticket-like tail record count. +- `telemt_tls_front_profile_change_cipher_spec_records` shows cached ChangeCipherSpec count. +- `telemt_tls_front_profile_app_data_bytes` shows total cached AppData bytes. + +Interpretation: + +- `source="merged"` or `source="raw"` means real TLS profile data is being used. +- `source="default"` or `is_default="true"` means the domain currently uses the synthetic default fallback. +- `has_cert_payload="true"` means certificate payload data is available for TLS emulation. +- Non-zero AppData/ticket/CCS counters show captured server-flight shape. + +Example healthy output: + +```text +telemt_tls_front_profile_domains{status="configured"} 1 +telemt_tls_front_profile_domains{status="emitted"} 1 +telemt_tls_front_profile_domains{status="suppressed"} 0 +telemt_tls_front_profile_info{domain="itunes.apple.com",source="merged",is_default="false",has_cert_info="true",has_cert_payload="true"} 1 +telemt_tls_front_profile_age_seconds{domain="itunes.apple.com"} 20 +telemt_tls_front_profile_app_data_records{domain="itunes.apple.com"} 3 +telemt_tls_front_profile_ticket_records{domain="itunes.apple.com"} 1 +telemt_tls_front_profile_change_cipher_spec_records{domain="itunes.apple.com"} 1 +telemt_tls_front_profile_app_data_bytes{domain="itunes.apple.com"} 5240 +``` + +These metrics do not prove byte-level origin equivalence. They are an operational health signal that the configured domain is backed by real cached profile data instead of default fallback data. + +### 5. Capture a direct-origin trace From a separate client host, connect directly to the origin: @@ -142,7 +183,7 @@ Capture with: sudo tcpdump -i any -w origin-direct.pcap host ORIGIN_IP and port 443 ``` -### 5. Capture a Telemt FakeTLS success-path trace +### 6. Capture a Telemt FakeTLS success-path trace Now connect to Telemt with a real Telegram client through an `ee` proxy link that targets the Telemt instance. @@ -154,7 +195,7 @@ Capture with: sudo tcpdump -i any -w telemt-emulated.pcap host TELEMT_IP and port 443 ``` -### 6. Decode TLS record structure +### 7. Decode TLS record structure Use `tshark` to print record-level structure: @@ -182,7 +223,7 @@ Focus on the server flight after ClientHello: - `20` = ChangeCipherSpec - `23` = ApplicationData -### 7. Build a comparison table +### 8. Build a comparison table A compact table like the following is usually enough: diff --git a/docs/Architecture/Fronting-splitting/TLS_FRONT_PROFILE_FIDELITY.ru.md b/docs/Architecture/Fronting-splitting/TLS_FRONT_PROFILE_FIDELITY.ru.md index be68ad7..b0214fc 100644 --- a/docs/Architecture/Fronting-splitting/TLS_FRONT_PROFILE_FIDELITY.ru.md +++ b/docs/Architecture/Fronting-splitting/TLS_FRONT_PROFILE_FIDELITY.ru.md @@ -126,9 +126,50 @@ openssl s_client -connect ORIGIN_IP:443 -servername YOUR_DOMAIN , shared_state: Arc, ip_tracker: Arc, + tls_cache: Option>, config_rx: watch::Receiver>, ) { // metrics_listen takes precedence; fall back to metrics_port for backward compat. @@ -363,6 +365,7 @@ pub(crate) async fn spawn_metrics_if_configured( let shared_state = shared_state.clone(); let config_rx_metrics = config_rx.clone(); let ip_tracker_metrics = ip_tracker.clone(); + let tls_cache_metrics = tls_cache.clone(); let whitelist = config.server.metrics_whitelist.clone(); let listen_backlog = config.server.listen_backlog; tokio::spawn(async move { @@ -374,6 +377,7 @@ pub(crate) async fn spawn_metrics_if_configured( beobachten, shared_state, ip_tracker_metrics, + tls_cache_metrics, config_rx_metrics, whitelist, ) diff --git a/src/metrics.rs b/src/metrics.rs index 670d0e0..e100d2d 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -18,12 +18,15 @@ use crate::ip_tracker::UserIpTracker; use crate::proxy::shared_state::ProxySharedState; use crate::stats::Stats; use crate::stats::beobachten::BeobachtenStore; +use crate::tls_front::TlsFrontCache; use crate::tls_front::cache; use crate::tls_front::fetcher; use crate::transport::{ListenOptions, create_listener}; // Keeps `/metrics` response size bounded when per-user telemetry is enabled. const USER_LABELED_METRICS_MAX_USERS: usize = 4096; +// Keeps TLS-front per-domain health series bounded for large generated configs. +const TLS_FRONT_PROFILE_HEALTH_MAX_DOMAINS: usize = 256; pub async fn serve( port: u16, @@ -33,6 +36,7 @@ pub async fn serve( beobachten: Arc, shared_state: Arc, ip_tracker: Arc, + tls_cache: Option>, config_rx: tokio::sync::watch::Receiver>, whitelist: Vec, ) { @@ -57,6 +61,7 @@ pub async fn serve( beobachten, shared_state, ip_tracker, + tls_cache, config_rx, whitelist, ) @@ -112,6 +117,7 @@ pub async fn serve( beobachten, shared_state, ip_tracker, + tls_cache, config_rx, whitelist, ) @@ -122,6 +128,7 @@ pub async fn serve( let beobachten_v6 = beobachten.clone(); let shared_state_v6 = shared_state.clone(); let ip_tracker_v6 = ip_tracker.clone(); + let tls_cache_v6 = tls_cache.clone(); let config_rx_v6 = config_rx.clone(); let whitelist_v6 = whitelist.clone(); tokio::spawn(async move { @@ -131,6 +138,7 @@ pub async fn serve( beobachten_v6, shared_state_v6, ip_tracker_v6, + tls_cache_v6, config_rx_v6, whitelist_v6, ) @@ -142,6 +150,7 @@ pub async fn serve( beobachten, shared_state, ip_tracker, + tls_cache, config_rx, whitelist, ) @@ -171,6 +180,7 @@ async fn serve_listener( beobachten: Arc, shared_state: Arc, ip_tracker: Arc, + tls_cache: Option>, config_rx: tokio::sync::watch::Receiver>, whitelist: Arc>, ) { @@ -192,6 +202,7 @@ async fn serve_listener( let beobachten = beobachten.clone(); let shared_state = shared_state.clone(); let ip_tracker = ip_tracker.clone(); + let tls_cache = tls_cache.clone(); let config_rx_conn = config_rx.clone(); tokio::spawn(async move { let svc = service_fn(move |req| { @@ -199,6 +210,7 @@ async fn serve_listener( let beobachten = beobachten.clone(); let shared_state = shared_state.clone(); let ip_tracker = ip_tracker.clone(); + let tls_cache = tls_cache.clone(); let config = config_rx_conn.borrow().clone(); async move { handle( @@ -207,6 +219,7 @@ async fn serve_listener( &beobachten, &shared_state, &ip_tracker, + tls_cache.as_deref(), &config, ) .await @@ -228,10 +241,11 @@ async fn handle( beobachten: &BeobachtenStore, shared_state: &ProxySharedState, ip_tracker: &UserIpTracker, + tls_cache: Option<&TlsFrontCache>, config: &ProxyConfig, ) -> Result>, Infallible> { if req.uri().path() == "/metrics" { - let body = render_metrics(stats, shared_state, config, ip_tracker).await; + let body = render_metrics(stats, shared_state, config, ip_tracker, tls_cache).await; let resp = Response::builder() .status(StatusCode::OK) .header("content-type", "text/plain; version=0.0.4; charset=utf-8") @@ -266,11 +280,151 @@ fn render_beobachten(beobachten: &BeobachtenStore, config: &ProxyConfig) -> Stri beobachten.snapshot_text(ttl) } +fn tls_front_domains(config: &ProxyConfig) -> Vec { + let mut domains = Vec::with_capacity(1 + config.censorship.tls_domains.len()); + if !config.censorship.tls_domain.is_empty() { + domains.push(config.censorship.tls_domain.clone()); + } + for domain in &config.censorship.tls_domains { + if !domain.is_empty() && !domains.contains(domain) { + domains.push(domain.clone()); + } + } + domains +} + +fn prometheus_label_value(value: &str) -> String { + value.replace('\\', "\\\\").replace('"', "\\\"") +} + +async fn render_tls_front_profile_health( + out: &mut String, + config: &ProxyConfig, + tls_cache: Option<&TlsFrontCache>, +) { + use std::fmt::Write; + + let domains = tls_front_domains(config); + let (health, suppressed) = match (config.censorship.tls_emulation, tls_cache) { + (true, Some(cache)) => { + cache + .profile_health_snapshot(&domains, TLS_FRONT_PROFILE_HEALTH_MAX_DOMAINS) + .await + } + _ => (Vec::new(), domains.len()), + }; + + let _ = writeln!( + out, + "# HELP telemt_tls_front_profile_domains TLS front configured profile domains by export status" + ); + let _ = writeln!(out, "# TYPE telemt_tls_front_profile_domains gauge"); + let _ = writeln!( + out, + "telemt_tls_front_profile_domains{{status=\"configured\"}} {}", + domains.len() + ); + let _ = writeln!( + out, + "telemt_tls_front_profile_domains{{status=\"emitted\"}} {}", + health.len() + ); + let _ = writeln!( + out, + "telemt_tls_front_profile_domains{{status=\"suppressed\"}} {}", + suppressed + ); + let _ = writeln!( + out, + "# HELP telemt_tls_front_profile_info TLS front profile source and feature flags per configured domain" + ); + let _ = writeln!(out, "# TYPE telemt_tls_front_profile_info gauge"); + let _ = writeln!( + out, + "# HELP telemt_tls_front_profile_age_seconds Age of cached TLS front profile data per configured domain" + ); + let _ = writeln!( + out, + "# TYPE telemt_tls_front_profile_age_seconds gauge" + ); + let _ = writeln!( + out, + "# HELP telemt_tls_front_profile_app_data_records TLS front cached app-data record count per configured domain" + ); + let _ = writeln!( + out, + "# TYPE telemt_tls_front_profile_app_data_records gauge" + ); + let _ = writeln!( + out, + "# HELP telemt_tls_front_profile_ticket_records TLS front cached ticket-like tail record count per configured domain" + ); + let _ = writeln!( + out, + "# TYPE telemt_tls_front_profile_ticket_records gauge" + ); + let _ = writeln!( + out, + "# HELP telemt_tls_front_profile_change_cipher_spec_records TLS front cached ChangeCipherSpec record count per configured domain" + ); + let _ = writeln!( + out, + "# TYPE telemt_tls_front_profile_change_cipher_spec_records gauge" + ); + let _ = writeln!( + out, + "# HELP telemt_tls_front_profile_app_data_bytes TLS front cached total app-data bytes per configured domain" + ); + let _ = writeln!( + out, + "# TYPE telemt_tls_front_profile_app_data_bytes gauge" + ); + + for item in health { + let domain = prometheus_label_value(&item.domain); + let _ = writeln!( + out, + "telemt_tls_front_profile_info{{domain=\"{}\",source=\"{}\",is_default=\"{}\",has_cert_info=\"{}\",has_cert_payload=\"{}\"}} 1", + domain, + item.source, + item.is_default, + item.has_cert_info, + item.has_cert_payload + ); + let _ = writeln!( + out, + "telemt_tls_front_profile_age_seconds{{domain=\"{}\"}} {}", + domain, item.age_seconds + ); + let _ = writeln!( + out, + "telemt_tls_front_profile_app_data_records{{domain=\"{}\"}} {}", + domain, item.app_data_records + ); + let _ = writeln!( + out, + "telemt_tls_front_profile_ticket_records{{domain=\"{}\"}} {}", + domain, item.ticket_records + ); + let _ = writeln!( + out, + "telemt_tls_front_profile_change_cipher_spec_records{{domain=\"{}\"}} {}", + domain, item.change_cipher_spec_count + ); + let _ = writeln!( + out, + "telemt_tls_front_profile_app_data_bytes{{domain=\"{}\"}} {}", + domain, item.total_app_data_len + ); + } +} + async fn render_metrics( stats: &Stats, shared_state: &ProxySharedState, config: &ProxyConfig, ip_tracker: &UserIpTracker, + tls_cache: Option<&TlsFrontCache>, ) -> String { use std::fmt::Write; let mut out = String::with_capacity(4096); @@ -423,6 +577,7 @@ async fn render_metrics( "telemt_tls_front_full_cert_budget_cap_drops_total {}", cache::full_cert_sent_cap_drops_for_metrics() ); + render_tls_front_profile_health(&mut out, config, tls_cache).await; let _ = writeln!( out, @@ -3361,6 +3516,11 @@ mod tests { use super::*; use http_body_util::BodyExt; use std::net::IpAddr; + use std::time::SystemTime; + + use crate::tls_front::types::{ + CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsCertPayload, TlsProfileSource, + }; #[tokio::test] async fn test_render_metrics_format() { @@ -3429,7 +3589,7 @@ mod tests { .await .unwrap(); - let output = render_metrics(&stats, shared_state.as_ref(), &config, &tracker).await; + let output = render_metrics(&stats, shared_state.as_ref(), &config, &tracker, None).await; assert!(output.contains(&format!( "telemt_build_info{{version=\"{}\"}} 1", @@ -3494,13 +3654,86 @@ mod tests { assert!(output.contains("telemt_ip_tracker_cleanup_queue_len 0")); } + #[tokio::test] + async fn test_render_tls_front_profile_health() { + let stats = Stats::new(); + let shared_state = ProxySharedState::new(); + let tracker = UserIpTracker::new(); + let mut config = ProxyConfig::default(); + config.censorship.tls_domain = "primary.example".to_string(); + config.censorship.tls_domains = vec!["fallback.example".to_string()]; + + let cache = TlsFrontCache::new( + &[ + "primary.example".to_string(), + "fallback.example".to_string(), + ], + 1024, + "tlsfront-profile-health-test", + ); + cache + .set( + "primary.example", + CachedTlsData { + server_hello_template: ParsedServerHello { + version: [0x03, 0x03], + random: [0u8; 32], + session_id: Vec::new(), + cipher_suite: [0x13, 0x01], + compression: 0, + extensions: Vec::new(), + }, + cert_info: None, + cert_payload: Some(TlsCertPayload { + cert_chain_der: vec![vec![0x30, 0x01]], + certificate_message: vec![0x0b, 0x00, 0x00, 0x00], + }), + app_data_records_sizes: vec![1024, 512], + total_app_data_len: 1536, + behavior_profile: TlsBehaviorProfile { + change_cipher_spec_count: 1, + app_data_record_sizes: vec![1024, 512], + ticket_record_sizes: vec![69], + source: TlsProfileSource::Merged, + }, + fetched_at: SystemTime::now(), + domain: "primary.example".to_string(), + }, + ) + .await; + + let output = render_metrics(&stats, &shared_state, &config, &tracker, Some(&cache)).await; + + assert!(output.contains("telemt_tls_front_profile_domains{status=\"configured\"} 2")); + assert!(output.contains("telemt_tls_front_profile_domains{status=\"emitted\"} 2")); + assert!(output.contains("telemt_tls_front_profile_domains{status=\"suppressed\"} 0")); + assert!( + output.contains("telemt_tls_front_profile_info{domain=\"primary.example\",source=\"merged\",is_default=\"false\",has_cert_info=\"false\",has_cert_payload=\"true\"} 1") + ); + assert!( + output.contains("telemt_tls_front_profile_info{domain=\"fallback.example\",source=\"default\",is_default=\"true\",has_cert_info=\"false\",has_cert_payload=\"false\"} 1") + ); + assert!( + output.contains("telemt_tls_front_profile_app_data_records{domain=\"primary.example\"} 2") + ); + assert!( + output.contains("telemt_tls_front_profile_ticket_records{domain=\"primary.example\"} 1") + ); + assert!( + output.contains("telemt_tls_front_profile_change_cipher_spec_records{domain=\"primary.example\"} 1") + ); + assert!( + output.contains("telemt_tls_front_profile_app_data_bytes{domain=\"primary.example\"} 1536") + ); + } + #[tokio::test] async fn test_render_empty_stats() { let stats = Stats::new(); let shared_state = ProxySharedState::new(); let tracker = UserIpTracker::new(); let config = ProxyConfig::default(); - let output = render_metrics(&stats, &shared_state, &config, &tracker).await; + let output = render_metrics(&stats, &shared_state, &config, &tracker, None).await; assert!(output.contains("telemt_connections_total 0")); assert!(output.contains("telemt_connections_bad_total 0")); assert!(output.contains("telemt_handshake_timeouts_total 0")); @@ -3524,7 +3757,7 @@ mod tests { let mut config = ProxyConfig::default(); config.access.user_max_unique_ips_global_each = 2; - let output = render_metrics(&stats, &shared_state, &config, &tracker).await; + let output = render_metrics(&stats, &shared_state, &config, &tracker, None).await; assert!(output.contains("telemt_user_unique_ips_limit{user=\"alice\"} 2")); assert!(output.contains("telemt_user_unique_ips_utilization{user=\"alice\"} 0.500000")); @@ -3536,7 +3769,7 @@ mod tests { let shared_state = ProxySharedState::new(); let tracker = UserIpTracker::new(); let config = ProxyConfig::default(); - let output = render_metrics(&stats, &shared_state, &config, &tracker).await; + let output = render_metrics(&stats, &shared_state, &config, &tracker, None).await; assert!(output.contains("# TYPE telemt_uptime_seconds gauge")); assert!(output.contains("# TYPE telemt_connections_total counter")); assert!(output.contains("# TYPE telemt_connections_bad_total counter")); @@ -3585,6 +3818,15 @@ mod tests { assert!( output.contains("# TYPE telemt_tls_front_full_cert_budget_cap_drops_total counter") ); + assert!(output.contains("# TYPE telemt_tls_front_profile_domains gauge")); + assert!(output.contains("# TYPE telemt_tls_front_profile_info gauge")); + assert!(output.contains("# TYPE telemt_tls_front_profile_age_seconds gauge")); + assert!(output.contains("# TYPE telemt_tls_front_profile_app_data_records gauge")); + assert!(output.contains("# TYPE telemt_tls_front_profile_ticket_records gauge")); + assert!( + output.contains("# TYPE telemt_tls_front_profile_change_cipher_spec_records gauge") + ); + assert!(output.contains("# TYPE telemt_tls_front_profile_app_data_bytes gauge")); } #[tokio::test] @@ -3605,6 +3847,7 @@ mod tests { &beobachten, shared_state.as_ref(), &tracker, + None, &config, ) .await @@ -3639,6 +3882,7 @@ mod tests { &beobachten, shared_state.as_ref(), &tracker, + None, &config, ) .await @@ -3656,6 +3900,7 @@ mod tests { &beobachten, shared_state.as_ref(), &tracker, + None, &config, ) .await diff --git a/src/tls_front/cache.rs b/src/tls_front/cache.rs index f18084b..dc3d34e 100644 --- a/src/tls_front/cache.rs +++ b/src/tls_front/cache.rs @@ -12,7 +12,7 @@ use tokio::time::sleep; use tracing::{debug, info, warn}; use crate::tls_front::types::{ - CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsFetchResult, + CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsFetchResult, TlsProfileSource, }; const FULL_CERT_SENT_SWEEP_INTERVAL_SECS: u64 = 30; @@ -42,6 +42,30 @@ pub struct TlsFrontCache { disk_path: PathBuf, } +/// Read-only health view for one configured TLS front domain. +#[derive(Debug, Clone)] +pub(crate) struct TlsFrontProfileHealth { + pub(crate) domain: String, + pub(crate) source: &'static str, + pub(crate) age_seconds: u64, + pub(crate) is_default: bool, + pub(crate) has_cert_info: bool, + pub(crate) has_cert_payload: bool, + pub(crate) app_data_records: usize, + pub(crate) ticket_records: usize, + pub(crate) change_cipher_spec_count: u8, + pub(crate) total_app_data_len: usize, +} + +fn profile_source_label(source: TlsProfileSource) -> &'static str { + match source { + TlsProfileSource::Default => "default", + TlsProfileSource::Raw => "raw", + TlsProfileSource::Rustls => "rustls", + TlsProfileSource::Merged => "merged", + } +} + #[allow(dead_code)] impl TlsFrontCache { pub fn new(domains: &[String], default_len: usize, disk_path: impl AsRef) -> Self { @@ -93,6 +117,51 @@ impl TlsFrontCache { self.memory.read().await.contains_key(domain) } + pub(crate) async fn profile_health_snapshot( + &self, + domains: &[String], + max_domains: usize, + ) -> (Vec, usize) { + let guard = self.memory.read().await; + let now = SystemTime::now(); + let mut snapshot = Vec::with_capacity(domains.len().min(max_domains)); + let mut suppressed = 0usize; + + for domain in domains { + if snapshot.len() >= max_domains { + suppressed = suppressed.saturating_add(1); + continue; + } + + let cached = guard + .get(domain) + .cloned() + .unwrap_or_else(|| self.default.clone()); + let behavior = &cached.behavior_profile; + let age_seconds = now + .duration_since(cached.fetched_at) + .map(|duration| duration.as_secs()) + .unwrap_or(0); + + snapshot.push(TlsFrontProfileHealth { + domain: domain.clone(), + source: profile_source_label(behavior.source), + age_seconds, + is_default: cached.domain == "default", + has_cert_info: cached.cert_info.is_some(), + has_cert_payload: cached.cert_payload.is_some(), + app_data_records: cached.app_data_records_sizes.len().max( + behavior.app_data_record_sizes.len(), + ), + ticket_records: behavior.ticket_record_sizes.len(), + change_cipher_spec_count: behavior.change_cipher_spec_count, + total_app_data_len: cached.total_app_data_len, + }); + } + + (snapshot, suppressed) + } + fn full_cert_sent_shard_index(client_ip: IpAddr) -> usize { let mut hasher = DefaultHasher::new(); client_ip.hash(&mut hasher); From e0f251ad827f77d627a33cd22b4ae63fdb1c76dc Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 6 May 2026 20:29:24 +0300 Subject: [PATCH 4/5] TLS Domains masking fixes --- src/proxy/masking.rs | 121 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 116 insertions(+), 5 deletions(-) diff --git a/src/proxy/masking.rs b/src/proxy/masking.rs index c48ec9c..b2a8e84 100644 --- a/src/proxy/masking.rs +++ b/src/proxy/masking.rs @@ -2,6 +2,7 @@ use crate::config::ProxyConfig; use crate::network::dns_overrides::resolve_socket_addr; +use crate::protocol::tls; use crate::stats::beobachten::BeobachtenStore; use crate::transport::proxy_protocol::{ProxyProtocolV1Builder, ProxyProtocolV2Builder}; #[cfg(unix)] @@ -328,6 +329,89 @@ async fn wait_mask_outcome_budget(started: Instant, config: &ProxyConfig) { } } +#[cfg(test)] +mod tls_domain_mask_host_tests { + use super::{mask_host_for_initial_data, matching_tls_domain_for_sni}; + use crate::config::ProxyConfig; + + fn client_hello_with_sni(sni_host: &str) -> Vec { + let mut body = Vec::new(); + body.extend_from_slice(&[0x03, 0x03]); + body.extend_from_slice(&[0u8; 32]); + body.push(32); + body.extend_from_slice(&[0x42u8; 32]); + body.extend_from_slice(&2u16.to_be_bytes()); + body.extend_from_slice(&[0x13, 0x01]); + body.push(1); + body.push(0); + + let host_bytes = sni_host.as_bytes(); + let mut sni_payload = Vec::new(); + sni_payload.extend_from_slice(&((host_bytes.len() + 3) as u16).to_be_bytes()); + sni_payload.push(0); + sni_payload.extend_from_slice(&(host_bytes.len() as u16).to_be_bytes()); + sni_payload.extend_from_slice(host_bytes); + + let mut extensions = Vec::new(); + extensions.extend_from_slice(&0x0000u16.to_be_bytes()); + extensions.extend_from_slice(&(sni_payload.len() as u16).to_be_bytes()); + extensions.extend_from_slice(&sni_payload); + body.extend_from_slice(&(extensions.len() as u16).to_be_bytes()); + body.extend_from_slice(&extensions); + + let mut handshake = Vec::new(); + handshake.push(0x01); + let body_len = (body.len() as u32).to_be_bytes(); + handshake.extend_from_slice(&body_len[1..4]); + handshake.extend_from_slice(&body); + + let mut record = Vec::new(); + record.push(0x16); + record.extend_from_slice(&[0x03, 0x01]); + record.extend_from_slice(&(handshake.len() as u16).to_be_bytes()); + record.extend_from_slice(&handshake); + record + } + + fn config_with_tls_domains() -> ProxyConfig { + let mut config = ProxyConfig::default(); + config.censorship.tls_domain = "a.com".to_string(); + config.censorship.tls_domains = vec!["b.com".to_string(), "c.com".to_string()]; + config.censorship.mask_host = Some("a.com".to_string()); + config + } + + #[test] + fn matching_tls_domain_accepts_primary_and_extra_domains_case_insensitively() { + let config = config_with_tls_domains(); + + assert_eq!(matching_tls_domain_for_sni(&config, "A.COM"), Some("a.com")); + assert_eq!(matching_tls_domain_for_sni(&config, "B.COM"), Some("b.com")); + assert_eq!(matching_tls_domain_for_sni(&config, "unknown.com"), None); + } + + #[test] + fn mask_host_preserves_explicit_non_primary_origin() { + let mut config = config_with_tls_domains(); + config.censorship.mask_host = Some("origin.example".to_string()); + + let initial_data = client_hello_with_sni("b.com"); + + assert_eq!( + mask_host_for_initial_data(&config, &initial_data), + "origin.example" + ); + } + + #[test] + fn mask_host_uses_matching_tls_domain_when_mask_host_is_primary_default() { + let config = config_with_tls_domains(); + let initial_data = client_hello_with_sni("b.com"); + + assert_eq!(mask_host_for_initial_data(&config, &initial_data), "b.com"); + } +} + /// Detect client type based on initial data fn detect_client_type(data: &[u8]) -> &'static str { // Check for HTTP request @@ -360,6 +444,37 @@ fn parse_mask_host_ip_literal(host: &str) -> Option { host.parse::().ok() } +fn matching_tls_domain_for_sni<'a>(config: &'a ProxyConfig, sni: &str) -> Option<&'a str> { + if config.censorship.tls_domain.eq_ignore_ascii_case(sni) { + return Some(config.censorship.tls_domain.as_str()); + } + + for domain in &config.censorship.tls_domains { + if domain.eq_ignore_ascii_case(sni) { + return Some(domain.as_str()); + } + } + + None +} + +fn mask_host_for_initial_data<'a>(config: &'a ProxyConfig, initial_data: &[u8]) -> &'a str { + let configured_mask_host = config + .censorship + .mask_host + .as_deref() + .unwrap_or(&config.censorship.tls_domain); + + if !configured_mask_host.eq_ignore_ascii_case(&config.censorship.tls_domain) { + return configured_mask_host; + } + + tls::extract_sni_from_client_hello(initial_data) + .as_deref() + .and_then(|sni| matching_tls_domain_for_sni(config, sni)) + .unwrap_or(configured_mask_host) +} + fn canonical_ip(ip: IpAddr) -> IpAddr { match ip { IpAddr::V6(v6) => v6 @@ -734,11 +849,7 @@ pub async fn handle_bad_client( return; } - let mask_host = config - .censorship - .mask_host - .as_deref() - .unwrap_or(&config.censorship.tls_domain); + let mask_host = mask_host_for_initial_data(config, initial_data); let mask_port = config.censorship.mask_port; // Fail closed when fallback points at our own listener endpoint. From 4995e83236443842856e56fdb70fd70336bb4589 Mon Sep 17 00:00:00 2001 From: Alexey <247128645+axkurcom@users.noreply.github.com> Date: Wed, 6 May 2026 20:38:55 +0300 Subject: [PATCH 5/5] Config Strict and Validator --- src/config/load.rs | 661 ++++++++++++++++++++++++++++++++++++++++++++ src/config/types.rs | 6 + 2 files changed, 667 insertions(+) diff --git a/src/config/load.rs b/src/config/load.rs index 20faf20..2f2cb55 100644 --- a/src/config/load.rs +++ b/src/config/load.rs @@ -23,6 +23,666 @@ const MAX_ME_C2ME_CHANNEL_CAPACITY: usize = 8_192; const MIN_MAX_CLIENT_FRAME_BYTES: usize = 4 * 1024; const MAX_MAX_CLIENT_FRAME_BYTES: usize = 16 * 1024 * 1024; +const TOP_LEVEL_CONFIG_KEYS: &[&str] = &[ + "general", + "network", + "server", + "timeouts", + "censorship", + "access", + "upstreams", + "show_link", + "dc_overrides", + "default_dc", + "beobachten", + "beobachten_minutes", + "beobachten_flush_secs", + "beobachten_file", + "include", +]; + +const GENERAL_CONFIG_KEYS: &[&str] = &[ + "data_path", + "config_strict", + "modes", + "prefer_ipv6", + "fast_mode", + "use_middle_proxy", + "proxy_secret_path", + "proxy_secret_url", + "proxy_config_v4_cache_path", + "proxy_config_v4_url", + "proxy_config_v6_cache_path", + "proxy_config_v6_url", + "ad_tag", + "middle_proxy_nat_ip", + "middle_proxy_nat_probe", + "middle_proxy_nat_stun", + "middle_proxy_nat_stun_servers", + "stun_nat_probe_concurrency", + "middle_proxy_pool_size", + "middle_proxy_warm_standby", + "me_init_retry_attempts", + "me2dc_fallback", + "me2dc_fast", + "me_keepalive_enabled", + "me_keepalive_interval_secs", + "me_keepalive_jitter_secs", + "me_keepalive_payload_random", + "rpc_proxy_req_every", + "me_writer_cmd_channel_capacity", + "me_route_channel_capacity", + "me_c2me_channel_capacity", + "me_c2me_send_timeout_ms", + "me_reader_route_data_wait_ms", + "me_d2c_flush_batch_max_frames", + "me_d2c_flush_batch_max_bytes", + "me_d2c_flush_batch_max_delay_us", + "me_d2c_ack_flush_immediate", + "me_quota_soft_overshoot_bytes", + "me_d2c_frame_buf_shrink_threshold_bytes", + "direct_relay_copy_buf_c2s_bytes", + "direct_relay_copy_buf_s2c_bytes", + "crypto_pending_buffer", + "max_client_frame", + "desync_all_full", + "beobachten", + "beobachten_minutes", + "beobachten_flush_secs", + "beobachten_file", + "hardswap", + "me_warmup_stagger_enabled", + "me_warmup_step_delay_ms", + "me_warmup_step_jitter_ms", + "me_reconnect_max_concurrent_per_dc", + "me_reconnect_backoff_base_ms", + "me_reconnect_backoff_cap_ms", + "me_reconnect_fast_retry_count", + "me_single_endpoint_shadow_writers", + "me_single_endpoint_outage_mode_enabled", + "me_single_endpoint_outage_disable_quarantine", + "me_single_endpoint_outage_backoff_min_ms", + "me_single_endpoint_outage_backoff_max_ms", + "me_single_endpoint_shadow_rotate_every_secs", + "me_floor_mode", + "me_adaptive_floor_idle_secs", + "me_adaptive_floor_min_writers_single_endpoint", + "me_adaptive_floor_min_writers_multi_endpoint", + "me_adaptive_floor_recover_grace_secs", + "me_adaptive_floor_writers_per_core_total", + "me_adaptive_floor_cpu_cores_override", + "me_adaptive_floor_max_extra_writers_single_per_core", + "me_adaptive_floor_max_extra_writers_multi_per_core", + "me_adaptive_floor_max_active_writers_per_core", + "me_adaptive_floor_max_warm_writers_per_core", + "me_adaptive_floor_max_active_writers_global", + "me_adaptive_floor_max_warm_writers_global", + "upstream_connect_retry_attempts", + "upstream_connect_retry_backoff_ms", + "upstream_connect_budget_ms", + "tg_connect", + "upstream_unhealthy_fail_threshold", + "upstream_connect_failfast_hard_errors", + "stun_iface_mismatch_ignore", + "unknown_dc_log_path", + "unknown_dc_file_log_enabled", + "log_level", + "disable_colors", + "telemetry", + "me_socks_kdf_policy", + "me_route_backpressure_enabled", + "me_route_fairshare_enabled", + "me_route_backpressure_base_timeout_ms", + "me_route_backpressure_high_timeout_ms", + "me_route_backpressure_high_watermark_pct", + "me_health_interval_ms_unhealthy", + "me_health_interval_ms_healthy", + "me_admission_poll_ms", + "me_warn_rate_limit_ms", + "me_route_no_writer_mode", + "me_route_no_writer_wait_ms", + "me_route_hybrid_max_wait_ms", + "me_route_blocking_send_timeout_ms", + "me_route_inline_recovery_attempts", + "me_route_inline_recovery_wait_ms", + "links", + "fast_mode_min_tls_record", + "update_every", + "me_reinit_every_secs", + "me_hardswap_warmup_delay_min_ms", + "me_hardswap_warmup_delay_max_ms", + "me_hardswap_warmup_extra_passes", + "me_hardswap_warmup_pass_backoff_base_ms", + "me_config_stable_snapshots", + "me_config_apply_cooldown_secs", + "me_snapshot_require_http_2xx", + "me_snapshot_reject_empty_map", + "me_snapshot_min_proxy_for_lines", + "proxy_secret_stable_snapshots", + "proxy_secret_rotate_runtime", + "me_secret_atomic_snapshot", + "proxy_secret_len_max", + "me_pool_drain_ttl_secs", + "me_instadrain", + "me_pool_drain_threshold", + "me_pool_drain_soft_evict_enabled", + "me_pool_drain_soft_evict_grace_secs", + "me_pool_drain_soft_evict_per_writer", + "me_pool_drain_soft_evict_budget_per_core", + "me_pool_drain_soft_evict_cooldown_ms", + "me_bind_stale_mode", + "me_bind_stale_ttl_secs", + "me_pool_min_fresh_ratio", + "me_reinit_drain_timeout_secs", + "proxy_secret_auto_reload_secs", + "proxy_config_auto_reload_secs", + "me_reinit_singleflight", + "me_reinit_trigger_channel", + "me_reinit_coalesce_window_ms", + "me_deterministic_writer_sort", + "me_writer_pick_mode", + "me_writer_pick_sample_size", + "ntp_check", + "ntp_servers", + "auto_degradation_enabled", + "degradation_min_unavailable_dc_groups", + "rst_on_close", +]; + +const NETWORK_CONFIG_KEYS: &[&str] = &[ + "ipv4", + "ipv6", + "prefer", + "multipath", + "stun_use", + "stun_servers", + "stun_tcp_fallback", + "http_ip_detect_urls", + "cache_public_ip_path", + "dns_overrides", +]; + +const SERVER_CONFIG_KEYS: &[&str] = &[ + "port", + "listen_addr_ipv4", + "listen_addr_ipv6", + "listen_unix_sock", + "listen_unix_sock_perm", + "listen_tcp", + "proxy_protocol", + "proxy_protocol_header_timeout_ms", + "proxy_protocol_trusted_cidrs", + "metrics_port", + "metrics_listen", + "metrics_whitelist", + "api", + "admin_api", + "listeners", + "listen_backlog", + "max_connections", + "accept_permit_timeout_ms", + "conntrack_control", +]; + +const API_CONFIG_KEYS: &[&str] = &[ + "enabled", + "listen", + "whitelist", + "gray_action", + "auth_header", + "request_body_limit_bytes", + "minimal_runtime_enabled", + "minimal_runtime_cache_ttl_ms", + "runtime_edge_enabled", + "runtime_edge_cache_ttl_ms", + "runtime_edge_top_n", + "runtime_edge_events_capacity", + "read_only", +]; + +const CONNTRACK_CONTROL_CONFIG_KEYS: &[&str] = &[ + "inline_conntrack_control", + "mode", + "backend", + "profile", + "hybrid_listener_ips", + "pressure_high_watermark_pct", + "pressure_low_watermark_pct", + "delete_budget_per_sec", +]; + +const LISTENER_CONFIG_KEYS: &[&str] = &[ + "ip", + "port", + "announce", + "announce_ip", + "proxy_protocol", + "reuse_allow", +]; + +const TIMEOUTS_CONFIG_KEYS: &[&str] = &[ + "client_first_byte_idle_secs", + "client_handshake", + "relay_idle_policy_v2_enabled", + "relay_client_idle_soft_secs", + "relay_client_idle_hard_secs", + "relay_idle_grace_after_downstream_activity_secs", + "client_keepalive", + "client_ack", + "me_one_retry", + "me_one_timeout_ms", +]; + +const CENSORSHIP_CONFIG_KEYS: &[&str] = &[ + "tls_domain", + "tls_domains", + "unknown_sni_action", + "tls_fetch_scope", + "tls_fetch", + "mask", + "mask_host", + "mask_port", + "mask_unix_sock", + "fake_cert_len", + "tls_emulation", + "tls_front_dir", + "server_hello_delay_min_ms", + "server_hello_delay_max_ms", + "tls_new_session_tickets", + "serverhello_compact", + "tls_full_cert_ttl_secs", + "alpn_enforce", + "mask_proxy_protocol", + "mask_shape_hardening", + "mask_shape_hardening_aggressive_mode", + "mask_shape_bucket_floor_bytes", + "mask_shape_bucket_cap_bytes", + "mask_shape_above_cap_blur", + "mask_shape_above_cap_blur_max_bytes", + "mask_relay_max_bytes", + "mask_relay_timeout_ms", + "mask_relay_idle_timeout_ms", + "mask_classifier_prefetch_timeout_ms", + "mask_timing_normalization_enabled", + "mask_timing_normalization_floor_ms", + "mask_timing_normalization_ceiling_ms", +]; + +const TLS_FETCH_CONFIG_KEYS: &[&str] = &[ + "profiles", + "strict_route", + "attempt_timeout_ms", + "total_budget_ms", + "grease_enabled", + "deterministic", + "profile_cache_ttl_secs", +]; + +const ACCESS_CONFIG_KEYS: &[&str] = &[ + "users", + "user_ad_tags", + "user_max_tcp_conns", + "user_max_tcp_conns_global_each", + "user_expirations", + "user_data_quota", + "user_rate_limits", + "cidr_rate_limits", + "user_max_unique_ips", + "user_max_unique_ips_global_each", + "user_max_unique_ips_mode", + "user_max_unique_ips_window_secs", + "replay_check_len", + "replay_window_secs", + "ignore_time_skew", +]; + +const RATE_LIMIT_BPS_CONFIG_KEYS: &[&str] = &["up_bps", "down_bps"]; + +const UPSTREAM_CONFIG_KEYS: &[&str] = &[ + "type", + "interface", + "bind_addresses", + "bindtodevice", + "force_bind", + "address", + "user_id", + "username", + "password", + "url", + "weight", + "enabled", + "scopes", + "ipv4", + "ipv6", +]; + +const PROXY_MODES_CONFIG_KEYS: &[&str] = &["classic", "secure", "tls"]; +const TELEMETRY_CONFIG_KEYS: &[&str] = &["core_enabled", "user_enabled", "me_level"]; +const LINKS_CONFIG_KEYS: &[&str] = &["show", "public_host", "public_port"]; + +#[derive(Debug)] +struct UnknownConfigKey { + path: String, + suggestion: Option, +} + +fn table_at<'a>(value: &'a toml::Value, path: &[&str]) -> Option<&'a toml::Table> { + let mut current = value; + for segment in path { + current = current.get(*segment)?; + } + current.as_table() +} + +fn is_strict_config(parsed_toml: &toml::Value) -> bool { + table_at(parsed_toml, &["general"]) + .and_then(|table| table.get("config_strict")) + .and_then(toml::Value::as_bool) + .unwrap_or(false) +} + +fn known_config_keys_for_suggestion() -> Vec<&'static str> { + let mut keys = Vec::new(); + for group in [ + TOP_LEVEL_CONFIG_KEYS, + GENERAL_CONFIG_KEYS, + NETWORK_CONFIG_KEYS, + SERVER_CONFIG_KEYS, + API_CONFIG_KEYS, + CONNTRACK_CONTROL_CONFIG_KEYS, + LISTENER_CONFIG_KEYS, + TIMEOUTS_CONFIG_KEYS, + CENSORSHIP_CONFIG_KEYS, + TLS_FETCH_CONFIG_KEYS, + ACCESS_CONFIG_KEYS, + RATE_LIMIT_BPS_CONFIG_KEYS, + UPSTREAM_CONFIG_KEYS, + PROXY_MODES_CONFIG_KEYS, + TELEMETRY_CONFIG_KEYS, + LINKS_CONFIG_KEYS, + ] { + keys.extend_from_slice(group); + } + keys +} + +fn levenshtein_distance(a: &str, b: &str) -> usize { + let b_chars: Vec = b.chars().collect(); + let mut prev: Vec = (0..=b_chars.len()).collect(); + let mut curr = vec![0usize; b_chars.len() + 1]; + + for (i, ca) in a.chars().enumerate() { + curr[0] = i + 1; + for (j, cb) in b_chars.iter().enumerate() { + let replace = if ca == *cb { prev[j] } else { prev[j] + 1 }; + curr[j + 1] = (prev[j + 1] + 1).min(curr[j] + 1).min(replace); + } + std::mem::swap(&mut prev, &mut curr); + } + + prev[b_chars.len()] +} + +fn unknown_key_suggestion(key: &str, known_keys: &[&'static str]) -> Option { + let normalized = key.to_ascii_lowercase(); + let mut best: Option<(&str, usize)> = None; + for known in known_keys { + let distance = levenshtein_distance(&normalized, known); + let is_better = match best { + Some((_, best_distance)) => distance < best_distance, + None => true, + }; + if distance <= 4 && is_better { + best = Some((known, distance)); + } + } + best.map(|(known, _)| known.to_string()) +} + +fn push_unknown_keys( + unknown: &mut Vec, + known_for_suggestion: &[&'static str], + path: &str, + table: &toml::Table, + allowed: &[&str], +) { + for key in table.keys() { + if !allowed.contains(&key.as_str()) { + let full_path = if path.is_empty() { + key.clone() + } else { + format!("{path}.{key}") + }; + unknown.push(UnknownConfigKey { + path: full_path, + suggestion: unknown_key_suggestion(key, known_for_suggestion), + }); + } + } +} + +fn check_known_table( + parsed_toml: &toml::Value, + unknown: &mut Vec, + known_for_suggestion: &[&'static str], + path: &[&str], + allowed: &[&str], +) { + if let Some(table) = table_at(parsed_toml, path) { + push_unknown_keys( + unknown, + known_for_suggestion, + &path.join("."), + table, + allowed, + ); + } +} + +fn check_nested_table_value( + unknown: &mut Vec, + known_for_suggestion: &[&'static str], + path: String, + value: &toml::Value, + allowed: &[&str], +) { + if let Some(table) = value.as_table() { + push_unknown_keys(unknown, known_for_suggestion, &path, table, allowed); + } +} + +fn collect_unknown_config_keys(parsed_toml: &toml::Value) -> Vec { + let known_for_suggestion = known_config_keys_for_suggestion(); + let mut unknown = Vec::new(); + + if let Some(root) = parsed_toml.as_table() { + push_unknown_keys( + &mut unknown, + &known_for_suggestion, + "", + root, + TOP_LEVEL_CONFIG_KEYS, + ); + } + + check_known_table( + parsed_toml, + &mut unknown, + &known_for_suggestion, + &["general"], + GENERAL_CONFIG_KEYS, + ); + check_known_table( + parsed_toml, + &mut unknown, + &known_for_suggestion, + &["general", "modes"], + PROXY_MODES_CONFIG_KEYS, + ); + check_known_table( + parsed_toml, + &mut unknown, + &known_for_suggestion, + &["general", "telemetry"], + TELEMETRY_CONFIG_KEYS, + ); + check_known_table( + parsed_toml, + &mut unknown, + &known_for_suggestion, + &["general", "links"], + LINKS_CONFIG_KEYS, + ); + check_known_table( + parsed_toml, + &mut unknown, + &known_for_suggestion, + &["network"], + NETWORK_CONFIG_KEYS, + ); + check_known_table( + parsed_toml, + &mut unknown, + &known_for_suggestion, + &["server"], + SERVER_CONFIG_KEYS, + ); + check_known_table( + parsed_toml, + &mut unknown, + &known_for_suggestion, + &["server", "api"], + API_CONFIG_KEYS, + ); + check_known_table( + parsed_toml, + &mut unknown, + &known_for_suggestion, + &["server", "admin_api"], + API_CONFIG_KEYS, + ); + check_known_table( + parsed_toml, + &mut unknown, + &known_for_suggestion, + &["server", "conntrack_control"], + CONNTRACK_CONTROL_CONFIG_KEYS, + ); + check_known_table( + parsed_toml, + &mut unknown, + &known_for_suggestion, + &["timeouts"], + TIMEOUTS_CONFIG_KEYS, + ); + check_known_table( + parsed_toml, + &mut unknown, + &known_for_suggestion, + &["censorship"], + CENSORSHIP_CONFIG_KEYS, + ); + check_known_table( + parsed_toml, + &mut unknown, + &known_for_suggestion, + &["censorship", "tls_fetch"], + TLS_FETCH_CONFIG_KEYS, + ); + check_known_table( + parsed_toml, + &mut unknown, + &known_for_suggestion, + &["access"], + ACCESS_CONFIG_KEYS, + ); + + if let Some(listeners) = table_at(parsed_toml, &["server"]) + .and_then(|table| table.get("listeners")) + .and_then(toml::Value::as_array) + { + for (idx, listener) in listeners.iter().enumerate() { + check_nested_table_value( + &mut unknown, + &known_for_suggestion, + format!("server.listeners[{idx}]"), + listener, + LISTENER_CONFIG_KEYS, + ); + } + } + + if let Some(upstreams) = parsed_toml + .get("upstreams") + .and_then(toml::Value::as_array) + { + for (idx, upstream) in upstreams.iter().enumerate() { + check_nested_table_value( + &mut unknown, + &known_for_suggestion, + format!("upstreams[{idx}]"), + upstream, + UPSTREAM_CONFIG_KEYS, + ); + } + } + + for access_map in ["user_rate_limits", "cidr_rate_limits"] { + if let Some(table) = table_at(parsed_toml, &["access"]) + .and_then(|access| access.get(access_map)) + .and_then(toml::Value::as_table) + { + for (entry_name, value) in table { + check_nested_table_value( + &mut unknown, + &known_for_suggestion, + format!("access.{access_map}.{entry_name}"), + value, + RATE_LIMIT_BPS_CONFIG_KEYS, + ); + } + } + } + + unknown +} + +fn handle_unknown_config_keys(parsed_toml: &toml::Value) -> Result<()> { + let unknown = collect_unknown_config_keys(parsed_toml); + if unknown.is_empty() { + return Ok(()); + } + + for item in &unknown { + if let Some(suggestion) = item.suggestion.as_deref() { + warn!( + key = %item.path, + suggestion = %suggestion, + "Unknown config key ignored; did you mean the suggested key?" + ); + } else { + warn!(key = %item.path, "Unknown config key ignored"); + } + } + + if is_strict_config(parsed_toml) { + let mut paths = Vec::with_capacity(unknown.len()); + for item in unknown { + if let Some(suggestion) = item.suggestion { + paths.push(format!("{} (did you mean `{}`?)", item.path, suggestion)); + } else { + paths.push(item.path); + } + } + return Err(ProxyError::Config(format!( + "unknown config keys are not allowed when general.config_strict=true: {}", + paths.join(", ") + ))); + } + + Ok(()) +} + #[derive(Debug, Clone)] pub(crate) struct LoadedConfig { pub(crate) config: ProxyConfig, @@ -337,6 +997,7 @@ impl ProxyConfig { let parsed_toml: toml::Value = toml::from_str(&processed).map_err(|e| ProxyError::Config(e.to_string()))?; + handle_unknown_config_keys(&parsed_toml)?; let general_table = parsed_toml .get("general") .and_then(|value| value.as_table()); diff --git a/src/config/types.rs b/src/config/types.rs index 4762083..912f2b2 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -375,6 +375,11 @@ pub struct GeneralConfig { #[serde(default)] pub data_path: Option, + /// Reject unknown TOML config keys during load. + /// Startup fails fast; hot-reload rejects the new snapshot and keeps the current config. + #[serde(default)] + pub config_strict: bool, + #[serde(default)] pub modes: ProxyModes, @@ -974,6 +979,7 @@ impl Default for GeneralConfig { fn default() -> Self { Self { data_path: None, + config_strict: false, modes: ProxyModes::default(), prefer_ipv6: false, fast_mode: default_true(),