mirror of
https://github.com/telemt/telemt.git
synced 2026-09-05 18:16:06 +03:00
Merge pull request #904 from telemt/flow-web-preserved
WEB: Fixed carrier for Apple iOS
This commit is contained in:
Alexey
Generated
+30
-4
@@ -2196,9 +2196,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.10.1"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
|
||||
checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80"
|
||||
dependencies = [
|
||||
"chacha20 0.10.0",
|
||||
"getrandom 0.4.3",
|
||||
@@ -2900,7 +2900,7 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
|
||||
|
||||
[[package]]
|
||||
name = "telemt"
|
||||
version = "3.5.2"
|
||||
version = "3.5.3"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"anyhow",
|
||||
@@ -2916,6 +2916,7 @@ dependencies = [
|
||||
"ctr",
|
||||
"dashmap",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"hex",
|
||||
"hmac",
|
||||
"http-body-util",
|
||||
@@ -2933,7 +2934,7 @@ dependencies = [
|
||||
"num-traits",
|
||||
"parking_lot",
|
||||
"proptest",
|
||||
"rand 0.10.1",
|
||||
"rand 0.10.2",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"rustls",
|
||||
@@ -2950,6 +2951,7 @@ dependencies = [
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tokio-test",
|
||||
"tokio-tungstenite",
|
||||
"tokio-util",
|
||||
"toml",
|
||||
"tracing",
|
||||
@@ -3147,6 +3149,18 @@ dependencies = [
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-tungstenite"
|
||||
version = "0.30.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17a073bfed563fa236697a068031408a93cd9522e08abf9933ead3e73411bd71"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"log",
|
||||
"tokio",
|
||||
"tungstenite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.7.18"
|
||||
@@ -3339,6 +3353,18 @@ version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
|
||||
|
||||
[[package]]
|
||||
name = "tungstenite"
|
||||
version = "0.30.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e48ac77174b19c110a50ab2128b24215ac9cb40e0e12e093fb602d175c569d22"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"log",
|
||||
"rand 0.10.2",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
|
||||
+3
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "telemt"
|
||||
version = "3.5.2"
|
||||
version = "3.5.3"
|
||||
edition = "2024"
|
||||
|
||||
[features]
|
||||
@@ -76,6 +76,8 @@ hyper = { version = "1.10.1", features = ["client", "server", "http1"] }
|
||||
hyper-util = { version = "0.1.20", features = ["tokio", "server-auto"] }
|
||||
http-body-util = "0.1.3"
|
||||
httpdate = "1.0.3"
|
||||
tokio-tungstenite = { version = "0.30.0", default-features = false }
|
||||
futures-util = { version = "0.3.32", default-features = false, features = ["sink", "std"] }
|
||||
tokio-rustls = { version = "0.26.4", default-features = false, features = [
|
||||
"tls12",
|
||||
] }
|
||||
|
||||
@@ -1524,7 +1524,7 @@ The revision is verified again after preparation. With `failure_policy=rollback`
|
||||
|
||||
## WEB Proxy Management
|
||||
|
||||
The API provides partial operational control for WEB mode; it does not expose a dedicated `/v1/web` resource.
|
||||
The API provides partial operational control for WEB mode. It does not expose a mutable `/v1/web` resource, but it serves bounded read-only HTML diagnostics at `GET /web-status`.
|
||||
|
||||
| Operation | Current contract |
|
||||
| --- | --- |
|
||||
@@ -1535,13 +1535,15 @@ The API provides partial operational control for WEB mode; it does not expose a
|
||||
| Manage access users | Use `/v1/users`. Creating a user does not add it to `web.vhosts.profiles`; profile membership remains file-managed. |
|
||||
| Disable one user | `POST /v1/users/{username}/disable` updates admission immediately and cancels the user's active sessions. |
|
||||
| Rotate a profiled user's secret | Use `/v1/users/{username}/rotate-secret`; the config watcher rebuilds WEB capabilities from the new access snapshot. The API returns the secret, not a `tg://webproxy` link. |
|
||||
| Read WEB-specific runtime statistics | No WEB-specific endpoint exists in the current API surface. |
|
||||
| Read WEB-specific runtime diagnostics | Use authenticated `GET /web-status`; filters cover client IP, process session ID, User-Agent, and non-secret key fingerprint, with optional grouping, expandable HTTP request-to-response details, and WebSocket handshake/message/frame rows. |
|
||||
|
||||
`web.enabled`, `web.carrier`, `web.timeouts`, vhosts, profiles, and decoy snapshots are runtime-generation fields. A changed carrier applies only to newly issued bridge sessions; existing sessions retain their creation-time carrier. WEB listener inventory and trust policy, plus all `[web.limits]`, are process-owned. A successful reload can therefore activate the runtime-owned subset while reporting the process-owned subset as deferred.
|
||||
`web.enabled`, `web.carrier`, `web.debug`, `web.timeouts`, vhosts, profiles, and decoy snapshots are runtime-generation fields. A changed carrier applies only to newly issued bridge sessions; existing sessions retain their creation-time carrier. WEB listener inventory and trust policy, plus all `[web.limits]`, are process-owned. A successful reload can therefore activate the runtime-owned subset while reporting the process-owned subset as deferred.
|
||||
|
||||
Before deleting a user referenced by a WEB profile, remove and apply the profile first. User mutations validate the complete resulting configuration, so a dangling WEB profile is rejected rather than persisted.
|
||||
|
||||
The API whitelist is evaluated against the direct TCP peer and does not use the WEB listener's `X-Forwarded-For` policy. Keep the API on a separate loopback or private bind, use a narrow whitelist and a non-empty exact `auth_header`, and do not expose it through the public WEB vhost.
|
||||
The API whitelist is evaluated against the direct TCP peer and does not use the WEB listener's `X-Forwarded-For` policy. `/web-status` inherits API enablement, whitelist, gray action, and exact authorization-header checks; it accepts only `GET`, normalizes a trailing slash, sets `no-store` and restrictive browser security headers, caps each page at 8 MiB, and permits at most two concurrent renderers. Keep the API on a separate loopback or private bind, use a narrow whitelist and a non-empty exact `auth_header`, and do not expose it through the public WEB vhost.
|
||||
|
||||
`window_secs` defaults to `[web.debug].default_window_secs = 180` and cannot exceed `max_window_secs`. The page can group by any combination of `ip`, `session`, `user_agent`, and `key`. Detail views retain policy-bounded HTTP method, sanitized headers, body, timing, and inner frames from request through response. For `websocket` and `websocket-lanes`, they additionally show the sanitized `GET` to `101` handshake and bounded per-message direction, type, payload/body capture, processing timing, connection/lane identifiers, and parsed inner frames. Raw query credentials, authorization values, WebSocket subprotocols, and session tokens are never retained.
|
||||
|
||||
Deployment, TLS-terminator examples, links, and WEB-specific verification are documented in the [WEB proxy guide](../../WEB/WEB_PROXY.en.md).
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ Dieses Dokument listet alle Konfigurationsschlüssel auf, die `config.toml` akze
|
||||
- [server.api](#serverapi)
|
||||
- [server.listeners](#serverlisteners)
|
||||
- [web](#web)
|
||||
- [web.debug](#webdebug)
|
||||
- [web.limits](#weblimits)
|
||||
- [web.timeouts](#webtimeouts)
|
||||
- [web.vhosts](#webvhosts)
|
||||
@@ -2556,12 +2557,32 @@ Der WEB-Modus transportiert MTProxy-Datenverkehr von Telegram Desktop über HTTP
|
||||
| Schlüssel | Typ | Default | Hot-Reload |
|
||||
| --- | --- | --- | --- |
|
||||
| `enabled` | `bool` | `false` | `✔` |
|
||||
| `carrier` | `"https"` oder `"https-lanes"` | `"https"` | `✔` |
|
||||
| `carrier` | `"https"`, `"https-lanes"`, `"websocket"` oder `"websocket-lanes"` | `"https"` | `✔` |
|
||||
| `debug` | Tabelle | deaktiviert, begrenzte Defaults | `✔` |
|
||||
| `limits` | Tabelle | begrenzte Defaults | `✘` |
|
||||
| `timeouts` | Tabelle | begrenzte Defaults | `✔` |
|
||||
| `vhosts` | Tabellen-Array | `[]` | `✔` |
|
||||
|
||||
`enabled = true` erfordert mindestens einen durch die Netzwerkrichtlinie zugelassenen WEB-Listener, einen vhost und mindestens ein Profil in jedem vhost. `carrier = "https"` behält den serialisierten HTTPS-Transport bei. Mit `carrier = "https-lanes"` erhalten Stream null und jeder logische Stream eigene Uplink-Sequenzen, Downlink-Cursor, Wiederholungen und Long Polls; dieser Carrier erfordert `max_http_handlers >= 2` und öffentliches HTTP/2 am TLS-Terminator, um anwendungsseitiges Head-of-Line-Blocking zwischen Streams zu entfernen. Ein Reload wendet `carrier` nur auf neu ausgegebene Bridge-Sitzungen an. Das Deaktivieren von WEB beendet nach dem Reload die Ausgabe neuer Bridge- und Session-Zugangsdaten; zum Widerrufen aktiver Sitzungen eines einzelnen Benutzers verwenden Sie die Users-API.
|
||||
`enabled = true` erfordert mindestens einen durch die Netzwerkrichtlinie zugelassenen WEB-Listener, einen vhost und mindestens ein Profil in jedem vhost. `https` behält den serialisierten HTTPS-Transport bei. Mit `https-lanes` erhalten Stream null und jeder logische Stream eigene Uplink-Sequenzen, Downlink-Cursor, Wiederholungen und Long Polls; dieser Carrier erfordert `max_http_handlers >= 2` und öffentliches HTTP/2 am TLS-Terminator. `websocket` transportiert alle logischen Streams über eine geordnete RFC-6455-Verbindung, während `websocket-lanes` jedem Stream ungleich null eine eigene Verbindung zuweist und Lane-Fehler isoliert. Beide WebSocket-Carrier verwenden nach der HTTPS-Sitzungserstellung `GET /api/v1/ws` und erfordern, dass der TLS-Terminator die HTTP/1.1-Upgrade-Header unverändert weiterleitet. Ein Reload wendet `carrier` nur auf neu ausgegebene Bridge-Sitzungen an. Das Deaktivieren von WEB beendet nach dem Reload die Ausgabe neuer Bridge- und Session-Zugangsdaten; zum Widerrufen aktiver Sitzungen eines einzelnen Benutzers verwenden Sie die Users-API.
|
||||
|
||||
# [web.debug]
|
||||
|
||||
Diese hot-reload-fähige Tabelle steuert den prozesseigenen serverseitigen WEB-Debug-Recorder, der am API-Listener als authentifiziertes HTML unter `GET /web-status` bereitgestellt wird. Die Erfassung ist standardmäßig deaktiviert. Gespeicherte und in Verarbeitung befindliche Datensätze bleiben durch die nur nach einem Neustart änderbaren Werte in `[web.limits]` begrenzt.
|
||||
|
||||
| Schlüssel | Typ | Default | Beschreibung |
|
||||
| --- | --- | --- | --- |
|
||||
| `enabled` | `bool` | `false` | Aktiviert WEB-HTTP-, WebSocket-Message-, Frame- und Lifecycle-Debugdatensätze. |
|
||||
| `capture_lifecycle` | `bool` | `true` | Zeichnet typisierte Bridge-, Sitzungs-, Stream-, Handshake-, Relay- und Close-Ereignisse auf. |
|
||||
| `capture_headers` | `bool` | `true` | Speichert Headernamen und nur ausdrücklich zugelassene Werte ohne Zugangsdaten. |
|
||||
| `capture_timings` | `bool` | `true` | Speichert Zeitpunkte für Request-Body, fertige Response, Response-Body und WebSocket-Message-Verarbeitung. |
|
||||
| `capture_frames` | `bool` | `true` | Zerlegt begrenzte Carrier-Bodys in Frame-Typ, Stream-ID, Länge, WINDOW- und Fehlermetadaten, ohne die Frame-Nutzlast zusätzlich zu speichern. |
|
||||
| `body_capture` | `"off"`, `"metadata"`, `"prefix"` oder `"full"` | `"metadata"` | Steuert die Speicherung von Bytes aus Request- und Response-Bodys. |
|
||||
| `body_prefix_bytes` | `usize` | `4096` | In `prefix` gespeicherter Präfix für erkannte WEB-Bodys. |
|
||||
| `decoy_body_prefix_bytes` | `usize` | `4096` | Maximal gespeicherter Präfix für gewöhnlichen Decoy-Verkehr in `prefix` und `full`. |
|
||||
| `default_window_secs` | `u64` | `180` | Standard-Beobachtungsfenster von `/web-status`. |
|
||||
| `max_window_secs` | `u64` | `3600` | Größtes von `/web-status` akzeptiertes Beobachtungsfenster; validiert auf höchstens 86400. |
|
||||
|
||||
Eine Änderung von `enabled` oder einem Erfassungsfeld löscht gespeicherte Datensätze und verwirft Commits, die unter der vorherigen Policy-Epoche begonnen wurden. Ändert sich nur das standardmäßige oder maximale Beobachtungsfenster, bleiben kompatible Datensätze erhalten. `full` speichert den vollständigen Body eines erkannten Carriers nur bis `web.limits.max_body_bytes`; Decoy-Bodys bleiben immer auf einen Präfix begrenzt. Ein Präfix, der nur mit einer gleichzeitig erhöhten, neustartpflichtigen Kapazität zulässig wäre, wird zusammen mit `web.debug` bis zum Neustart zurückgestellt. URI-Queries werden nie gespeichert, Werte von Credential-Headern werden ausgelassen, Body-Kopien werden von bekannten WEB-Capabilities und Bearer-Tokens bereinigt und Profilschlüssel ausschließlich als domänengetrennter Fingerprint mit 16 Hex-Zeichen dargestellt.
|
||||
|
||||
# [web.limits]
|
||||
|
||||
@@ -2576,6 +2597,10 @@ Diese prozessweiten Obergrenzen begrenzen alle WEB-Register, Warteschlangen, Req
|
||||
| `max_frames_per_body` | `usize` | `4096` | Maximale Zahl geparster oder ausgegebener Frames pro Carrier-Body. |
|
||||
| `max_http_connections` | `usize` | `1024` | Prozessweit akzeptierte WEB-HTTP-Verbindungen. |
|
||||
| `max_http_handlers` | `usize` | `512` | Prozessweit gleichzeitig ausgeführte HTTP-Handler; HTTPS-Lanes dürfen höchstens die Hälfte mit Long Polls belegen, der Rest bleibt für Session-, Uplink- und Steuerarbeit verfügbar. |
|
||||
| `websocket_bytes_global` | `usize` | `268435456` | Transientes Teilbudget für WebSocket-Codecs, Messages und Write-Staging innerhalb von `pending_bytes_global`. |
|
||||
| `websocket_admission_watermark_pct` | `u8` | `75` | WebSocket-Byte-Anteil, ab dem neue Admission eine Owner-First-Verbindung ersetzen darf. |
|
||||
| `websocket_eviction_watermark_pct` | `u8` | `90` | WebSocket-Byte-Anteil, ab dem Queue-Druck die zulässige Verbindung mit dem ältesten Fortschritt verdrängen darf. |
|
||||
| `websocket_http_connection_reserve` | `usize` | `64` | Für WebSocket-Upgrades gesperrte HTTP-Verbindungen, die Kapazität für gewöhnliches HTTP und Decoys erhalten. |
|
||||
| `max_body_readers` | `usize` | `32` | Prozessweit gleichzeitig gesammelte Request-Bodys. |
|
||||
| `max_body_bytes_global` | `usize` | `67108864` | Globales Byte-Budget für gesammelte Bodys. |
|
||||
| `max_sessions_global` | `usize` | `128` | Prozessweit aktive WEB-Sitzungen. |
|
||||
@@ -2597,7 +2622,9 @@ Diese prozessweiten Obergrenzen begrenzen alle WEB-Register, Warteschlangen, Req
|
||||
| `max_static_files` | `usize` | `4096` | Einträge statischer Snapshots über alle vhosts. |
|
||||
| `max_static_file_bytes` | `usize` | `8388608` | Maximale Größe einer statischen Datei. |
|
||||
| `max_static_bytes` | `usize` | `67108864` | Bytes statischer Snapshots über alle vhosts. |
|
||||
| `memory_envelope_bytes` | `usize` | `805306368` | Deklarierter Rahmen für HTTP-Heads, Bodys, Queues und statische Snapshots; maximal 4 GiB. |
|
||||
| `debug_records_capacity` | `usize` | `65536` | Maximale Zahl gespeicherter WEB-Debugdatensätze. |
|
||||
| `debug_bytes_global` | `usize` | `67108864` | Globale Byte-Obergrenze für gespeicherte und in Verarbeitung befindliche WEB-Debugdaten; mindestens 4096. |
|
||||
| `memory_envelope_bytes` | `usize` | `805306368` | Deklarierter Rahmen für HTTP-Heads, Bodys, gemeinsame Queues/WebSocket-I/O, statische Snapshots und begrenzte Debug-/Statuspuffer; maximal 4 GiB. |
|
||||
| `new_bootstraps_per_minute` | `u32` | `1200` | Nachhaltige prozessweite Ausgaberate für Bootstraps. |
|
||||
| `new_bootstraps_burst` | `u32` | `256` | Prozessweiter Burst für die Bootstrap-Ausgabe. |
|
||||
| `new_sessions_per_minute` | `u32` | `600` | Nachhaltige prozessweite Erstellungsrate für Sitzungen. |
|
||||
@@ -2615,6 +2642,9 @@ Alle Timeouts werden in Sekunden angegeben und müssen im Bereich `1..=3600` lie
|
||||
| `body_secs` | `u64` | `30` | `✔` | Sammeln eines authentifizierten Carrier-Bodys. |
|
||||
| `stream_handshake_secs` | `u64` | `10` | `✔` | Abschluss eines inneren MTProxy-Handshakes. |
|
||||
| `long_poll_secs` | `u64` | `25` | `✔` | Maximale Dauer eines leeren Downlink-Long-Polls. |
|
||||
| `websocket_write_secs` | `u64` | `30` | `✔` | Maximale Wartezeit für einen WebSocket-Write oder Flush. |
|
||||
| `websocket_backpressure_secs` | `u64` | `30` | `✔` | Maximale Wartezeit auf Fortschritt des gemeinsamen Byte-Budgets oder einer Queue, bevor die betroffene Verbindung geschlossen wird. |
|
||||
| `websocket_eviction_secs` | `u64` | `1` | `✔` | Karenzzeit, in der ein verdrängter WebSocket Slot und Budget freigeben muss, bevor Admission fehlschlägt. |
|
||||
| `bootstrap_lifetime_secs` | `u64` | `120` | `✔` | Lebensdauer ungenutzter Bootstraps und geschlossener Token-Replay-Marker. |
|
||||
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Maximale Carrier-Inaktivität bis zum Schließen der Sitzung. |
|
||||
| `http_idle_secs` | `u64` | `75` | `✔` | Idle-Lebensdauer einer WEB-HTTP-Keep-Alive-Verbindung. |
|
||||
@@ -2655,9 +2685,9 @@ Profilgrenzen müssen ungleich null sein und dürfen die zugehörigen globalen G
|
||||
|
||||
## WEB-Lebenszyklus und API-Verwaltung
|
||||
|
||||
- Config-Watcher und Generations-Reload wenden `web.enabled`, `web.carrier`, `web.timeouts`, vhosts, Profile und Decoy-Snapshots ohne Prozessneustart an. Bestehende Sitzungen behalten Carrier, Grenzen und Deadlines ihres Erstellungszeitpunkts; neu ausgegebene Bridge-Sitzungen verwenden die aktive Generation.
|
||||
- Config-Watcher und Generations-Reload wenden `web.enabled`, `web.carrier`, `web.debug`, `web.timeouts`, vhosts, Profile und Decoy-Snapshots ohne Prozessneustart an. Bestehende Sitzungen behalten Carrier, Grenzen und Deadlines ihres Erstellungszeitpunkts; neu ausgegebene Bridge-Sitzungen verwenden die aktive Generation.
|
||||
- Bestand und Vertrauensrichtlinie der WEB-Listener unter `server.listeners` sowie alle Werte in `web.limits` sind prozesseigen und erfordern einen Neustart.
|
||||
- Es gibt keinen eigenen Endpunkt `/v1/web`. `GET /v1/config` lässt `[web]` aus und `PATCH /v1/config` lehnt einen Schlüssel `web` mit `400 section_not_editable` ab.
|
||||
- Es gibt keine veränderbare Ressource `/v1/web`. `GET /web-status` stellt authentifizierte, schreibgeschützte HTML-Diagnosen bereit; `GET /v1/config` lässt `[web]` aus und `PATCH /v1/config` lehnt einen Schlüssel `web` mit `400 section_not_editable` ab.
|
||||
- Zum entfernten Anwenden einer WEB-Richtlinie ändern Sie die zuständige TOML-Datei und rufen `POST /v1/system/reload` auf. Prüfen Sie anschließend `GET /v1/system/reload/{id}` und dessen `deferred_process_fields`. Starten Sie Telemt neu, wenn das Feld `server.listeners` oder `web.limits` enthält.
|
||||
- Vorhandene Access-Benutzer können über `/v1/users` erstellt, geändert, rotiert, aktiviert, deaktiviert und gelöscht werden. Das Erstellen eines Benutzers fügt kein WEB-Profil hinzu. Das Deaktivieren aktualisiert die Admission sofort und beendet die aktiven Sitzungen dieses Benutzers.
|
||||
- `PATCH /v1/config` kann `server.listeners` einschließlich der WEB-Listener-Felder speichern; ein geänderter WEB-Listener wird jedoch erst nach einem Prozessneustart aktiv.
|
||||
|
||||
@@ -25,6 +25,7 @@ This document lists all configuration keys accepted by `config.toml`.
|
||||
- [server.api](#serverapi)
|
||||
- [server.listeners](#serverlisteners)
|
||||
- [web](#web)
|
||||
- [web.debug](#webdebug)
|
||||
- [web.limits](#weblimits)
|
||||
- [web.timeouts](#webtimeouts)
|
||||
- [web.vhosts](#webvhosts)
|
||||
@@ -2556,12 +2557,32 @@ WEB mode carries Telegram Desktop MTProxy traffic through HTTPS terminated by an
|
||||
| Key | Type | Default | Hot-Reload |
|
||||
| --- | --- | --- | --- |
|
||||
| `enabled` | `bool` | `false` | `✔` |
|
||||
| `carrier` | `"https"` or `"https-lanes"` | `"https"` | `✔` |
|
||||
| `carrier` | `"https"`, `"https-lanes"`, `"websocket"`, or `"websocket-lanes"` | `"https"` | `✔` |
|
||||
| `debug` | table | disabled, bounded defaults | `✔` |
|
||||
| `limits` | table | bounded defaults | `✘` |
|
||||
| `timeouts` | table | bounded defaults | `✔` |
|
||||
| `vhosts` | array of tables | `[]` | `✔` |
|
||||
|
||||
`enabled = true` requires at least one network-eligible WEB listener, at least one vhost, and at least one profile in every vhost. `carrier = "https"` preserves the serialized HTTPS transport. `carrier = "https-lanes"` gives stream zero and every logical stream independent uplink sequencing, downlink cursors, retries, and long polls; it requires `max_http_handlers >= 2` and public HTTP/2 on the TLS terminator to remove application-level inter-stream head-of-line blocking. A reload applies `carrier` only to newly issued bridge sessions. Disabling WEB stops issuance of new bridge and session credentials after reload; use the users API to revoke one user's active sessions.
|
||||
`enabled = true` requires at least one network-eligible WEB listener, at least one vhost, and at least one profile in every vhost. `https` preserves the serialized HTTPS transport. `https-lanes` gives stream zero and every logical stream independent uplink sequencing, downlink cursors, retries, and long polls; it requires `max_http_handlers >= 2` and public HTTP/2 on the TLS terminator. `websocket` carries all logical streams over one ordered RFC 6455 connection, while `websocket-lanes` owns one connection per non-zero logical stream and isolates lane failures. Both WebSocket carriers use `GET /api/v1/ws` after HTTPS session creation and require the TLS terminator to preserve HTTP/1.1 Upgrade headers. A reload applies `carrier` only to newly issued bridge sessions. Disabling WEB stops issuance of new bridge and session credentials after reload; use the users API to revoke one user's active sessions.
|
||||
|
||||
# [web.debug]
|
||||
|
||||
This hot-reloadable table controls the process-owned server-side WEB debug recorder exposed as authenticated HTML at `GET /web-status` on the API listener. Collection is disabled by default. Retained and in-flight records remain bounded by restart-only values in `[web.limits]`.
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
| --- | --- | --- | --- |
|
||||
| `enabled` | `bool` | `false` | Enables WEB HTTP, WebSocket-message, frame, and lifecycle debug records. |
|
||||
| `capture_lifecycle` | `bool` | `true` | Records typed bridge, session, stream, handshake, relay, and close events. |
|
||||
| `capture_headers` | `bool` | `true` | Retains header names and only allowlisted non-credential values. |
|
||||
| `capture_timings` | `bool` | `true` | Retains request-body, response-ready, response-body, and WebSocket message-processing timing points. |
|
||||
| `capture_frames` | `bool` | `true` | Parses bounded carrier bodies into frame type, stream ID, length, WINDOW, and error metadata without retaining frame payload separately. |
|
||||
| `body_capture` | `"off"`, `"metadata"`, `"prefix"`, or `"full"` | `"metadata"` | Controls request and response body byte retention. |
|
||||
| `body_prefix_bytes` | `usize` | `4096` | Prefix retained for recognized WEB bodies in `prefix` mode. |
|
||||
| `decoy_body_prefix_bytes` | `usize` | `4096` | Maximum retained prefix for ordinary decoy traffic in both `prefix` and `full` modes. |
|
||||
| `default_window_secs` | `u64` | `180` | Default observation window rendered by `/web-status`. |
|
||||
| `max_window_secs` | `u64` | `3600` | Largest observation window accepted by `/web-status`; validated at no more than 86400. |
|
||||
|
||||
Changing `enabled` or any capture field clears retained records and rejects commits started under the previous policy epoch. Changing only the default or maximum observation window preserves compatible retained records. `full` retains a complete recognized carrier body only up to `web.limits.max_body_bytes`; decoy bodies always remain prefix-bounded. A prefix that depends on a simultaneously increased restart-only capacity is deferred with `web.debug` until restart. URI queries are never retained, credential header values are omitted, body copies are scrubbed for known WEB capabilities and bearer tokens, and profile keys are represented only by a domain-separated 16-hex fingerprint.
|
||||
|
||||
# [web.limits]
|
||||
|
||||
@@ -2576,6 +2597,10 @@ These process-wide ceilings make every WEB registry, queue, request body, static
|
||||
| `max_frames_per_body` | `usize` | `4096` | Maximum frames parsed or emitted per carrier body. |
|
||||
| `max_http_connections` | `usize` | `1024` | Accepted WEB HTTP connections process-wide. |
|
||||
| `max_http_handlers` | `usize` | `512` | Concurrent HTTP handlers process-wide; HTTPS lanes may park at most half, preserving the remainder for session, uplink, and control work. |
|
||||
| `websocket_bytes_global` | `usize` | `268435456` | Transient WebSocket codec, message, and write-staging sub-budget inside `pending_bytes_global`. |
|
||||
| `websocket_admission_watermark_pct` | `u8` | `75` | WebSocket byte percentage at which new admission may replace an owner-first victim. |
|
||||
| `websocket_eviction_watermark_pct` | `u8` | `90` | WebSocket byte percentage at which queue pressure may evict the least-recently-progressed eligible connection. |
|
||||
| `websocket_http_connection_reserve` | `usize` | `64` | Accepted HTTP connections unavailable to WebSocket upgrades, preserving ordinary HTTP and decoy capacity. |
|
||||
| `max_body_readers` | `usize` | `32` | Concurrent collected request bodies process-wide. |
|
||||
| `max_body_bytes_global` | `usize` | `67108864` | Global byte reservation for collected bodies. |
|
||||
| `max_sessions_global` | `usize` | `128` | Live WEB sessions process-wide. |
|
||||
@@ -2597,7 +2622,9 @@ These process-wide ceilings make every WEB registry, queue, request body, static
|
||||
| `max_static_files` | `usize` | `4096` | Static snapshot entries across all vhosts. |
|
||||
| `max_static_file_bytes` | `usize` | `8388608` | Maximum bytes in one static file. |
|
||||
| `max_static_bytes` | `usize` | `67108864` | Static snapshot bytes across all vhosts. |
|
||||
| `memory_envelope_bytes` | `usize` | `805306368` | Declared envelope for HTTP heads, bodies, queues, and static snapshots; maximum 4 GiB. |
|
||||
| `debug_records_capacity` | `usize` | `65536` | Maximum retained WEB debug record count. |
|
||||
| `debug_bytes_global` | `usize` | `67108864` | Retained plus in-flight WEB debug byte ceiling; minimum 4096. |
|
||||
| `memory_envelope_bytes` | `usize` | `805306368` | Declared envelope for HTTP heads, bodies, shared queues/WebSocket I/O, static snapshots, and bounded debug/status buffers; maximum 4 GiB. |
|
||||
| `new_bootstraps_per_minute` | `u32` | `1200` | Sustained process-wide bootstrap issuance rate. |
|
||||
| `new_bootstraps_burst` | `u32` | `256` | Process-wide bootstrap issuance burst. |
|
||||
| `new_sessions_per_minute` | `u32` | `600` | Sustained process-wide session creation rate. |
|
||||
@@ -2615,6 +2642,9 @@ Every timeout is measured in seconds and must be within `1..=3600`. The longest
|
||||
| `body_secs` | `u64` | `30` | `✔` | Collect one authenticated carrier body. |
|
||||
| `stream_handshake_secs` | `u64` | `10` | `✔` | Complete one inner MTProxy handshake. |
|
||||
| `long_poll_secs` | `u64` | `25` | `✔` | Maximum empty downlink long poll. |
|
||||
| `websocket_write_secs` | `u64` | `30` | `✔` | Maximum wait for one WebSocket write or flush. |
|
||||
| `websocket_backpressure_secs` | `u64` | `30` | `✔` | Maximum wait for shared byte-budget or queue progress before closing the affected connection. |
|
||||
| `websocket_eviction_secs` | `u64` | `1` | `✔` | Grace allowed for a pressure-evicted WebSocket to release its slot and budget before admission fails. |
|
||||
| `bootstrap_lifetime_secs` | `u64` | `120` | `✔` | Unused bootstrap and closed-token replay lifetime. |
|
||||
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Maximum carrier inactivity before session closure. |
|
||||
| `http_idle_secs` | `u64` | `75` | `✔` | WEB HTTP keep-alive idle lifetime. |
|
||||
@@ -2655,9 +2685,9 @@ Profile limits must be non-zero and no greater than their corresponding global l
|
||||
|
||||
## WEB lifecycle and API management
|
||||
|
||||
- The config watcher and generation reload apply `web.enabled`, `web.carrier`, `web.timeouts`, vhosts, profiles, and decoy snapshots without a process restart. Existing sessions keep their acquisition-time carrier, limits, and deadlines; newly issued bridge sessions use the active generation.
|
||||
- The config watcher and generation reload apply `web.enabled`, `web.carrier`, `web.debug`, `web.timeouts`, vhosts, profiles, and decoy snapshots without a process restart. Existing sessions keep their acquisition-time carrier, limits, and deadlines; newly issued bridge sessions use the active generation.
|
||||
- WEB listener inventory and trust policy under `server.listeners`, and every `web.limits` value, are process-owned and restart-required.
|
||||
- There is no dedicated `/v1/web` endpoint. `GET /v1/config` omits `[web]`, and `PATCH /v1/config` rejects a `web` key with `400 section_not_editable`.
|
||||
- There is no mutable `/v1/web` resource. `GET /web-status` provides authenticated read-only HTML diagnostics; `GET /v1/config` omits `[web]`, and `PATCH /v1/config` rejects a `web` key with `400 section_not_editable`.
|
||||
- To manage WEB policy remotely, update the owned TOML file and call `POST /v1/system/reload`; inspect `GET /v1/system/reload/{id}` and its `deferred_process_fields`. Restart Telemt when it contains `server.listeners` or `web.limits`.
|
||||
- Existing access users can be created, changed, rotated, enabled, disabled, and deleted through `/v1/users`. Creating a user does not add a WEB profile. Disabling a user immediately updates admission and cancels that user's active sessions.
|
||||
- `PATCH /v1/config` can persist `server.listeners`, including WEB listener fields, but a changed WEB listener does not become active until process restart.
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
- [server.api](#serverapi)
|
||||
- [server.listeners](#serverlisteners)
|
||||
- [web](#web)
|
||||
- [web.debug](#webdebug)
|
||||
- [web.limits](#weblimits)
|
||||
- [web.timeouts](#webtimeouts)
|
||||
- [web.vhosts](#webvhosts)
|
||||
@@ -2482,12 +2483,32 @@ WEB-режим переносит MTProxy-трафик Telegram Desktop внут
|
||||
| Ключ | Тип | По умолчанию | Hot-Reload |
|
||||
| --- | --- | --- | --- |
|
||||
| `enabled` | `bool` | `false` | `✔` |
|
||||
| `carrier` | `"https"` или `"https-lanes"` | `"https"` | `✔` |
|
||||
| `carrier` | `"https"`, `"https-lanes"`, `"websocket"` или `"websocket-lanes"` | `"https"` | `✔` |
|
||||
| `debug` | таблица | выключено, ограниченные defaults | `✔` |
|
||||
| `limits` | таблица | ограниченные defaults | `✘` |
|
||||
| `timeouts` | таблица | ограниченные defaults | `✔` |
|
||||
| `vhosts` | массив таблиц | `[]` | `✔` |
|
||||
|
||||
Для `enabled = true` нужен как минимум один доступный по сетевой политике WEB-listener, один vhost и один профиль в каждом vhost. `carrier = "https"` сохраняет сериализованный HTTPS transport. При `carrier = "https-lanes"` stream zero и каждый logical stream получают независимые uplink sequence, downlink cursor, retry и long poll; этот carrier требует `max_http_handlers >= 2` и публичного HTTP/2 на TLS-терминаторе, чтобы убрать application-level inter-stream head-of-line blocking. Reload применяет `carrier` только к новым bridge sessions. Отключение WEB после reload прекращает выдачу новых bridge- и session-credentials; для отзыва активных сессий отдельного пользователя используйте users API.
|
||||
Для `enabled = true` нужен как минимум один доступный по сетевой политике WEB-listener, один vhost и один профиль в каждом vhost. `https` сохраняет сериализованный HTTPS transport. В `https-lanes` stream zero и каждый logical stream получают независимые uplink sequence, downlink cursor, retry и long poll; carrier требует `max_http_handlers >= 2` и публичного HTTP/2 на TLS-терминаторе. `websocket` переносит все logical streams через одно упорядоченное RFC 6455 connection, а `websocket-lanes` выделяет отдельное connection каждому ненулевому stream и изолирует сбои lane. Оба WebSocket carrier используют `GET /api/v1/ws` после создания HTTPS-сессии и требуют от TLS-терминатора сохранять HTTP/1.1 Upgrade headers. Reload применяет `carrier` только к новым bridge sessions. Отключение WEB после reload прекращает выдачу новых bridge- и session-credentials; для отзыва активных сессий отдельного пользователя используйте users API.
|
||||
|
||||
# [web.debug]
|
||||
|
||||
Эта hot-reloadable таблица управляет process-owned серверным WEB debug recorder, доступным на API-listener’е как аутентифицированный HTML по `GET /web-status`. Сбор по умолчанию отключён. Сохранённые и находящиеся в обработке записи ограничены значениями из `[web.limits]`, изменение которых требует перезапуска.
|
||||
|
||||
| Ключ | Тип | По умолчанию | Описание |
|
||||
| --- | --- | --- | --- |
|
||||
| `enabled` | `bool` | `false` | Включает WEB HTTP, WebSocket-message, frame и lifecycle debug records. |
|
||||
| `capture_lifecycle` | `bool` | `true` | Записывает типизированные события bridge, session, stream, handshake, relay и close. |
|
||||
| `capture_headers` | `bool` | `true` | Сохраняет имена headers и только разрешённые значения без credentials. |
|
||||
| `capture_timings` | `bool` | `true` | Сохраняет timing points для request body, готового response, response body и обработки WebSocket messages. |
|
||||
| `capture_frames` | `bool` | `true` | Разбирает bounded carrier bodies в тип frame, stream ID, длину, WINDOW и метаданные ошибок, не сохраняя frame payload отдельно. |
|
||||
| `body_capture` | `"off"`, `"metadata"`, `"prefix"` или `"full"` | `"metadata"` | Управляет сохранением байтов request и response body. |
|
||||
| `body_prefix_bytes` | `usize` | `4096` | Prefix распознанного WEB body, сохраняемый в режиме `prefix`. |
|
||||
| `decoy_body_prefix_bytes` | `usize` | `4096` | Максимальный сохраняемый prefix обычного decoy-трафика в режимах `prefix` и `full`. |
|
||||
| `default_window_secs` | `u64` | `180` | Стандартное окно наблюдения, отображаемое `/web-status`. |
|
||||
| `max_window_secs` | `u64` | `3600` | Максимальное окно наблюдения, принимаемое `/web-status`; при валидации ограничено значением 86400. |
|
||||
|
||||
Изменение `enabled` или любого поля capture очищает сохранённые записи и отклоняет commits, начатые в предыдущую policy epoch. Изменение только стандартного или максимального окна наблюдения сохраняет совместимые записи. `full` сохраняет полное тело распознанного carrier только до `web.limits.max_body_bytes`; decoy bodies всегда остаются ограничены настроенным prefix. Prefix, который помещается только в одновременно увеличенную restart-only ёмкость, откладывается вместе с `web.debug` до перезапуска. URI queries никогда не сохраняются, значения credential headers исключаются, копии body очищаются от известных WEB capabilities и bearer tokens, а ключи профилей представлены только domain-separated fingerprint из 16 hex-символов.
|
||||
|
||||
# [web.limits]
|
||||
|
||||
@@ -2502,6 +2523,10 @@ WEB-режим переносит MTProxy-трафик Telegram Desktop внут
|
||||
| `max_frames_per_body` | `usize` | `4096` | Максимальное число frames в одном carrier body. |
|
||||
| `max_http_connections` | `usize` | `1024` | Принятые WEB HTTP connections на весь процесс. |
|
||||
| `max_http_handlers` | `usize` | `512` | Одновременно выполняемые HTTP handlers на весь процесс; HTTPS lanes могут занять long polls не более половины лимита, оставляя остаток для session, uplink и control work. |
|
||||
| `websocket_bytes_global` | `usize` | `268435456` | Подбюджет transient WebSocket codec, messages и write staging внутри `pending_bytes_global`. |
|
||||
| `websocket_admission_watermark_pct` | `u8` | `75` | Доля WebSocket byte-budget, после которой новый admission может вытеснить owner-first victim. |
|
||||
| `websocket_eviction_watermark_pct` | `u8` | `90` | Доля WebSocket byte-budget, после которой queue pressure может вытеснить подходящее connection с наиболее старым прогрессом. |
|
||||
| `websocket_http_connection_reserve` | `usize` | `64` | Число принятых HTTP connections, недоступных WebSocket upgrades и сохраняющих capacity для обычного HTTP и decoy. |
|
||||
| `max_body_readers` | `usize` | `32` | Одновременно собираемые request bodies на весь процесс. |
|
||||
| `max_body_bytes_global` | `usize` | `67108864` | Глобальный байтовый резерв для собранных bodies. |
|
||||
| `max_sessions_global` | `usize` | `128` | Активные WEB-сессии на весь процесс. |
|
||||
@@ -2523,7 +2548,9 @@ WEB-режим переносит MTProxy-трафик Telegram Desktop внут
|
||||
| `max_static_files` | `usize` | `4096` | Элементы static snapshot всех vhosts. |
|
||||
| `max_static_file_bytes` | `usize` | `8388608` | Максимальный размер одного статического файла. |
|
||||
| `max_static_bytes` | `usize` | `67108864` | Размер static snapshots всех vhosts. |
|
||||
| `memory_envelope_bytes` | `usize` | `805306368` | Заявленный envelope для HTTP heads, bodies, очередей и static snapshots; максимум 4 GiB. |
|
||||
| `debug_records_capacity` | `usize` | `65536` | Максимальное число сохранённых WEB debug records. |
|
||||
| `debug_bytes_global` | `usize` | `67108864` | Глобальная байтовая граница сохранённых и находящихся в обработке WEB debug данных; минимум 4096. |
|
||||
| `memory_envelope_bytes` | `usize` | `805306368` | Заявленный envelope для HTTP heads, bodies, общих queues/WebSocket I/O, static snapshots и bounded debug/status buffers; максимум 4 GiB. |
|
||||
| `new_bootstraps_per_minute` | `u32` | `1200` | Устойчивая process-wide скорость выдачи bootstrap. |
|
||||
| `new_bootstraps_burst` | `u32` | `256` | Process-wide burst выдачи bootstrap. |
|
||||
| `new_sessions_per_minute` | `u32` | `600` | Устойчивая process-wide скорость создания сессий. |
|
||||
@@ -2541,6 +2568,9 @@ WEB-режим переносит MTProxy-трафик Telegram Desktop внут
|
||||
| `body_secs` | `u64` | `30` | `✔` | Сбор одного аутентифицированного carrier body. |
|
||||
| `stream_handshake_secs` | `u64` | `10` | `✔` | Выполнение внутреннего MTProxy handshake. |
|
||||
| `long_poll_secs` | `u64` | `25` | `✔` | Максимальная длительность пустого downlink long poll. |
|
||||
| `websocket_write_secs` | `u64` | `30` | `✔` | Максимальное ожидание одной WebSocket write или flush операции. |
|
||||
| `websocket_backpressure_secs` | `u64` | `30` | `✔` | Максимальное ожидание прогресса общего byte-budget или queue перед закрытием затронутого connection. |
|
||||
| `websocket_eviction_secs` | `u64` | `1` | `✔` | Grace period для освобождения slot и budget вытесненным WebSocket до отказа admission. |
|
||||
| `bootstrap_lifetime_secs` | `u64` | `120` | `✔` | Срок неиспользованного bootstrap и replay-marker закрытого token. |
|
||||
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Максимальная неактивность carrier до закрытия сессии. |
|
||||
| `http_idle_secs` | `u64` | `75` | `✔` | Idle lifetime WEB HTTP keep-alive connection. |
|
||||
@@ -2581,9 +2611,9 @@ Hostname нормализуется при валидации и должен п
|
||||
|
||||
## Lifecycle WEB и управление через API
|
||||
|
||||
- Config watcher и generation reload применяют `web.enabled`, `web.carrier`, `web.timeouts`, vhosts, profiles и decoy snapshots без перезапуска процесса. Существующие сессии сохраняют carrier, лимиты и deadlines своего момента создания; новые bridge sessions используют активное поколение.
|
||||
- Config watcher и generation reload применяют `web.enabled`, `web.carrier`, `web.debug`, `web.timeouts`, vhosts, profiles и decoy snapshots без перезапуска процесса. Существующие сессии сохраняют carrier, лимиты и deadlines своего момента создания; новые bridge sessions используют активное поколение.
|
||||
- Состав WEB-listeners и их trust policy в `server.listeners`, а также все значения `web.limits` принадлежат процессу и требуют перезапуска.
|
||||
- Отдельного endpoint `/v1/web` нет. `GET /v1/config` не возвращает `[web]`, а `PATCH /v1/config` отклоняет ключ `web` с `400 section_not_editable`.
|
||||
- Изменяемого ресурса `/v1/web` нет. `GET /web-status` предоставляет аутентифицированную read-only HTML-диагностику; `GET /v1/config` не возвращает `[web]`, а `PATCH /v1/config` отклоняет ключ `web` с `400 section_not_editable`.
|
||||
- Для удалённого применения WEB policy измените соответствующий TOML-файл и вызовите `POST /v1/system/reload`; проверьте `GET /v1/system/reload/{id}` и поле `deferred_process_fields`. Если оно содержит `server.listeners` или `web.limits`, перезапустите Telemt.
|
||||
- Существующих access users можно создавать, изменять, ротировать, включать, выключать и удалять через `/v1/users`. Создание пользователя не добавляет WEB-профиль. Отключение пользователя немедленно обновляет admission и завершает его активные сессии.
|
||||
- `PATCH /v1/config` может сохранить `server.listeners`, включая поля WEB-listener’а, но изменённый WEB-listener активируется только после перезапуска процесса.
|
||||
|
||||
+56
-15
@@ -2,7 +2,7 @@
|
||||
|
||||
[English](WEB_PROXY.en.md) | [Русский](WEB_PROXY.ru.md) | [Deutsch](WEB_PROXY.de.md)
|
||||
|
||||
Der WEB-Modus transportiert gewöhnliche MTProxy-Streams über begrenzte HTTPS-Carrier, die mit dem Proxy-Typ `WEB` von Telegram Desktop kompatibel sind. Telemt terminiert TLS nicht selbst: NGINX oder HAProxy verwaltet das öffentliche Zertifikat und leitet unverschlüsseltes HTTP/1.1 an einen privaten Telemt-Listener weiter.
|
||||
Der WEB-Modus transportiert gewöhnliche MTProxy-Streams über begrenzte HTTPS- oder WebSocket-Carrier, die mit dem Proxy-Typ `WEB` von Telegram Desktop kompatibel sind. Telemt terminiert TLS nicht selbst: NGINX oder HAProxy verwaltet das öffentliche Zertifikat und leitet unverschlüsseltes HTTP/1.1 an einen privaten Telemt-Listener weiter.
|
||||
|
||||
> [!IMPORTANT]
|
||||
>
|
||||
@@ -12,7 +12,7 @@ Der WEB-Modus transportiert gewöhnliche MTProxy-Streams über begrenzte HTTPS-C
|
||||
|
||||
```text
|
||||
Telegram Desktop
|
||||
| HTTPS :443
|
||||
| HTTPS oder WSS :443
|
||||
v
|
||||
NGINX oder HAProxy (TLS-Terminierung, kanonischer Host und eine X-Forwarded-For-Adresse)
|
||||
| unverschlüsseltes HTTP/1.1 in einem privaten Netz
|
||||
@@ -28,7 +28,7 @@ Leiten Sie den vollständigen öffentlichen vhost an Telemt weiter. Wenn der TLS
|
||||
|
||||
- Der öffentliche Endpunkt ist immer `https://HOST:443`.
|
||||
- Unterstützt werden 16-Byte-MTProxy-Secrets in den Modi `plain` und `dd`. FakeTLS-Secrets mit `ee` werden im WEB-Modus nicht unterstützt.
|
||||
- `web.carrier = "https"` wählt serialisierte HTTPS-Uplinks und Long Polling. `web.carrier = "https-lanes"` wählt unabhängige HTTPS-Sequenzen und Polls pro logischem Stream. WebSocket-Carrier werden nicht angeboten.
|
||||
- `web.carrier = "https"` wählt serialisierte HTTPS-Uplinks und Long Polling. `https-lanes` wählt unabhängige HTTPS-Sequenzen und Polls pro logischem Stream. `websocket` wählt einen geordneten WebSocket für alle Streams. `websocket-lanes` wählt einen unabhängig verwalteten WebSocket für jeden logischen Stream ungleich null.
|
||||
- Capability-, Bootstrap- und Session-Zugangsdaten sind getrennte Werte mit begrenzter Lebensdauer. Carrier-Zugangsdaten sind geheim und dürfen nicht in Access-Logs erscheinen.
|
||||
- Ein Bootstrap ist ein Bearer-Token und nicht an eine Quelladresse gebunden. Client-Adresse und IP-Familie dürfen sich zwischen dem Laden der Bridge und der Sitzungserstellung ändern. Die Ausstellungsadresse bleibt dem Limit ungenutzter Bootstraps zugeordnet; die Adresse des ersten gültigen Erstellungs-Requests wird der Sitzung zugeordnet.
|
||||
- Die innere MTProxy-Authentifizierung ist auf den Benutzer und Secret-Modus des vhost-Profils beschränkt. Ein ungültiger innerer Handshake schließt nur seinen logischen Stream und gelangt niemals in den TCP-Masking-Pfad.
|
||||
@@ -99,6 +99,12 @@ Alle Lane-Queues bleiben innerhalb der vorhandenen Byte-/Item-Budgets pro Sitzun
|
||||
|
||||
Die Pfade `/api/v1/up` und `/api/v1/down` ändern sich nicht. Bei `https-lanes` enthält jeder Request an diese Pfade genau einen kanonischen dezimalen `X-Lane-ID`-Header. Die Uplink-Sequenz beginnt pro Lane unabhängig bei `1`, der Downlink-Cursor bei `0`. Lane null akzeptiert nur Session-`PONG`; jeder Frame einer Lane ungleich null muss dieselbe Stream-ID tragen, und eine neue Lane muss mit `OPEN` beginnen. Nachdem eingereihte und nicht bestätigte Downlink-Daten einer geschlossenen Lane vollständig abgearbeitet sind, antwortet Telemt leer mit `X-Lane-Closed: 1`, und die Bridge beendet deren Polling. Wiederholungen bleiben byte-identisch und spielen die ursprüngliche Bestätigung oder den Downlink-Batch erneut aus.
|
||||
|
||||
Beide WebSocket-Carrier erstellen und löschen die übergeordnete Sitzung weiterhin über HTTPS und verwenden danach einen strikten Upgrade-Request ohne Body an `GET /api/v1/ws`. `websocket` übermittelt in `Sec-WebSocket-Protocol` exakt `tproxy-v1.<session-token>`; binäre Messages sind geordnete Carrier-Batches, und ein Protokoll-, Deadline- oder Verbindungsfehler schließt die gesamte übergeordnete Sitzung. `websocket-lanes` übermittelt exakt `tproxy-lane-v1.<session-token>.<stream-id>`, wobei die Stream-ID kanonisch dezimal im Bereich `1..=16777215` steht. Die erste binäre Message muss mit `OPEN` beginnen, alle Frames müssen diese Stream-ID verwenden und ein Fehler nach dem Upgrade schließt nur diese Lane. Es gibt keinen Lane-null-WebSocket: HTTPS transportiert `HELLO` und `WELCOME`, während RFC-6455-Ping/Pong die Verbindungsliveness gewährleistet.
|
||||
|
||||
WebSocket-Codec-Puffer und laufende Read-/Write-Messages teilen das prozesseigene Budget `pending_bytes_global` mit den Carrier-Queues und sind zusätzlich durch `websocket_bytes_global` begrenzt. Admission reserviert `websocket_http_connection_reserve` angenommene Verbindungen für gewöhnliches HTTP und Decoys. Unter Druck erfolgt die Verdrängung zuerst beim selben Owner und danach nach dem ältesten Fortschritt; Pre-Upgrade- und tote Verbindungen stehen vor aktiven Lanes und multiplexierten Sitzungen. Nach `long_poll_secs` ohne Peer-Aktivität wird auch bei kontinuierlichem Downlink-Verkehr ein Transport-Ping gesendet; fehlende Peer-Aktivität während des doppelten, beim Verbindungsaufbau festgelegten Intervalls macht die Verbindung zum Cleanup-Kandidaten.
|
||||
|
||||
Jeder Authentifizierungs-, Shape-, Lane-Reservierungs- oder Kapazitätsfehler vor dem Upgrade folgt dem bereinigten Decoy-Pfad und legt keinen WebSocket-spezifischen Status offen. Das exakte Subprotokoll enthält den Session-Bearer und darf nicht protokolliert werden.
|
||||
|
||||
Der WEB-Listener muss `proxy_protocol = false` und `reuse_allow = false` verwenden. `client_mss`, `synlimit`, `announce` und `announce_ip` sind nicht zulässig. `web_trusted_proxy_cidrs` muss nicht leer sein und darf nur die unmittelbar vorgeschalteten NGINX- oder HAProxy-Peers enthalten; `/0`-Netze werden abgelehnt.
|
||||
|
||||
Der HTTP-Decoy-Origin muss eine Loopback-, Link-Local- oder private IP-Adresse als Literal verwenden. Telemt bewahrt bei gewöhnlichen Requests Methode, Pfad, Query, Header, gestreamten Body, Response-Status, Header und Body und entfernt Hop-by-Hop-Header. Vor dem Fallback auf den Decoy entfernt Telemt Carrier-Zugangsdaten und Bodys aus fehlerhaften Carrier-Requests.
|
||||
@@ -119,6 +125,11 @@ Alle WEB-Schlüssel und Defaults sind in der [Konfigurationsreferenz](../Config_
|
||||
## TLS-Terminierung mit NGINX
|
||||
|
||||
```nginx
|
||||
map $http_upgrade $telemt_connection_upgrade {
|
||||
default upgrade;
|
||||
'' '';
|
||||
}
|
||||
|
||||
upstream telemt_web {
|
||||
server 127.0.0.1:18080;
|
||||
keepalive 64;
|
||||
@@ -140,11 +151,12 @@ server {
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $telemt_connection_upgrade;
|
||||
|
||||
proxy_connect_timeout 5s;
|
||||
proxy_send_timeout 35s;
|
||||
proxy_read_timeout 35s;
|
||||
proxy_send_timeout 65s;
|
||||
proxy_read_timeout 65s;
|
||||
proxy_request_buffering off;
|
||||
proxy_buffering off;
|
||||
proxy_next_upstream off;
|
||||
@@ -152,9 +164,9 @@ server {
|
||||
}
|
||||
```
|
||||
|
||||
`client_max_body_size` muss mindestens `web.limits.max_body_bytes` entsprechen. `proxy_read_timeout` und `proxy_send_timeout` müssen größer als `web.timeouts.long_poll_secs` sein, dessen Default 25 Sekunden beträgt. Überschreiben Sie `X-Forwarded-For`, statt einen Wert anzuhängen. Telemt akzeptiert eine syntaktisch gültige IP-Adresse; fehlt der Header bei einem vertrauenswürdigen TLS-Terminator, verwendet Telemt die Adresse des direkten Peers, doch clientbezogene Limits und Quellrichtlinien sehen dann den Terminator statt des echten Clients. Aktivieren Sie keine Upstream-Wiederholungen: Der Bridge-Transport führt byte-identische Wiederholungen über sein eigenes Sequenzprotokoll aus.
|
||||
Platzieren Sie `map` im NGINX-Kontext `http`. `client_max_body_size` muss mindestens `web.limits.max_body_bytes` entsprechen. Read-, Send- und Client-Timeouts müssen sowohl den standardmäßigen 25-Sekunden-Long-Poll als auch das doppelte WebSocket-Liveness-Intervall überschreiten; 65 Sekunden decken die Defaults ab. Überschreiben Sie `X-Forwarded-For`, statt einen Wert anzuhängen. Telemt akzeptiert eine syntaktisch gültige IP-Adresse; fehlt der Header bei einem vertrauenswürdigen TLS-Terminator, verwendet Telemt die Adresse des direkten Peers, doch clientbezogene Limits und Quellrichtlinien sehen dann den Terminator statt des echten Clients. Aktivieren Sie keine Upstream-Wiederholungen: Die Bridge führt byte-identische HTTPS-Wiederholungen aus, ein etablierter WebSocket wird jedoch nie transparent wiederholt.
|
||||
|
||||
Öffentliches HTTP/2 ist für `https-lanes` obligatorisch; verwenden Sie die entsprechende HTTP/2-Direktive der installierten NGINX-Version. Der private Hop von NGINX zu Telemt bleibt absichtlich HTTP/1.1. Die Upstream-Verbindungskapazität muss die erwarteten gleichzeitigen Lane-Polls tragen; `keepalive` steuert den Idle-Pool und ist keine Nebenläufigkeitsgrenze.
|
||||
Öffentliches HTTP/2 ist für `https-lanes` obligatorisch; verwenden Sie die entsprechende HTTP/2-Direktive der installierten NGINX-Version. WebSocket-Upgrade erfordert HTTP/1.1, daher muss der öffentliche Endpunkt auch HTTP/1.1 zulassen und der private Hop von NGINX zu Telemt bleibt HTTP/1.1. Bewahren Sie `Connection`, `Upgrade` und `Sec-WebSocket-*` wie gezeigt unverändert. Die Upstream-Verbindungskapazität muss die erwarteten gleichzeitigen Lane-Polls oder WebSocket-Lanes tragen; `keepalive` steuert den Idle-Pool und ist keine Nebenläufigkeitsgrenze.
|
||||
|
||||
## TLS-Terminierung mit HAProxy
|
||||
|
||||
@@ -171,14 +183,14 @@ backend telemt_web
|
||||
option http-keep-alive
|
||||
retries 0
|
||||
timeout connect 5s
|
||||
timeout server 35s
|
||||
timeout server 65s
|
||||
http-request set-header Host proxy.example.com
|
||||
http-request del-header X-Forwarded-For
|
||||
http-request set-header X-Forwarded-For %[src]
|
||||
server telemt_web_1 127.0.0.1:18080 check
|
||||
```
|
||||
|
||||
Im Frontend oder im Abschnitt `defaults` muss auch `timeout client` oberhalb der Long-Poll-Deadline liegen. Für `https-lanes` muss das öffentliche HAProxy-ALPN `h2` enthalten. Pfad, Raw Query, Body sowie die Carrier-Header `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor` und `X-Lane-ID` dürfen nicht umgeschrieben werden.
|
||||
Im Frontend oder im Abschnitt `defaults` muss für das standardmäßige WebSocket-Liveness-Intervall auch `timeout client 65s` oder länger gesetzt sein. Für `https-lanes` muss das öffentliche HAProxy-ALPN `h2`, für WebSocket-Upgrade außerdem `http/1.1` enthalten. Bewahren Sie `Connection`, `Upgrade` und `Sec-WebSocket-*` unverändert; Pfad, Raw Query, Body sowie die Carrier-Header `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor` und `X-Lane-ID` dürfen nicht umgeschrieben werden.
|
||||
|
||||
## Lebenszyklus und Reload-Verhalten
|
||||
|
||||
@@ -186,21 +198,22 @@ Im Frontend oder im Abschnitt `defaults` muss auch `timeout client` oberhalb der
|
||||
| --- | --- |
|
||||
| Bestand der WEB-Listener, Bind-Adresse und Vertrauensrichtlinie | Prozesseigen; Telemt neu starten. |
|
||||
| Jeder Wert in `[web.limits]` | Prozesseigener Speicher- und Ressourcenvertrag; Telemt neu starten. |
|
||||
| `web.enabled`, `web.carrier`, Timeouts, vhosts, Profile und Decoys | Werden vom Config-Watcher oder durch einen Runtime-Generations-Reload angewendet. |
|
||||
| Bestehende HTTP-Verbindungen und WEB-Sitzungen | Behalten Carrier, Grenzen und Deadlines ihres Erstellungszeitpunkts; neu ausgegebene Bridge-Sitzungen verwenden den aktiven Carrier. Neue logische Streams verwenden die aktive Relay-Generation. |
|
||||
| `web.enabled`, `web.carrier`, `web.debug`, Timeouts, vhosts, Profile und Decoys | Werden vom Config-Watcher oder durch einen Runtime-Generations-Reload angewendet. |
|
||||
| Bestehende HTTP-Verbindungen und WEB-Sitzungen | Behalten Carrier, Grenzen und Session-Deadlines ihres Erstellungszeitpunkts; neu ausgegebene Bridge-Sitzungen verwenden den aktiven Carrier. WebSocket-Write-, Backpressure- und Eviction-Vorgänge lesen die aktiven hot-reload-fähigen Deadlines. Neue logische Streams verwenden die aktive Relay-Generation. |
|
||||
| Beenden des Prozesses | Verwendet den zuletzt geladenen Wert von `web.timeouts.shutdown_secs`. |
|
||||
|
||||
Jeder logische Stream behält die Client-IP seiner Sitzung und besitzt während der gesamten Relay-Lebensdauer einen prozessweit eindeutigen, von null verschiedenen synthetischen Quellport. Damit bleibt für Direct- und Middle-End-KDF-Routing ein stabiles, kollisionsfreies Quell-/Ziel-Tupel erhalten.
|
||||
|
||||
## Verwaltung über die API
|
||||
|
||||
API-Verwaltung ist verfügbar, aber absichtlich eingeschränkt. Es gibt weder einen eigenen Endpunkt `/v1/web` noch einen WEB-spezifischen Runtime-Statistik-Endpunkt.
|
||||
API-Verwaltung ist verfügbar, aber absichtlich eingeschränkt. Es gibt keine veränderbare Ressource `/v1/web`; der API-Listener stellt die schreibgeschützte HTML-Debug-Ansicht unter `/web-status` bereit.
|
||||
|
||||
| Operation | API-Unterstützung |
|
||||
| --- | --- |
|
||||
| `[web]`, vhosts, Profile, Decoys, Timeouts oder Limits lesen oder ändern | Nein. `GET /v1/config` lässt `[web]` aus; `PATCH /v1/config` antwortet für `web` mit `400 section_not_editable`. |
|
||||
| `server.listeners` speichern | Ja, über `PATCH /v1/config`; ein geänderter WEB-Listener bleibt jedoch bis zum Prozessneustart zurückgestellt. |
|
||||
| Außerhalb der API geänderte WEB-Konfiguration anwenden | Ja, über `POST /v1/system/reload` und anschließende Abfrage des Vorgangsstatus. |
|
||||
| Begrenzte serverseitige WEB-Request- und Lifecycle-Details untersuchen | Ja, über ein authentifiziertes `GET /web-status`. |
|
||||
| `[access.users]` verwalten | Ja, über `/v1/users`. Das Erstellen eines Benutzers erzeugt kein WEB-Profil. |
|
||||
| Einen Benutzer widerrufen | Ja. `/v1/users/{username}/disable` aktualisiert die Admission sofort und beendet die aktiven Sitzungen dieses Benutzers. |
|
||||
|
||||
@@ -217,6 +230,30 @@ read_only = false
|
||||
|
||||
Die API-Whitelist prüft den direkten TCP-Peer und vertraut `X-Forwarded-For` nicht. Änderungen an `[server.api]` selbst erfordern einen Prozessneustart.
|
||||
|
||||
### Serverseitige WEB-Debug-Ansicht
|
||||
|
||||
Aktivieren Sie die begrenzte Erfassung in der zuständigen Konfigurationsdatei:
|
||||
|
||||
```toml
|
||||
[web.debug]
|
||||
enabled = true
|
||||
capture_lifecycle = true
|
||||
capture_headers = true
|
||||
capture_timings = true
|
||||
capture_frames = true
|
||||
body_capture = "metadata"
|
||||
body_prefix_bytes = 4096
|
||||
decoy_body_prefix_bytes = 4096
|
||||
default_window_secs = 180
|
||||
max_window_secs = 3600
|
||||
```
|
||||
|
||||
Öffnen Sie `http://127.0.0.1:9091/web-status` mit derselben Whitelist direkter Peers und demselben exakten `Authorization`-Header wie für die API. Ein abschließender Slash wird akzeptiert. Nur `GET` ist zulässig. Die Seite unterstützt die Filter `window_secs`, kanonische `ip`, numerische `session`, `user_agent` ohne Beachtung der Groß-/Kleinschreibung und `key`. Wiederholen Sie `group_by=ip`, `group_by=session`, `group_by=user_agent` oder `group_by=key`, um gruppierte Zusammenfassungen zu erstellen; `limit` ist auf `1..=1000` beschränkt. HTTP-Zeilen lassen sich vom Request bis zur Response zu Methode, Pfad, bereinigten Headern, Body-Metadaten oder -Bytes, Zeitpunkten, Frames und typisierten Lifecycle-Ereignissen aufklappen. Für WebSocket kommen der bereinigte Handshake `GET` → `101` sowie begrenzte Angaben pro Message zu Richtung, Message-Typ, Payload-/Body-Erfassung, Verarbeitungszeit, Verbindungs-/Lane-ID und geparsten inneren Frames hinzu. Rohe Subprotokolle und Session-Tokens werden nie gespeichert.
|
||||
|
||||
Der prozesseigene Ring übersteht den Austausch einer Runtime-Generation. Änderungen der Erfassungs-Policy löschen inkompatible gespeicherte Datensätze; reine Änderungen des Beobachtungsfensters tun dies nicht. Der Ring ist standardmäßig auf 65536 Datensätze und 64 MiB gespeicherte plus in Verarbeitung befindliche Daten begrenzt, die HTML-Response auf 8 MiB und die Gruppierung auf 1024 Gruppen; gleichzeitig dürfen höchstens zwei Response-Bodys Seiten-Permits halten. Ändern Sie `web.limits.debug_records_capacity` oder `web.limits.debug_bytes_global` nur zusammen mit einem Prozessneustart. Ein hot-reload-fähiger Präfix, der nur in eine gleichzeitig erhöhte neustartpflichtige Kapazität passt, wird bis zu diesem Neustart zurückgestellt.
|
||||
|
||||
`body_capture = "off"` lässt Bodys aus, `metadata` speichert Längen und Endzustände, `prefix` die konfigurierten Präfixe und `full` erkannte Carrier-Bodys bis `web.limits.max_body_bytes`. Gewöhnliche Decoy-Bodys bleiben auch in `full` auf `decoy_body_prefix_bytes` begrenzt. Queries und rohe Capabilities werden nie gespeichert; Werte von Credential-Headern werden ausgelassen; bekannte WEB-Capabilities und Bearer-Tokens werden aus erfassten Bodys entfernt; der angezeigte Schlüssel ist ein nicht geheimer, domänengetrennter Fingerprint. Die Zeitmessung endet beim Polling des Hyper-Bodys und behauptet weder einen Kernel-Flush noch eine TCP-Bestätigung.
|
||||
|
||||
Nachdem ein Administrator oder Konfigurationssystem die TOML-Datei atomar aktualisiert hat, setzen Sie `TELEMT_API_AUTH` auf den exakten Wert von `auth_header` und starten Sie einen beobachtbaren Generations-Reload:
|
||||
|
||||
```bash
|
||||
@@ -264,8 +301,9 @@ Der vollständige Vertrag für Requests, Revisionen, Fehler und alle Benutzer-En
|
||||
3. Prüfen Sie, dass Telemt genau eine syntaktisch gültige `X-Forwarded-For`-Adresse und `Host: proxy.example.com` oder `Host: proxy.example.com:443` erhält.
|
||||
4. Importieren Sie den ausgegebenen `tg://webproxy`-Link in den vorgesehenen Telegram-Desktop-Build und stellen Sie eine Proxy-Verbindung her.
|
||||
5. Bestätigen Sie für `https-lanes`, dass die öffentliche Verbindung HTTP/2 ausgehandelt hat, und testen Sie mindestens zwei gleichzeitige logische Streams; der private Hop zu Telemt bleibt HTTP/1.1.
|
||||
6. Testen Sie einen Reconnect und mindestens einen Long Poll über 25 Sekunden, um sicherzustellen, dass Frontend-Timeouts den Carrier nicht abbrechen.
|
||||
7. Prüfen Sie Benutzer- und logische MTProxy-Verbindungslimits anhand der Logical-Stream-Zähler und nicht anhand der Zahl der HTTP-Verbindungen.
|
||||
6. Bestätigen Sie für `websocket` eine `101`-Response, binären Relay-Datenverkehr und RFC-6455-Ping/Pong nach 25 Sekunden. Testen Sie für `websocket-lanes` mindestens zwei gleichzeitige Stream-Sockets und prüfen Sie, dass das Schließen oder Beschädigen einer Lane weder Geschwister noch die übergeordnete Sitzung schließt.
|
||||
7. Testen Sie einen Reconnect und mindestens einen Long Poll über 25 Sekunden, um sicherzustellen, dass Frontend-Timeouts den Carrier nicht abbrechen.
|
||||
8. Prüfen Sie Benutzer- und logische MTProxy-Verbindungslimits anhand der Logical-Stream-Zähler und nicht anhand der Zahl der HTTP-Verbindungen.
|
||||
|
||||
## Fehlerbehebung
|
||||
|
||||
@@ -274,6 +312,9 @@ Der vollständige Vertrag für Requests, Revisionen, Fehler und alle Benutzer-En
|
||||
| WEB-Konfiguration ist auf dem Datenträger gültig, aber das Listener-Verhalten hat sich nicht geändert | Prüfen Sie `deferred_process_fields`; Listener- und `[web.limits]`-Änderungen erfordern einen Neustart. |
|
||||
| Carrier-Requests erreichen den Decoy | Prüfen Sie den exakten vhost, den Secret-Modus des Links, das CIDR des direkten Proxys und genau einen syntaktisch gültigen `X-Forwarded-For`-Wert. |
|
||||
| Long Polls werden nach einem festen Intervall getrennt | Setzen Sie Client-, Server-, Sende- und Lese-Timeouts von NGINX/HAProxy über `web.timeouts.long_poll_secs`. |
|
||||
| WebSocket-Upgrade erreicht statt `101` den Decoy | Bewahren Sie HTTP/1.1 `Connection: Upgrade`, `Upgrade: websocket`, das einzelne exakte `Sec-WebSocket-Protocol` und den kanonischen bodylosen Request `/api/v1/ws`. Prüfen Sie außerdem Carrier-/Session-Kompatibilität und die Prozess-Verbindungsreserve. |
|
||||
| Ein `websocket-lanes`-Stream wurde geschlossen, Geschwister bleiben aber verbunden | Dies ist die beabsichtigte Fehlergrenze. Prüfen Sie die Message-/Frame-Zeilen dieser Lane in `/web-status`; fehlerhafte oder lane-fremde Frames, Write-Timeouts und Backend-Close schließen nur die betroffene Lane. |
|
||||
| `/web-status` ist leer | Prüfen Sie, dass `[web.debug].enabled = true` gesetzt ist, wenden Sie die Konfiguration an, wählen Sie ein Fenster innerhalb von `max_window_secs` und erzeugen Sie nach der Policy-Änderung neuen WEB-Datenverkehr. |
|
||||
| `https-lanes` funktioniert, Streams blockieren sich aber weiterhin | Prüfen Sie die öffentliche HTTP/2-Aushandlung, die unveränderte Weitergabe von `X-Lane-ID` und genügend TLS-Terminator-Upstream-Verbindungen für parallele private HTTP/1.1-Polls. |
|
||||
| Telegram Desktop lehnt den Link ab | Lassen Sie den Port weg und verwenden Sie einen gültigen FQDN, extern Port 443 sowie ausschließlich `plain` oder `dd`. |
|
||||
| Ein Knoten funktioniert, ein Load-Balancing-Pool aber nur sporadisch | Konfigurieren Sie Affinität für den gesamten vhost; WEB-Zugangsdatenregister sind prozesslokal. |
|
||||
|
||||
+56
-15
@@ -2,7 +2,7 @@
|
||||
|
||||
[English](WEB_PROXY.en.md) | [Русский](WEB_PROXY.ru.md) | [Deutsch](WEB_PROXY.de.md)
|
||||
|
||||
WEB mode carries ordinary MTProxy streams through bounded HTTPS carriers compatible with Telegram Desktop's `WEB` proxy type. Telemt does not terminate TLS: NGINX or HAProxy owns the public certificate and forwards plain HTTP/1.1 to a private Telemt listener.
|
||||
WEB mode carries ordinary MTProxy streams through bounded HTTPS or WebSocket carriers compatible with Telegram Desktop's `WEB` proxy type. Telemt does not terminate TLS: NGINX or HAProxy owns the public certificate and forwards plain HTTP/1.1 to a private Telemt listener.
|
||||
|
||||
> [!IMPORTANT]
|
||||
>
|
||||
@@ -12,7 +12,7 @@ WEB mode carries ordinary MTProxy streams through bounded HTTPS carriers compati
|
||||
|
||||
```text
|
||||
Telegram Desktop
|
||||
| HTTPS :443
|
||||
| HTTPS or WSS :443
|
||||
v
|
||||
NGINX or HAProxy (TLS termination, canonical Host and one X-Forwarded-For address)
|
||||
| plain HTTP/1.1 on a private network
|
||||
@@ -28,7 +28,7 @@ Route the complete public vhost to Telemt. Splitting only recognized carrier pat
|
||||
|
||||
- The public endpoint is always `https://HOST:443`.
|
||||
- `plain` and `dd` 16-byte MTProxy secrets are supported. `ee` FakeTLS secrets are not supported by WEB mode.
|
||||
- `web.carrier = "https"` selects serialized HTTPS uplink and long polling. `web.carrier = "https-lanes"` selects independent HTTPS sequencing and polling per logical stream. WebSocket carriers are not advertised.
|
||||
- `web.carrier = "https"` selects serialized HTTPS uplink and long polling. `https-lanes` selects independent HTTPS sequencing and polling per logical stream. `websocket` selects one ordered WebSocket for all streams. `websocket-lanes` selects one independently owned WebSocket per non-zero logical stream.
|
||||
- Capability, bootstrap, and session credentials are separate bounded-lifetime values. Carrier credentials must be treated as secrets and must not appear in access logs.
|
||||
- A bootstrap is a bearer credential, not a source-address-bound token. The client address and IP family may change between bridge loading and session creation. The issuing address retains unused-bootstrap accounting, while the address on the first valid creation request owns the session.
|
||||
- Inner MTProxy authentication is restricted to the user and secret mode selected by the vhost profile. Invalid inner handshakes close only their logical stream and never enter the TCP masking path.
|
||||
@@ -99,6 +99,12 @@ All lane queues remain inside the existing per-session and process-wide byte/ite
|
||||
|
||||
The `/api/v1/up` and `/api/v1/down` paths do not change. In `https-lanes`, every request on those paths carries one canonical decimal `X-Lane-ID`. Uplink sequence starts at `1` and downlink cursor at `0` independently for each lane. Lane zero accepts only session `PONG`; every frame in a non-zero lane must have the same stream ID, and a new lane must begin with `OPEN`. After a closed lane's queued and unacknowledged downlink data is drained, Telemt returns an empty response with `X-Lane-Closed: 1`, and the bridge stops polling it. Retries remain byte-identical and replay the original acknowledgement or downlink batch.
|
||||
|
||||
Both WebSocket carriers still create and delete the parent session over HTTPS. They then use a strict bodyless `GET /api/v1/ws` Upgrade request. `websocket` offers exactly `tproxy-v1.<session-token>` in `Sec-WebSocket-Protocol`; binary messages are ordered carrier batches, and a protocol, deadline, or connection failure closes the complete parent session. `websocket-lanes` offers exactly `tproxy-lane-v1.<session-token>.<stream-id>`, where the stream ID is canonical decimal in `1..=16777215`. Its first binary message must begin with `OPEN`, every frame must use that stream ID, and failure after upgrade closes only that lane. There is no lane-zero WebSocket: HTTPS carries `HELLO` and `WELCOME`, while RFC 6455 Ping/Pong supplies connection liveness.
|
||||
|
||||
WebSocket codec buffers and in-flight read/write messages share the process-owned `pending_bytes_global` budget with carrier queues and are additionally bounded by `websocket_bytes_global`. Admission leaves `websocket_http_connection_reserve` accepted connections for ordinary HTTP and decoys. Under pressure, replacement is owner-first, then least-recently-progressed with pre-Upgrade and dead connections ahead of live lanes and multiplexed sessions. A transport Ping is sent after `long_poll_secs` without peer activity, including during continuous downlink traffic; missing peer activity for twice that creation-time interval makes a connection eligible for cleanup.
|
||||
|
||||
Every pre-Upgrade authentication, shape, lane-reservation, or capacity failure follows the sanitized decoy path instead of exposing a WebSocket-specific status. The exact subprotocol contains the session bearer and must not be logged.
|
||||
|
||||
The WEB listener must use `proxy_protocol = false` and `reuse_allow = false`. It cannot use `client_mss`, `synlimit`, `announce`, or `announce_ip`. `web_trusted_proxy_cidrs` must be non-empty and must contain only the immediate NGINX or HAProxy peers; `/0` networks are rejected.
|
||||
|
||||
The HTTP decoy origin must be a loopback, link-local, or private IP literal. Telemt preserves ordinary request method, path, query, headers, streamed body, response status, headers, and body while removing hop-by-hop headers. Malformed carrier requests have carrier credentials and bodies removed before falling back to the decoy.
|
||||
@@ -119,6 +125,11 @@ All WEB keys and defaults are listed in the [configuration reference](../Config_
|
||||
## NGINX TLS termination
|
||||
|
||||
```nginx
|
||||
map $http_upgrade $telemt_connection_upgrade {
|
||||
default upgrade;
|
||||
'' '';
|
||||
}
|
||||
|
||||
upstream telemt_web {
|
||||
server 127.0.0.1:18080;
|
||||
keepalive 64;
|
||||
@@ -140,11 +151,12 @@ server {
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $telemt_connection_upgrade;
|
||||
|
||||
proxy_connect_timeout 5s;
|
||||
proxy_send_timeout 35s;
|
||||
proxy_read_timeout 35s;
|
||||
proxy_send_timeout 65s;
|
||||
proxy_read_timeout 65s;
|
||||
proxy_request_buffering off;
|
||||
proxy_buffering off;
|
||||
proxy_next_upstream off;
|
||||
@@ -152,9 +164,9 @@ server {
|
||||
}
|
||||
```
|
||||
|
||||
`client_max_body_size` must be at least `web.limits.max_body_bytes`. `proxy_read_timeout` and `proxy_send_timeout` must exceed `web.timeouts.long_poll_secs`, which defaults to 25 seconds. Overwrite, rather than append to, `X-Forwarded-For`. Telemt accepts one parseable IP address; if a trusted terminator omits the header, Telemt falls back to the direct peer address, but per-client limits and source policy then see the terminator rather than the real client. Do not enable upstream retries: the bridge performs byte-identical retries through its own sequence protocol.
|
||||
Place the `map` in NGINX's `http` context. `client_max_body_size` must be at least `web.limits.max_body_bytes`. Read, send, and client timeouts must exceed both the 25-second default long poll and twice the configured WebSocket liveness interval; 65 seconds covers the defaults. Overwrite, rather than append to, `X-Forwarded-For`. Telemt accepts one parseable IP address; if a trusted terminator omits the header, Telemt falls back to the direct peer address, but per-client limits and source policy then see the terminator rather than the real client. Do not enable upstream retries: the bridge performs byte-identical HTTPS retries, while an established WebSocket is never transparently replayed.
|
||||
|
||||
Public HTTP/2 is mandatory for `https-lanes`; use the equivalent HTTP/2 directive supported by the installed NGINX release. The private NGINX-to-Telemt hop intentionally remains HTTP/1.1. Ensure the upstream connection capacity can sustain the expected simultaneous lane polls; `keepalive` controls the idle pool and is not a concurrency limit.
|
||||
Public HTTP/2 is mandatory for `https-lanes`; use the equivalent HTTP/2 directive supported by the installed NGINX release. WebSocket Upgrade requires HTTP/1.1, so the public endpoint must also permit HTTP/1.1 and the private NGINX-to-Telemt hop remains HTTP/1.1. Preserve `Connection`, `Upgrade`, and `Sec-WebSocket-*` exactly as shown. Ensure the upstream connection capacity can sustain the expected simultaneous lane polls or WebSocket lanes; `keepalive` controls the idle pool and is not a concurrency limit.
|
||||
|
||||
## HAProxy TLS termination
|
||||
|
||||
@@ -171,14 +183,14 @@ backend telemt_web
|
||||
option http-keep-alive
|
||||
retries 0
|
||||
timeout connect 5s
|
||||
timeout server 35s
|
||||
timeout server 65s
|
||||
http-request set-header Host proxy.example.com
|
||||
http-request del-header X-Forwarded-For
|
||||
http-request set-header X-Forwarded-For %[src]
|
||||
server telemt_web_1 127.0.0.1:18080 check
|
||||
```
|
||||
|
||||
The frontend or `defaults` section must also set `timeout client` above the long-poll deadline. HAProxy's public ALPN must include `h2` for `https-lanes`. Do not rewrite the path, raw query, body, or the `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor`, and `X-Lane-ID` carrier headers.
|
||||
The frontend or `defaults` section must also set `timeout client 65s` or longer for the default WebSocket liveness interval. HAProxy's public ALPN must include `h2` for `https-lanes` and `http/1.1` for WebSocket Upgrade. Preserve `Connection`, `Upgrade`, and `Sec-WebSocket-*`; do not rewrite the path, raw query, body, or the `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor`, and `X-Lane-ID` carrier headers.
|
||||
|
||||
## Lifecycle and reload behavior
|
||||
|
||||
@@ -186,21 +198,22 @@ The frontend or `defaults` section must also set `timeout client` above the long
|
||||
| --- | --- |
|
||||
| WEB listener inventory, bind address, and trust policy | Process-owned; restart Telemt. |
|
||||
| Any `[web.limits]` value | Process-owned memory/resource contract; restart Telemt. |
|
||||
| `web.enabled`, `web.carrier`, timeouts, vhosts, profiles, and decoys | Applied by the config watcher or a runtime generation reload. |
|
||||
| Existing HTTP connections and WEB sessions | Keep their acquisition-time carrier, limits, and deadlines; newly issued bridge sessions use the active carrier. New logical streams use the active relay generation. |
|
||||
| `web.enabled`, `web.carrier`, `web.debug`, timeouts, vhosts, profiles, and decoys | Applied by the config watcher or a runtime generation reload. |
|
||||
| Existing HTTP connections and WEB sessions | Keep their acquisition-time carrier, limits, and session deadlines; newly issued bridge sessions use the active carrier. WebSocket write, backpressure, and eviction operations read the active hot-reloaded deadlines. New logical streams use the active relay generation. |
|
||||
| Process shutdown | Uses the latest reloaded `web.timeouts.shutdown_secs`. |
|
||||
|
||||
Each logical stream keeps its session's creation-time client IP and owns a process-unique, non-zero synthetic source port for the complete relay lifetime. This preserves one stable, non-colliding source/destination tuple for Direct and Middle-End KDF routing.
|
||||
|
||||
## API management
|
||||
|
||||
API management is available, but it is intentionally partial. There is no dedicated `/v1/web` endpoint and no WEB-specific runtime statistics endpoint.
|
||||
API management is available, but it is intentionally partial. There is no mutable `/v1/web` resource; the API listener exposes the read-only HTML debug view at `/web-status`.
|
||||
|
||||
| Operation | API support |
|
||||
| --- | --- |
|
||||
| Read or patch `[web]`, vhosts, profiles, decoys, timeouts, or limits | No. `GET /v1/config` omits `[web]`; `PATCH /v1/config` returns `400 section_not_editable` for `web`. |
|
||||
| Persist `server.listeners` | Yes, through `PATCH /v1/config`, but a changed WEB listener remains deferred until process restart. |
|
||||
| Apply an externally edited WEB configuration | Yes, through `POST /v1/system/reload`, then inspect the operation status. |
|
||||
| Inspect bounded server-side WEB request and lifecycle details | Yes, through authenticated `GET /web-status`. |
|
||||
| Manage `[access.users]` | Yes, through `/v1/users`. User creation does not create a WEB profile. |
|
||||
| Revoke one user | Yes. `/v1/users/{username}/disable` updates admission immediately and cancels that user's active sessions. |
|
||||
|
||||
@@ -217,6 +230,30 @@ read_only = false
|
||||
|
||||
The API whitelist checks the direct TCP peer and does not trust `X-Forwarded-For`. Changes to `[server.api]` itself require a process restart.
|
||||
|
||||
### Server-side WEB debug view
|
||||
|
||||
Enable bounded collection in the owned configuration file:
|
||||
|
||||
```toml
|
||||
[web.debug]
|
||||
enabled = true
|
||||
capture_lifecycle = true
|
||||
capture_headers = true
|
||||
capture_timings = true
|
||||
capture_frames = true
|
||||
body_capture = "metadata"
|
||||
body_prefix_bytes = 4096
|
||||
decoy_body_prefix_bytes = 4096
|
||||
default_window_secs = 180
|
||||
max_window_secs = 3600
|
||||
```
|
||||
|
||||
Open `http://127.0.0.1:9091/web-status` with the same direct-peer whitelist and exact `Authorization` header used by the API. A trailing slash is accepted. Only `GET` is allowed. The page supports `window_secs`, canonical `ip`, numeric `session`, case-insensitive `user_agent`, and `key` filters. Repeat `group_by=ip`, `group_by=session`, `group_by=user_agent`, or `group_by=key` to build grouped summaries; `limit` is restricted to `1..=1000`. HTTP rows expand from request through response with method, path, sanitized headers, body metadata or bytes, timing points, parsed frames, and typed lifecycle events. WebSocket operation adds the sanitized `GET` to `101` handshake plus bounded per-message direction, message type, payload/body capture, processing time, connection/lane identifiers, and parsed inner frames. Raw subprotocols and session tokens are never retained.
|
||||
|
||||
The process-owned ring survives runtime generation replacement. Capture-policy changes clear incompatible retained records; window-only changes do not. The ring defaults to 65536 records and 64 MiB retained plus in-flight bytes, the HTML response is capped at 8 MiB, grouping is capped at 1024 groups, and no more than two response bodies retain page permits concurrently. Change `web.limits.debug_records_capacity` or `web.limits.debug_bytes_global` only with a process restart. A hot prefix that fits only a simultaneously increased restart-only capacity is deferred until that restart.
|
||||
|
||||
`body_capture = "off"` omits bodies, `metadata` retains lengths and terminal states, `prefix` retains configured prefixes, and `full` retains recognized carrier bodies up to `web.limits.max_body_bytes`. Ordinary decoy bodies remain limited by `decoy_body_prefix_bytes` even in `full` mode. Queries and raw capabilities are never stored; credential header values are omitted; known WEB capabilities and bearer tokens are scrubbed from captured bodies; the displayed key is a non-secret domain-separated fingerprint. Timing ends at Hyper body polling and does not claim kernel flush or TCP acknowledgment.
|
||||
|
||||
After an administrator or configuration system atomically updates the TOML file, set `TELEMT_API_AUTH` to the exact value configured in `auth_header` and submit an observable generation reload:
|
||||
|
||||
```bash
|
||||
@@ -264,8 +301,9 @@ See the complete [Control API contract](../Architecture/API/API.md) for request
|
||||
3. Confirm that Telemt receives one parseable `X-Forwarded-For` address and `Host: proxy.example.com` or `Host: proxy.example.com:443`.
|
||||
4. Import the printed `tg://webproxy` link in the intended Telegram Desktop build and establish a proxy connection.
|
||||
5. For `https-lanes`, confirm that the public connection negotiated HTTP/2 and exercise at least two simultaneous logical streams; the private Telemt hop remains HTTP/1.1.
|
||||
6. Exercise reconnect and at least one long poll beyond 25 seconds to prove the frontend timeouts do not truncate the carrier.
|
||||
7. Verify user and logical MTProxy connection limits using logical-stream counters, not the number of HTTP connections.
|
||||
6. For `websocket`, confirm one `101` response, binary relay traffic, and RFC 6455 Ping/Pong beyond 25 seconds. For `websocket-lanes`, exercise at least two simultaneous stream sockets and verify that closing or corrupting one lane does not close its sibling or parent session.
|
||||
7. Exercise reconnect and at least one long poll beyond 25 seconds to prove the frontend timeouts do not truncate the carrier.
|
||||
8. Verify user and logical MTProxy connection limits using logical-stream counters, not the number of HTTP connections.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
@@ -274,6 +312,9 @@ See the complete [Control API contract](../Architecture/API/API.md) for request
|
||||
| WEB configuration is valid on disk but listener behavior did not change | Inspect reload `deferred_process_fields`; listener and `[web.limits]` changes require restart. |
|
||||
| Carrier requests reach the decoy | Verify exact vhost, link secret mode, direct proxy CIDR, and one parseable `X-Forwarded-For` value. |
|
||||
| Long polls disconnect near a fixed interval | Raise NGINX/HAProxy client, server, send, and read timeouts above `web.timeouts.long_poll_secs`. |
|
||||
| WebSocket Upgrade reaches the decoy instead of returning `101` | Preserve HTTP/1.1 `Connection: Upgrade`, `Upgrade: websocket`, the single exact `Sec-WebSocket-Protocol`, and the canonical bodyless `/api/v1/ws` request. Also check carrier/session compatibility and the process connection reserve. |
|
||||
| One `websocket-lanes` stream closes while siblings stay connected | This is the intended failure boundary. Inspect that lane's message/frame rows in `/web-status`; malformed, cross-lane, write-timeout, and backend-close paths terminate only the affected lane. |
|
||||
| `/web-status` is empty | Confirm `[web.debug].enabled = true`, apply the configuration, select a window within `max_window_secs`, and generate new WEB traffic after the policy change. |
|
||||
| `https-lanes` works but streams still block each other | Confirm public HTTP/2 negotiation, preserve `X-Lane-ID`, and provide enough TLS-terminator upstream connections for concurrent private HTTP/1.1 polls. |
|
||||
| Telegram Desktop rejects the link | Omit the port, use a valid FQDN, port 443 externally, and only `plain` or `dd` secret mode. |
|
||||
| One node works but a load-balanced pool is intermittent | Add complete-vhost affinity; WEB credential registries are process-local. |
|
||||
|
||||
+56
-15
@@ -2,7 +2,7 @@
|
||||
|
||||
[English](WEB_PROXY.en.md) | [Русский](WEB_PROXY.ru.md) | [Deutsch](WEB_PROXY.de.md)
|
||||
|
||||
WEB-режим переносит обычные MTProxy-потоки через bounded HTTPS carriers, совместимые с типом прокси `WEB` в Telegram Desktop. Telemt не терминирует TLS: публичный сертификат обслуживает NGINX или HAProxy, который передаёт обычный HTTP/1.1 на приватный listener Telemt.
|
||||
WEB-режим переносит обычные MTProxy-потоки через bounded HTTPS или WebSocket carriers, совместимые с типом прокси `WEB` в Telegram Desktop. Telemt не терминирует TLS: публичный сертификат обслуживает NGINX или HAProxy, который передаёт обычный HTTP/1.1 на приватный listener Telemt.
|
||||
|
||||
> [!IMPORTANT]
|
||||
>
|
||||
@@ -12,7 +12,7 @@ WEB-режим переносит обычные MTProxy-потоки через
|
||||
|
||||
```text
|
||||
Telegram Desktop
|
||||
| HTTPS :443
|
||||
| HTTPS или WSS :443
|
||||
v
|
||||
NGINX или HAProxy (TLS termination, канонический Host и один адрес X-Forwarded-For)
|
||||
| обычный HTTP/1.1 в приватной сети
|
||||
@@ -28,7 +28,7 @@ WEB-listener Telemt
|
||||
|
||||
- Публичный endpoint всегда имеет вид `https://HOST:443`.
|
||||
- Поддерживаются 16-байтовые MTProxy-секреты `plain` и `dd`. FakeTLS-секреты `ee` в WEB-режиме не поддерживаются.
|
||||
- `web.carrier = "https"` выбирает сериализованные HTTPS uplink и long polling. `web.carrier = "https-lanes"` выбирает независимые HTTPS sequencing и polling для каждого logical stream. WebSocket carriers не анонсируются.
|
||||
- `web.carrier = "https"` выбирает сериализованные HTTPS uplink и long polling. `https-lanes` выбирает независимые HTTPS sequencing и polling для каждого logical stream. `websocket` выбирает один упорядоченный WebSocket для всех streams. `websocket-lanes` выбирает отдельный WebSocket с независимым ownership для каждого ненулевого logical stream.
|
||||
- Capability, bootstrap и session credentials — отдельные значения с ограниченным сроком жизни. Carrier credentials считаются секретами и не должны попадать в access logs.
|
||||
- Bootstrap является bearer credential, а не token с привязкой к source address. Адрес клиента и его IP-семейство могут измениться между загрузкой bridge и созданием session. Адрес выдачи продолжает учитываться в лимите неиспользованных bootstrap, а владельцем session становится адрес первого корректного запроса создания.
|
||||
- Внутренняя MTProxy-аутентификация ограничена пользователем и режимом секрета, выбранными профилем vhost. Некорректный внутренний handshake закрывает только свой logical stream и никогда не попадает в TCP masking path.
|
||||
@@ -99,6 +99,12 @@ max_streams_per_session = 64
|
||||
|
||||
Paths `/api/v1/up` и `/api/v1/down` не меняются. В `https-lanes` каждый запрос к ним содержит один канонический десятичный `X-Lane-ID`. Uplink sequence начинается с `1`, а downlink cursor — с `0` независимо для каждой lane. Lane zero принимает только session `PONG`; все frames ненулевой lane должны иметь тот же stream ID, а новая lane должна начинаться с `OPEN`. После отправки всей queued и unacknowledged downlink data закрытой lane Telemt возвращает пустой ответ с `X-Lane-Closed: 1`, и bridge прекращает её polling. Retry остаются byte-identical и повторяют исходный acknowledgement или downlink batch.
|
||||
|
||||
Оба WebSocket carrier по-прежнему создают и удаляют parent session через HTTPS, после чего используют строгий bodyless Upgrade-запрос `GET /api/v1/ws`. `websocket` передаёт в `Sec-WebSocket-Protocol` ровно `tproxy-v1.<session-token>`; binary messages являются упорядоченными carrier batches, а ошибка протокола, deadline или connection закрывает всю parent session. `websocket-lanes` передаёт ровно `tproxy-lane-v1.<session-token>.<stream-id>`, где stream ID записан каноническим десятичным числом из диапазона `1..=16777215`. Первое binary message должно начинаться с `OPEN`, все frames должны содержать этот stream ID, а сбой после Upgrade закрывает только данную lane. Lane-zero WebSocket отсутствует: HTTPS переносит `HELLO` и `WELCOME`, а liveness connection обеспечивает RFC 6455 Ping/Pong.
|
||||
|
||||
WebSocket codec buffers и находящиеся в обработке read/write messages делят process-owned `pending_bytes_global` с carrier queues и дополнительно ограничены `websocket_bytes_global`. Admission оставляет `websocket_http_connection_reserve` принятых connections для обычного HTTP и decoy. При pressure вытеснение сначала выбирает того же owner, затем connection с наиболее старым прогрессом; pre-Upgrade и dead connections идут раньше активных lanes и multiplexed sessions. После `long_poll_secs` без peer activity отправляется transport Ping, в том числе при непрерывном downlink traffic, а отсутствие peer activity в течение удвоенного creation-time интервала делает connection кандидатом на cleanup.
|
||||
|
||||
Любая ошибка authentication, shape, lane reservation или capacity до Upgrade следует по очищенному decoy path и не раскрывает WebSocket-специфичный status. Точный subprotocol содержит session bearer и не должен попадать в logs.
|
||||
|
||||
Для WEB-listener обязательны `proxy_protocol = false` и `reuse_allow = false`. В нём нельзя использовать `client_mss`, `synlimit`, `announce` и `announce_ip`. Массив `web_trusted_proxy_cidrs` должен быть непустым и содержать только непосредственные адреса NGINX или HAProxy; сети `/0` запрещены.
|
||||
|
||||
HTTP decoy origin должен быть loopback, link-local или private IP literal. Для обычных запросов Telemt сохраняет method, path, query, headers, streamed body, response status, headers и body, удаляя hop-by-hop headers. Перед отправкой некорректного carrier-запроса в decoy Telemt удаляет из него carrier credentials и body.
|
||||
@@ -119,6 +125,11 @@ index = "index.html"
|
||||
## Терминация TLS на NGINX
|
||||
|
||||
```nginx
|
||||
map $http_upgrade $telemt_connection_upgrade {
|
||||
default upgrade;
|
||||
'' '';
|
||||
}
|
||||
|
||||
upstream telemt_web {
|
||||
server 127.0.0.1:18080;
|
||||
keepalive 64;
|
||||
@@ -140,11 +151,12 @@ server {
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $telemt_connection_upgrade;
|
||||
|
||||
proxy_connect_timeout 5s;
|
||||
proxy_send_timeout 35s;
|
||||
proxy_read_timeout 35s;
|
||||
proxy_send_timeout 65s;
|
||||
proxy_read_timeout 65s;
|
||||
proxy_request_buffering off;
|
||||
proxy_buffering off;
|
||||
proxy_next_upstream off;
|
||||
@@ -152,9 +164,9 @@ server {
|
||||
}
|
||||
```
|
||||
|
||||
`client_max_body_size` должен быть не меньше `web.limits.max_body_bytes`. Значения `proxy_read_timeout` и `proxy_send_timeout` должны превышать `web.timeouts.long_poll_secs`, по умолчанию равный 25 секундам. Перезаписывайте `X-Forwarded-For`, а не дополняйте его. Telemt принимает один корректно разбираемый IP-адрес; если доверенный TLS-терминатор не передал header, Telemt использует адрес непосредственного peer, но per-client limits и source policy тогда видят терминатор вместо реального клиента. Не включайте upstream retries: byte-identical retry выполняет сам bridge по своему sequence protocol.
|
||||
Разместите `map` в контексте `http` NGINX. `client_max_body_size` должен быть не меньше `web.limits.max_body_bytes`. Read, send и client timeouts должны превышать как default long poll в 25 секунд, так и удвоенный WebSocket liveness interval; 65 секунд покрывают defaults. Перезаписывайте `X-Forwarded-For`, а не дополняйте его. Telemt принимает один корректно разбираемый IP-адрес; если доверенный TLS-терминатор не передал header, Telemt использует адрес непосредственного peer, но per-client limits и source policy тогда видят терминатор вместо реального клиента. Не включайте upstream retries: bridge выполняет byte-identical HTTPS retries, но установленный WebSocket никогда не replay’ится прозрачно.
|
||||
|
||||
Для `https-lanes` обязателен публичный HTTP/2; используйте эквивалентную HTTP/2-директиву, поддерживаемую установленной версией NGINX. Приватный hop NGINX-to-Telemt намеренно остаётся HTTP/1.1. Upstream connection capacity должна выдерживать ожидаемое число одновременных lane polls; `keepalive` управляет idle pool и не является лимитом concurrency.
|
||||
Для `https-lanes` обязателен публичный HTTP/2; используйте эквивалентную HTTP/2-директиву, поддерживаемую установленной версией NGINX. WebSocket Upgrade требует HTTP/1.1, поэтому публичный endpoint должен также разрешать HTTP/1.1, а приватный hop NGINX-to-Telemt остаётся HTTP/1.1. Сохраняйте `Connection`, `Upgrade` и `Sec-WebSocket-*` ровно как в примере. Upstream connection capacity должна выдерживать ожидаемое число одновременных lane polls или WebSocket lanes; `keepalive` управляет idle pool и не является лимитом concurrency.
|
||||
|
||||
## Терминация TLS на HAProxy
|
||||
|
||||
@@ -171,14 +183,14 @@ backend telemt_web
|
||||
option http-keep-alive
|
||||
retries 0
|
||||
timeout connect 5s
|
||||
timeout server 35s
|
||||
timeout server 65s
|
||||
http-request set-header Host proxy.example.com
|
||||
http-request del-header X-Forwarded-For
|
||||
http-request set-header X-Forwarded-For %[src]
|
||||
server telemt_web_1 127.0.0.1:18080 check
|
||||
```
|
||||
|
||||
Во frontend или секции `defaults` также задайте `timeout client` выше long-poll deadline. Для `https-lanes` публичный ALPN HAProxy должен содержать `h2`. Не переписывайте path, raw query, body и carrier headers `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor`, `X-Lane-ID`.
|
||||
Во frontend или секции `defaults` также задайте `timeout client 65s` или больше для default WebSocket liveness interval. Для `https-lanes` публичный ALPN HAProxy должен содержать `h2`, а для WebSocket Upgrade — `http/1.1`. Сохраняйте `Connection`, `Upgrade` и `Sec-WebSocket-*`; не переписывайте path, raw query, body и carrier headers `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor`, `X-Lane-ID`.
|
||||
|
||||
## Lifecycle и reload
|
||||
|
||||
@@ -186,21 +198,22 @@ backend telemt_web
|
||||
| --- | --- |
|
||||
| Состав WEB-listeners, bind address и trust policy | Принадлежат процессу; перезапустите Telemt. |
|
||||
| Любое значение `[web.limits]` | Process-owned контракт памяти и ресурсов; перезапустите Telemt. |
|
||||
| `web.enabled`, `web.carrier`, timeouts, vhosts, profiles и decoys | Применяются config watcher или runtime generation reload. |
|
||||
| Существующие HTTP connections и WEB sessions | Сохраняют carrier, лимиты и deadlines своего момента создания; новые bridge sessions получают активный carrier. Новые logical streams используют активное relay generation. |
|
||||
| `web.enabled`, `web.carrier`, `web.debug`, timeouts, vhosts, profiles и decoys | Применяются config watcher или runtime generation reload. |
|
||||
| Существующие HTTP connections и WEB sessions | Сохраняют carrier, лимиты и session deadlines своего момента создания; новые bridge sessions получают активный carrier. WebSocket write, backpressure и eviction operations читают активные hot-reloaded deadlines. Новые logical streams используют активное relay generation. |
|
||||
| Завершение процесса | Использует последнее применённое значение `web.timeouts.shutdown_secs`. |
|
||||
|
||||
Каждый logical stream сохраняет client IP своей сессии и владеет уникальным в пределах процесса ненулевым synthetic source port до завершения relay. Это сохраняет один стабильный непересекающийся source/destination tuple для Direct и Middle-End KDF routing.
|
||||
|
||||
## Управление через API
|
||||
|
||||
Управление через API доступно, но намеренно ограничено. Отдельных endpoint `/v1/web` и WEB-specific runtime statistics endpoint сейчас нет.
|
||||
Управление через API доступно, но намеренно ограничено. Изменяемого ресурса `/v1/web` нет; API-listener предоставляет read-only HTML debug view по адресу `/web-status`.
|
||||
|
||||
| Операция | Поддержка API |
|
||||
| --- | --- |
|
||||
| Чтение или изменение `[web]`, vhosts, profiles, decoys, timeouts или limits | Нет. `GET /v1/config` не возвращает `[web]`; `PATCH /v1/config` отвечает `400 section_not_editable` на ключ `web`. |
|
||||
| Сохранение `server.listeners` | Да, через `PATCH /v1/config`, но изменённый WEB-listener остаётся deferred до перезапуска процесса. |
|
||||
| Применение WEB-конфигурации, изменённой вне API | Да, через `POST /v1/system/reload` с последующей проверкой статуса операции. |
|
||||
| Просмотр bounded серверных WEB request- и lifecycle-деталей | Да, через аутентифицированный `GET /web-status`. |
|
||||
| Управление `[access.users]` | Да, через `/v1/users`. Создание пользователя не создаёт WEB-профиль. |
|
||||
| Отзыв отдельного пользователя | Да. `/v1/users/{username}/disable` немедленно обновляет admission и завершает активные сессии пользователя. |
|
||||
|
||||
@@ -217,6 +230,30 @@ read_only = false
|
||||
|
||||
API whitelist проверяет непосредственный TCP peer и не доверяет `X-Forwarded-For`. Изменения самой секции `[server.api]` требуют перезапуска процесса.
|
||||
|
||||
### Серверная WEB-отладка
|
||||
|
||||
Включите bounded сбор в конфигурационном файле, которому принадлежит эта секция:
|
||||
|
||||
```toml
|
||||
[web.debug]
|
||||
enabled = true
|
||||
capture_lifecycle = true
|
||||
capture_headers = true
|
||||
capture_timings = true
|
||||
capture_frames = true
|
||||
body_capture = "metadata"
|
||||
body_prefix_bytes = 4096
|
||||
decoy_body_prefix_bytes = 4096
|
||||
default_window_secs = 180
|
||||
max_window_secs = 3600
|
||||
```
|
||||
|
||||
Откройте `http://127.0.0.1:9091/web-status`, используя те же whitelist непосредственных peers и точный header `Authorization`, что и для API. Завершающий slash разрешён. Допускается только `GET`. Страница поддерживает фильтры `window_secs`, канонический `ip`, числовой `session`, регистронезависимый `user_agent` и `key`. Повторяйте `group_by=ip`, `group_by=session`, `group_by=user_agent` или `group_by=key` для построения сгруппированных сводок; `limit` ограничен диапазоном `1..=1000`. HTTP rows раскрываются от request до response с method, path, очищенными headers, метаданными или байтами body, timing points, frames и типизированными lifecycle events. Для WebSocket добавляются очищенный handshake `GET` → `101` и bounded per-message direction, message type, payload/body capture, processing time, connection/lane identifiers и разобранные inner frames. Raw subprotocol и session tokens никогда не сохраняются.
|
||||
|
||||
Process-owned кольцевой буфер переживает замену runtime generation. Изменения capture policy очищают несовместимые сохранённые записи; изменения только окна наблюдения этого не делают. По умолчанию кольцо ограничено 65536 записями и 64 MiB сохранённых плюс находящихся в обработке данных, HTML-response — 8 MiB, grouping — 1024 группами; одновременно page permits могут удерживать не более двух response bodies. Изменяйте `web.limits.debug_records_capacity` или `web.limits.debug_bytes_global` только с перезапуском процесса. Hot prefix, который помещается только в одновременно увеличенную restart-only ёмкость, откладывается до этого перезапуска.
|
||||
|
||||
`body_capture = "off"` исключает bodies, `metadata` сохраняет длину и terminal state, `prefix` — настроенные prefixes, а `full` — распознанные carrier bodies до `web.limits.max_body_bytes`. Обычные decoy bodies даже в режиме `full` ограничены `decoy_body_prefix_bytes`. Queries и raw capabilities никогда не сохраняются; значения credential headers исключаются; известные WEB capabilities и bearer tokens удаляются из захваченных bodies; отображаемый ключ является несекретным domain-separated fingerprint. Timing заканчивается на polling Hyper body и не означает kernel flush или TCP acknowledgment.
|
||||
|
||||
После атомарного изменения TOML-файла администратором или системой управления конфигурацией задайте в `TELEMT_API_AUTH` точное значение `auth_header` и отправьте наблюдаемый generation reload:
|
||||
|
||||
```bash
|
||||
@@ -264,8 +301,9 @@ curl -sS -X POST http://127.0.0.1:9091/v1/users/web-user/rotate-secret \
|
||||
3. Убедитесь, что Telemt получает один корректно разбираемый адрес `X-Forwarded-For` и `Host: proxy.example.com` либо `Host: proxy.example.com:443`.
|
||||
4. Импортируйте напечатанную ссылку `tg://webproxy` в целевую сборку Telegram Desktop и установите соединение через прокси.
|
||||
5. Для `https-lanes` подтвердите согласование HTTP/2 на публичном connection и проверьте как минимум два одновременных logical streams; приватный hop к Telemt остаётся HTTP/1.1.
|
||||
6. Проверьте reconnect и как минимум один long poll длительнее 25 секунд, чтобы frontend timeouts не обрывали carrier.
|
||||
7. Проверяйте лимиты пользователя и logical MTProxy connections по logical-stream counters, а не по числу HTTP connections.
|
||||
6. Для `websocket` подтвердите один response `101`, binary relay traffic и RFC 6455 Ping/Pong после 25 секунд. Для `websocket-lanes` проверьте как минимум два одновременных stream sockets и убедитесь, что закрытие или повреждение одной lane не закрывает sibling или parent session.
|
||||
7. Проверьте reconnect и как минимум один long poll длительнее 25 секунд, чтобы frontend timeouts не обрывали carrier.
|
||||
8. Проверяйте лимиты пользователя и logical MTProxy connections по logical-stream counters, а не по числу HTTP connections.
|
||||
|
||||
## Диагностика
|
||||
|
||||
@@ -274,6 +312,9 @@ curl -sS -X POST http://127.0.0.1:9091/v1/users/web-user/rotate-secret \
|
||||
| WEB-конфигурация валидна на диске, но поведение listener’а не изменилось | Проверьте `deferred_process_fields`; listener и `[web.limits]` требуют перезапуска. |
|
||||
| Carrier-запросы попадают в decoy | Проверьте точный vhost, secret mode ссылки, CIDR непосредственного proxy и единственное корректно разбираемое значение `X-Forwarded-For`. |
|
||||
| Long polls разрываются через фиксированный интервал | Поднимите client, server, send и read timeouts NGINX/HAProxy выше `web.timeouts.long_poll_secs`. |
|
||||
| WebSocket Upgrade попадает в decoy вместо `101` | Сохраните HTTP/1.1 `Connection: Upgrade`, `Upgrade: websocket`, единственный точный `Sec-WebSocket-Protocol` и канонический bodyless request `/api/v1/ws`. Также проверьте соответствие carrier/session и process connection reserve. |
|
||||
| Один stream `websocket-lanes` закрылся, а siblings остались подключены | Это штатная failure boundary. Проверьте message/frame rows этой lane в `/web-status`; malformed, cross-lane, write-timeout и backend-close закрывают только затронутую lane. |
|
||||
| `/web-status` пуст | Убедитесь, что `[web.debug].enabled = true`, примените конфигурацию, выберите окно в пределах `max_window_secs` и создайте новый WEB-трафик после изменения policy. |
|
||||
| `https-lanes` работает, но streams всё ещё блокируют друг друга | Проверьте согласование публичного HTTP/2, сохранение `X-Lane-ID` и достаточное число upstream connections TLS-терминатора для параллельных приватных HTTP/1.1 polls. |
|
||||
| Telegram Desktop отклоняет ссылку | Не указывайте порт, используйте валидный FQDN, внешний порт 443 и только `plain` или `dd`. |
|
||||
| Один узел работает, но load-balanced pool нестабилен | Настройте affinity всего vhost: WEB credential registries локальны для процесса. |
|
||||
|
||||
+12
-2
@@ -30,6 +30,7 @@ use crate::startup::StartupTracker;
|
||||
use crate::stats::Stats;
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
use crate::web::trace::WebTraceStore;
|
||||
|
||||
mod config_edit;
|
||||
pub(crate) mod config_store;
|
||||
@@ -47,6 +48,7 @@ mod runtime_stats;
|
||||
mod runtime_watch;
|
||||
mod runtime_zero;
|
||||
mod users;
|
||||
mod web_status;
|
||||
|
||||
use config_store::{
|
||||
current_revision, ensure_expected_revision, load_config_for_reload, load_config_from_disk,
|
||||
@@ -122,6 +124,7 @@ pub(super) struct ApiShared {
|
||||
pub(super) proxy_shared: Arc<ProxySharedState>,
|
||||
pub(super) reload_control: ReloadControl,
|
||||
pub(super) active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
pub(super) web_trace: Arc<WebTraceStore>,
|
||||
}
|
||||
|
||||
impl ApiShared {
|
||||
@@ -155,6 +158,7 @@ impl ApiShared {
|
||||
proxy_shared: runtime.proxy_shared.clone(),
|
||||
reload_control: self.reload_control.clone(),
|
||||
active_runtime: self.active_runtime.clone(),
|
||||
web_trace: self.web_trace.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -243,7 +247,8 @@ fn allowed_methods_for_path(path: &str) -> Option<&'static str> {
|
||||
| "/v1/runtime/tls-fingerprints"
|
||||
| "/v1/stats/users/active-ips"
|
||||
| "/v1/stats/users/quota"
|
||||
| "/v1/stats/users" => Some(ALLOW_GET),
|
||||
| "/v1/stats/users"
|
||||
| "/web-status" => Some(ALLOW_GET),
|
||||
"/v1/system/reload" => Some(ALLOW_POST),
|
||||
"/v1/users" => Some(ALLOW_GET_POST),
|
||||
"/v1/config" => Some(ALLOW_GET_PATCH),
|
||||
@@ -279,6 +284,7 @@ pub async fn serve(
|
||||
reload_control: ReloadControl,
|
||||
mut active_runtime_rx: watch::Receiver<Option<Arc<ArcSwap<RuntimeGeneration>>>>,
|
||||
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||
web_trace: Arc<WebTraceStore>,
|
||||
) {
|
||||
let active_runtime = loop {
|
||||
if let Some(active_runtime) = active_runtime_rx.borrow().clone() {
|
||||
@@ -312,7 +318,7 @@ pub async fn serve(
|
||||
}
|
||||
};
|
||||
|
||||
info!("API endpoint: http://{}/v1/*", listen);
|
||||
info!("API endpoint: http://{}/v1/* and /web-status", listen);
|
||||
|
||||
let runtime_state = Arc::new(ApiRuntimeState {
|
||||
process_started_at_epoch_secs,
|
||||
@@ -344,6 +350,7 @@ pub async fn serve(
|
||||
proxy_shared,
|
||||
reload_control,
|
||||
active_runtime,
|
||||
web_trace,
|
||||
});
|
||||
|
||||
spawn_runtime_watchers(
|
||||
@@ -492,6 +499,9 @@ async fn handle(
|
||||
|
||||
let result: Result<Response<Full<Bytes>>, ApiFailure> = async {
|
||||
match (method.as_str(), normalized_path) {
|
||||
("GET", "/web-status") => {
|
||||
Ok(web_status::render(query.as_deref(), &shared.web_trace, &cfg.web.debug).await)
|
||||
}
|
||||
("GET", "/v1/health") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = HealthData {
|
||||
|
||||
@@ -112,6 +112,7 @@ fn reload_routes_expose_only_documented_methods_and_ids() {
|
||||
Some(ALLOW_GET)
|
||||
);
|
||||
assert_eq!(reload_status_route_id("/v1/system/reload/42"), Some(42));
|
||||
assert_eq!(allowed_methods_for_path("/web-status"), Some(ALLOW_GET));
|
||||
assert_eq!(
|
||||
reload_status_route_id("/v1/system/reload/not-a-number"),
|
||||
None
|
||||
|
||||
@@ -0,0 +1,521 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use http_body_util::Full;
|
||||
use hyper::body::Bytes;
|
||||
use hyper::header::{self, HeaderValue};
|
||||
use hyper::{Response, StatusCode};
|
||||
use tokio::sync::OwnedSemaphorePermit;
|
||||
|
||||
use crate::config::WebDebugConfig;
|
||||
use crate::web::trace::{StoredTraceRecord, TraceRecord, TraceRecordKind, WebTraceStore};
|
||||
|
||||
const MAX_PAGE_BYTES: usize = 8 * 1024 * 1024;
|
||||
const MAX_GROUPS: usize = 1024;
|
||||
|
||||
// Record-detail rendering remains isolated from filtering and page layout.
|
||||
mod details;
|
||||
// Query parsing and matching remain independent from bounded HTML rendering.
|
||||
mod query;
|
||||
|
||||
use details::{push_body, push_frames, push_headers};
|
||||
use query::{GroupBy, StatusQuery, client_ip, parse_query, record_matches};
|
||||
|
||||
struct GroupSummary {
|
||||
count: usize,
|
||||
latest_seq: u64,
|
||||
}
|
||||
|
||||
struct RenderedPage {
|
||||
html: String,
|
||||
_permit: OwnedSemaphorePermit,
|
||||
}
|
||||
|
||||
impl AsRef<[u8]> for RenderedPage {
|
||||
fn as_ref(&self) -> &[u8] {
|
||||
self.html.as_bytes()
|
||||
}
|
||||
}
|
||||
|
||||
/// Renders the authenticated server-side WEB debugging table.
|
||||
pub(super) async fn render(
|
||||
raw_query: Option<&str>,
|
||||
store: &Arc<WebTraceStore>,
|
||||
policy: &WebDebugConfig,
|
||||
) -> Response<Full<Bytes>> {
|
||||
store.apply_policy(policy);
|
||||
let query = match parse_query(raw_query, policy) {
|
||||
Ok(query) => query,
|
||||
Err(error) => return html_error(StatusCode::BAD_REQUEST, "Invalid query", &error),
|
||||
};
|
||||
let Some(render_permit) = store.try_render_permit() else {
|
||||
return html_error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"Renderer busy",
|
||||
"Two WEB status pages are already rendering",
|
||||
);
|
||||
};
|
||||
let now_millis = crate::web::trace::store_epoch_millis();
|
||||
let since_millis = query
|
||||
.record
|
||||
.is_none()
|
||||
.then(|| now_millis.saturating_sub(query.window_secs.saturating_mul(1000)))
|
||||
.unwrap_or(0);
|
||||
let records = store.snapshot_matching(|record| record_matches(record, &query, since_millis));
|
||||
let status = store.status();
|
||||
let mut html = String::with_capacity(MAX_PAGE_BYTES);
|
||||
push_page_start(&mut html);
|
||||
html.push_str("<h1>WEB status</h1>");
|
||||
push_filter_form(&mut html, &query);
|
||||
html.push_str("<section><h2>Store</h2><table><tbody>");
|
||||
summary_row(&mut html, "debug enabled", yes_no(status.policy.enabled));
|
||||
summary_row(&mut html, "body capture", body_mode(&status.policy));
|
||||
summary_row(&mut html, "window seconds", &query.window_secs.to_string());
|
||||
summary_row(
|
||||
&mut html,
|
||||
"records",
|
||||
&format!("{} / {}", status.records, status.records_capacity),
|
||||
);
|
||||
summary_row(
|
||||
&mut html,
|
||||
"bytes",
|
||||
&format!("{} / {}", status.used_bytes, status.bytes_capacity),
|
||||
);
|
||||
summary_row(&mut html, "matched", &records.len().to_string());
|
||||
summary_row(
|
||||
&mut html,
|
||||
"contention drops",
|
||||
&status.contention_drops.to_string(),
|
||||
);
|
||||
summary_row(&mut html, "evictions", &status.evictions.to_string());
|
||||
summary_row(
|
||||
&mut html,
|
||||
"byte truncations",
|
||||
&status.byte_truncations.to_string(),
|
||||
);
|
||||
summary_row(
|
||||
&mut html,
|
||||
"sequence range",
|
||||
&format!(
|
||||
"{} .. {}",
|
||||
option_u64(status.earliest_seq),
|
||||
option_u64(status.latest_seq)
|
||||
),
|
||||
);
|
||||
html.push_str("</tbody></table></section>");
|
||||
if !query.group_by.is_empty() {
|
||||
push_groups(&mut html, &records, &query.group_by);
|
||||
}
|
||||
push_records(&mut html, &records, &query);
|
||||
html.push_str("</main></body></html>");
|
||||
truncate_page(&mut html);
|
||||
retained_html_response(StatusCode::OK, html, render_permit)
|
||||
}
|
||||
|
||||
fn push_page_start(html: &mut String) {
|
||||
html.push_str("<!doctype html><html lang=\"en\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"><title>WEB status</title><style>body{font:14px system-ui,sans-serif;margin:0;background:#f5f7fa;color:#17202a}main{max-width:1600px;margin:auto;padding:20px}h1,h2{margin:.4em 0}section{background:#fff;border:1px solid #d9e0e7;border-radius:8px;padding:12px;margin:12px 0;overflow:auto}form{display:flex;flex-wrap:wrap;gap:8px;align-items:end}label{display:grid;gap:3px}input,select,button{font:inherit;padding:5px}table{border-collapse:collapse;width:100%}th,td{border:1px solid #d9e0e7;padding:5px;text-align:left;vertical-align:top}th{background:#edf2f7;position:sticky;top:0}code,pre{font:12px ui-monospace,monospace;white-space:pre-wrap;overflow-wrap:anywhere}details{max-width:900px}.muted{color:#657786}.bad{color:#a00}</style></head><body><main>");
|
||||
}
|
||||
|
||||
fn push_filter_form(html: &mut String, query: &StatusQuery) {
|
||||
html.push_str("<section><h2>Filters</h2><form method=\"get\" action=\"/web-status\">");
|
||||
input(html, "window_secs", &query.window_secs.to_string());
|
||||
input(
|
||||
html,
|
||||
"ip",
|
||||
&query.ip.map(|value| value.to_string()).unwrap_or_default(),
|
||||
);
|
||||
input(
|
||||
html,
|
||||
"session",
|
||||
&query
|
||||
.session
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_default(),
|
||||
);
|
||||
input(
|
||||
html,
|
||||
"user_agent",
|
||||
query.user_agent.as_deref().unwrap_or_default(),
|
||||
);
|
||||
input(html, "key", query.key.as_deref().unwrap_or_default());
|
||||
input(html, "limit", &query.limit.to_string());
|
||||
html.push_str("<label>group_by<select name=\"group_by\" multiple size=\"4\">");
|
||||
for group in [
|
||||
GroupBy::Ip,
|
||||
GroupBy::Session,
|
||||
GroupBy::UserAgent,
|
||||
GroupBy::Key,
|
||||
] {
|
||||
html.push_str("<option value=\"");
|
||||
html.push_str(group.as_str());
|
||||
if query.group_by.contains(&group) {
|
||||
html.push_str("\" selected>");
|
||||
} else {
|
||||
html.push_str("\">");
|
||||
}
|
||||
html.push_str(group.as_str());
|
||||
html.push_str("</option>");
|
||||
}
|
||||
html.push_str("</select></label><button type=\"submit\">Observe</button></form></section>");
|
||||
}
|
||||
|
||||
fn input(html: &mut String, name: &str, value: &str) {
|
||||
html.push_str("<label>");
|
||||
escape(html, name);
|
||||
html.push_str("<input name=\"");
|
||||
escape(html, name);
|
||||
html.push_str("\" value=\"");
|
||||
escape(html, value);
|
||||
html.push_str("\"></label>");
|
||||
}
|
||||
|
||||
fn summary_row(html: &mut String, name: &str, value: &str) {
|
||||
html.push_str("<tr><th>");
|
||||
escape(html, name);
|
||||
html.push_str("</th><td>");
|
||||
escape(html, value);
|
||||
html.push_str("</td></tr>");
|
||||
}
|
||||
|
||||
fn push_groups(html: &mut String, records: &[Arc<StoredTraceRecord>], groups: &[GroupBy]) {
|
||||
let mut summaries = BTreeMap::<Vec<String>, GroupSummary>::new();
|
||||
let mut overflow = 0usize;
|
||||
for stored in records {
|
||||
let values = groups
|
||||
.iter()
|
||||
.map(|group| group_value(&stored.record, *group))
|
||||
.collect::<Vec<_>>();
|
||||
if let Some(summary) = summaries.get_mut(&values) {
|
||||
summary.count += 1;
|
||||
summary.latest_seq = summary.latest_seq.max(stored.record.seq);
|
||||
} else if summaries.len() < MAX_GROUPS {
|
||||
summaries.insert(
|
||||
values,
|
||||
GroupSummary {
|
||||
count: 1,
|
||||
latest_seq: stored.record.seq,
|
||||
},
|
||||
);
|
||||
} else {
|
||||
overflow += 1;
|
||||
}
|
||||
}
|
||||
let mut summaries = summaries.into_iter().collect::<Vec<_>>();
|
||||
summaries.sort_by(|(left_values, left), (right_values, right)| {
|
||||
right
|
||||
.count
|
||||
.cmp(&left.count)
|
||||
.then_with(|| left_values.cmp(right_values))
|
||||
});
|
||||
html.push_str("<section><h2>Groups</h2><table><thead><tr>");
|
||||
for group in groups {
|
||||
html.push_str("<th>");
|
||||
html.push_str(group.as_str());
|
||||
html.push_str("</th>");
|
||||
}
|
||||
html.push_str("<th>records</th><th>latest seq</th></tr></thead><tbody>");
|
||||
for (values, summary) in summaries {
|
||||
html.push_str("<tr>");
|
||||
for value in values {
|
||||
html.push_str("<td>");
|
||||
escape(html, &value);
|
||||
html.push_str("</td>");
|
||||
}
|
||||
html.push_str("<td>");
|
||||
html.push_str(&summary.count.to_string());
|
||||
html.push_str("</td><td>");
|
||||
html.push_str(&summary.latest_seq.to_string());
|
||||
html.push_str("</td></tr>");
|
||||
if html.len() >= MAX_PAGE_BYTES / 2 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if overflow != 0 {
|
||||
html.push_str("<tr><td colspan=\"6\" class=\"muted\">Additional groups omitted: ");
|
||||
html.push_str(&overflow.to_string());
|
||||
html.push_str("</td></tr>");
|
||||
}
|
||||
html.push_str("</tbody></table></section>");
|
||||
}
|
||||
|
||||
fn group_value(record: &TraceRecord, group: GroupBy) -> String {
|
||||
match group {
|
||||
GroupBy::Ip => client_ip(record).map(|value| value.to_string()),
|
||||
GroupBy::Session => record.identity.session_id.map(|value| value.to_string()),
|
||||
GroupBy::UserAgent => record.user_agent.clone(),
|
||||
GroupBy::Key => record.identity.key_fingerprint.clone(),
|
||||
}
|
||||
.unwrap_or_else(|| "-".to_string())
|
||||
}
|
||||
|
||||
fn push_records(html: &mut String, records: &[Arc<StoredTraceRecord>], query: &StatusQuery) {
|
||||
html.push_str("<section><h2>Records</h2><table><thead><tr><th>seq</th><th>time</th><th>kind</th><th>route/event</th><th>method</th><th>status</th><th>IP</th><th>session</th><th>user / key</th><th>User-Agent</th><th>details</th></tr></thead><tbody>");
|
||||
let mut shown = 0usize;
|
||||
for stored in records.iter().take(query.limit) {
|
||||
if html.len() >= MAX_PAGE_BYTES.saturating_sub(64 * 1024) {
|
||||
break;
|
||||
}
|
||||
push_record(html, &stored.record);
|
||||
shown += 1;
|
||||
}
|
||||
if shown == 0 {
|
||||
html.push_str("<tr><td colspan=\"11\" class=\"muted\">No matching records</td></tr>");
|
||||
}
|
||||
html.push_str("</tbody></table>");
|
||||
if records.len() > shown && shown != 0 {
|
||||
let before = records[shown - 1].record.seq;
|
||||
html.push_str("<p><a href=\"");
|
||||
escape(html, &pagination_url(query, before));
|
||||
html.push_str("\">Next page</a></p>");
|
||||
}
|
||||
html.push_str("</section>");
|
||||
}
|
||||
|
||||
fn push_record(html: &mut String, record: &TraceRecord) {
|
||||
html.push_str("<tr><td><a href=\"/web-status?record=");
|
||||
html.push_str(&record.seq.to_string());
|
||||
html.push_str("\">");
|
||||
html.push_str(&record.seq.to_string());
|
||||
html.push_str("</a></td><td>");
|
||||
escape(html, &format_time(record.epoch_millis));
|
||||
let (kind, route, method, status) = match &record.kind {
|
||||
TraceRecordKind::Http(http) => (
|
||||
"http",
|
||||
http.route.as_str(),
|
||||
http.method.as_str(),
|
||||
http.status
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "-".to_string()),
|
||||
),
|
||||
TraceRecordKind::Websocket(message) => (
|
||||
"websocket",
|
||||
message.direction.as_str(),
|
||||
message.message_type,
|
||||
message.payload_bytes.to_string(),
|
||||
),
|
||||
TraceRecordKind::Lifecycle(event) => (
|
||||
"lifecycle",
|
||||
event.event.as_str(),
|
||||
"-",
|
||||
event.reason.unwrap_or("-").to_string(),
|
||||
),
|
||||
};
|
||||
for value in [kind, route, method, status.as_str()] {
|
||||
html.push_str("</td><td>");
|
||||
escape(html, value);
|
||||
}
|
||||
html.push_str("</td><td>");
|
||||
escape(
|
||||
html,
|
||||
&client_ip(record)
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "-".to_string()),
|
||||
);
|
||||
html.push_str("</td><td>");
|
||||
escape(html, &option_u64(record.identity.session_id));
|
||||
html.push_str("</td><td>");
|
||||
escape(html, record.identity.user.as_deref().unwrap_or("-"));
|
||||
html.push_str(" / ");
|
||||
escape(
|
||||
html,
|
||||
record.identity.key_fingerprint.as_deref().unwrap_or("-"),
|
||||
);
|
||||
html.push_str("</td><td>");
|
||||
escape(html, record.user_agent.as_deref().unwrap_or("-"));
|
||||
html.push_str("</td><td><details><summary>request → response</summary>");
|
||||
match &record.kind {
|
||||
TraceRecordKind::Http(http) => {
|
||||
html.push_str("<p><code>");
|
||||
escape(html, &http.method);
|
||||
html.push(' ');
|
||||
escape(html, &http.path);
|
||||
html.push_str("</code></p>");
|
||||
push_headers(html, "request headers", &http.request_headers);
|
||||
push_body(html, "request body", http.request_body.as_ref());
|
||||
push_headers(html, "response headers", &http.response_headers);
|
||||
push_body(html, "response body", http.response_body.as_ref());
|
||||
if let Some(timings) = &http.timings {
|
||||
html.push_str("<h3>timings</h3><pre>service/head accepted: 0 us\nrequest body: ");
|
||||
html.push_str(&option_u64(timings.request_body_us));
|
||||
html.push_str(" us\nresponse ready: ");
|
||||
html.push_str(&option_u64(timings.response_ready_us));
|
||||
html.push_str(" us\nresponse body consumed/polled: ");
|
||||
html.push_str(&option_u64(timings.response_body_us));
|
||||
html.push_str(" us\n(kernel flush and TCP ACK are not observed)</pre>");
|
||||
}
|
||||
push_frames(html, &http.frames);
|
||||
}
|
||||
TraceRecordKind::Websocket(message) => {
|
||||
html.push_str("<pre>connection: ");
|
||||
html.push_str(&message.connection_id.to_string());
|
||||
html.push_str("\nlane: ");
|
||||
html.push_str(
|
||||
&message
|
||||
.lane_id
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "-".to_string()),
|
||||
);
|
||||
html.push_str("\ndirection: ");
|
||||
html.push_str(message.direction.as_str());
|
||||
html.push_str("\nmessage: ");
|
||||
html.push_str(message.message_type);
|
||||
html.push_str("\npayload bytes: ");
|
||||
html.push_str(&message.payload_bytes.to_string());
|
||||
html.push_str("\nduration: ");
|
||||
html.push_str(&option_u64(message.duration_us));
|
||||
html.push_str(" us</pre>");
|
||||
push_body(html, "message body", message.body.as_ref());
|
||||
push_frames(html, &message.frames);
|
||||
}
|
||||
TraceRecordKind::Lifecycle(event) => {
|
||||
html.push_str("<pre>event: ");
|
||||
html.push_str(event.event.as_str());
|
||||
html.push_str("\nstream: ");
|
||||
html.push_str(
|
||||
&event
|
||||
.stream_id
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_else(|| "-".to_string()),
|
||||
);
|
||||
html.push_str("\nreason: ");
|
||||
html.push_str(event.reason.unwrap_or("-"));
|
||||
html.push_str("</pre>");
|
||||
}
|
||||
}
|
||||
html.push_str("</details></td></tr>");
|
||||
}
|
||||
|
||||
fn pagination_url(query: &StatusQuery, before_seq: u64) -> String {
|
||||
let mut serializer = url::form_urlencoded::Serializer::new(String::from("/web-status?"));
|
||||
serializer.append_pair("window_secs", &query.window_secs.to_string());
|
||||
if let Some(ip) = query.ip {
|
||||
serializer.append_pair("ip", &ip.to_string());
|
||||
}
|
||||
if let Some(session) = query.session {
|
||||
serializer.append_pair("session", &session.to_string());
|
||||
}
|
||||
if let Some(user_agent) = &query.user_agent {
|
||||
serializer.append_pair("user_agent", user_agent);
|
||||
}
|
||||
if let Some(key) = &query.key {
|
||||
serializer.append_pair("key", key);
|
||||
}
|
||||
for group in &query.group_by {
|
||||
serializer.append_pair("group_by", group.as_str());
|
||||
}
|
||||
serializer.append_pair("limit", &query.limit.to_string());
|
||||
serializer.append_pair("before_seq", &before_seq.to_string());
|
||||
serializer.finish()
|
||||
}
|
||||
|
||||
fn format_time(epoch_millis: u64) -> String {
|
||||
chrono::DateTime::from_timestamp_millis(epoch_millis as i64)
|
||||
.map(|value| value.to_rfc3339_opts(chrono::SecondsFormat::Millis, true))
|
||||
.unwrap_or_else(|| epoch_millis.to_string())
|
||||
}
|
||||
|
||||
fn option_u64(value: Option<u64>) -> String {
|
||||
value
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "-".to_string())
|
||||
}
|
||||
|
||||
fn body_mode(policy: &WebDebugConfig) -> &'static str {
|
||||
match policy.body_capture {
|
||||
crate::config::WebDebugBodyCapture::Off => "off",
|
||||
crate::config::WebDebugBodyCapture::Metadata => "metadata",
|
||||
crate::config::WebDebugBodyCapture::Prefix => "prefix",
|
||||
crate::config::WebDebugBodyCapture::Full => "full",
|
||||
}
|
||||
}
|
||||
|
||||
fn yes_no(value: bool) -> &'static str {
|
||||
if value { "yes" } else { "no" }
|
||||
}
|
||||
|
||||
fn escape(output: &mut String, value: &str) {
|
||||
for character in value.chars() {
|
||||
match character {
|
||||
'&' => output.push_str("&"),
|
||||
'<' => output.push_str("<"),
|
||||
'>' => output.push_str(">"),
|
||||
'"' => output.push_str("""),
|
||||
'\'' => output.push_str("'"),
|
||||
_ => output.push(character),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate_page(html: &mut String) {
|
||||
const SUFFIX: &str = "[page output truncated]";
|
||||
if html.len() <= MAX_PAGE_BYTES {
|
||||
return;
|
||||
}
|
||||
let mut end = MAX_PAGE_BYTES.saturating_sub(SUFFIX.len());
|
||||
while !html.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
html.truncate(end);
|
||||
html.push_str(SUFFIX);
|
||||
}
|
||||
|
||||
fn html_error(status: StatusCode, title: &str, message: &str) -> Response<Full<Bytes>> {
|
||||
let mut html = String::new();
|
||||
push_page_start(&mut html);
|
||||
html.push_str("<h1 class=\"bad\">");
|
||||
escape(&mut html, title);
|
||||
html.push_str("</h1><p>");
|
||||
escape(&mut html, message);
|
||||
html.push_str("</p></main></body></html>");
|
||||
html_response(status, html)
|
||||
}
|
||||
|
||||
fn html_response(status: StatusCode, html: String) -> Response<Full<Bytes>> {
|
||||
html_bytes_response(status, Bytes::from(html))
|
||||
}
|
||||
|
||||
fn retained_html_response(
|
||||
status: StatusCode,
|
||||
html: String,
|
||||
permit: OwnedSemaphorePermit,
|
||||
) -> Response<Full<Bytes>> {
|
||||
html_bytes_response(
|
||||
status,
|
||||
Bytes::from_owner(RenderedPage {
|
||||
html,
|
||||
_permit: permit,
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
fn html_bytes_response(status: StatusCode, html: Bytes) -> Response<Full<Bytes>> {
|
||||
let mut response = Response::new(Full::new(html));
|
||||
*response.status_mut() = status;
|
||||
response.headers_mut().insert(
|
||||
header::CONTENT_TYPE,
|
||||
HeaderValue::from_static("text/html; charset=utf-8"),
|
||||
);
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
||||
response.headers_mut().insert(
|
||||
header::CONTENT_SECURITY_POLICY,
|
||||
HeaderValue::from_static("default-src 'none'; style-src 'unsafe-inline'; frame-ancestors 'none'; base-uri 'none'; form-action 'self'"),
|
||||
);
|
||||
response.headers_mut().insert(
|
||||
header::REFERRER_POLICY,
|
||||
HeaderValue::from_static("no-referrer"),
|
||||
);
|
||||
response.headers_mut().insert(
|
||||
header::X_CONTENT_TYPE_OPTIONS,
|
||||
HeaderValue::from_static("nosniff"),
|
||||
);
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::X_FRAME_OPTIONS, HeaderValue::from_static("DENY"));
|
||||
response
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "web_status/tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,86 @@
|
||||
use base64::Engine as _;
|
||||
|
||||
use super::{MAX_PAGE_BYTES, escape, yes_no};
|
||||
|
||||
pub(super) fn push_frames(html: &mut String, frames: &[crate::web::trace::TraceFrame]) {
|
||||
if frames.is_empty() {
|
||||
return;
|
||||
}
|
||||
html.push_str("<h3>frames</h3><table><tr><th>dir</th><th>type</th><th>stream/lane</th><th>payload</th><th>WINDOW</th><th>error</th></tr>");
|
||||
for frame in frames {
|
||||
html.push_str("<tr>");
|
||||
for value in [
|
||||
frame.direction.as_str().to_string(),
|
||||
frame.frame_type.unwrap_or("-").to_string(),
|
||||
frame
|
||||
.stream_id
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "-".to_string()),
|
||||
frame
|
||||
.payload_len
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "-".to_string()),
|
||||
frame
|
||||
.window_delta
|
||||
.map(|value| value.to_string())
|
||||
.unwrap_or_else(|| "-".to_string()),
|
||||
frame.parse_error.unwrap_or("-").to_string(),
|
||||
] {
|
||||
html.push_str("<td>");
|
||||
escape(html, &value);
|
||||
html.push_str("</td>");
|
||||
}
|
||||
html.push_str("</tr>");
|
||||
}
|
||||
html.push_str("</table>");
|
||||
}
|
||||
|
||||
pub(super) fn push_headers(
|
||||
html: &mut String,
|
||||
title: &str,
|
||||
headers: &[crate::web::trace::TraceHeader],
|
||||
) {
|
||||
html.push_str("<h3>");
|
||||
escape(html, title);
|
||||
html.push_str("</h3><pre>");
|
||||
for header in headers {
|
||||
escape(html, &header.name);
|
||||
html.push_str(": ");
|
||||
escape(html, header.value.as_deref().unwrap_or("[value omitted]"));
|
||||
html.push('\n');
|
||||
}
|
||||
html.push_str("</pre>");
|
||||
}
|
||||
|
||||
pub(super) fn push_body(
|
||||
html: &mut String,
|
||||
title: &str,
|
||||
body: Option<&crate::web::trace::TraceBodySnapshot>,
|
||||
) {
|
||||
html.push_str("<h3>");
|
||||
escape(html, title);
|
||||
html.push_str("</h3>");
|
||||
let Some(body) = body else {
|
||||
html.push_str("<p class=\"muted\">capture off</p>");
|
||||
return;
|
||||
};
|
||||
html.push_str("<p>observed=");
|
||||
html.push_str(&body.observed_bytes.to_string());
|
||||
html.push_str(" captured=");
|
||||
html.push_str(&body.captured.len().to_string());
|
||||
html.push_str(" state=");
|
||||
html.push_str(body.state.as_str());
|
||||
html.push_str(" truncated=");
|
||||
html.push_str(yes_no(body.truncated));
|
||||
html.push_str("</p><pre>");
|
||||
let available = MAX_PAGE_BYTES
|
||||
.saturating_sub(html.len())
|
||||
.saturating_sub(4096);
|
||||
let raw_limit = available.saturating_mul(3) / 4;
|
||||
let shown = body.captured.len().min(raw_limit);
|
||||
base64::engine::general_purpose::STANDARD.encode_string(&body.captured[..shown], html);
|
||||
if shown < body.captured.len() {
|
||||
html.push_str("\n[page output truncated]");
|
||||
}
|
||||
html.push_str("</pre>");
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
use std::collections::BTreeSet;
|
||||
use std::net::IpAddr;
|
||||
|
||||
use crate::config::WebDebugConfig;
|
||||
use crate::web::trace::TraceRecord;
|
||||
|
||||
const DEFAULT_LIMIT: usize = 200;
|
||||
const MAX_LIMIT: usize = 1000;
|
||||
|
||||
/// Supported status-page grouping dimensions.
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum GroupBy {
|
||||
Ip,
|
||||
Session,
|
||||
UserAgent,
|
||||
Key,
|
||||
}
|
||||
|
||||
impl GroupBy {
|
||||
fn parse(value: &str) -> Option<Self> {
|
||||
match value {
|
||||
"ip" => Some(Self::Ip),
|
||||
"session" => Some(Self::Session),
|
||||
"user_agent" => Some(Self::UserAgent),
|
||||
"key" => Some(Self::Key),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the canonical query and table label.
|
||||
pub(super) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Ip => "ip",
|
||||
Self::Session => "session",
|
||||
Self::UserAgent => "user_agent",
|
||||
Self::Key => "key",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Validated bounded status-page filter and pagination state.
|
||||
pub(super) struct StatusQuery {
|
||||
pub(super) window_secs: u64,
|
||||
pub(super) ip: Option<IpAddr>,
|
||||
pub(super) session: Option<u64>,
|
||||
pub(super) user_agent: Option<String>,
|
||||
pub(super) key: Option<String>,
|
||||
pub(super) group_by: Vec<GroupBy>,
|
||||
pub(super) limit: usize,
|
||||
pub(super) before_seq: Option<u64>,
|
||||
pub(super) record: Option<u64>,
|
||||
}
|
||||
|
||||
/// Parses a strict query without accepting unknown or ambiguous fields.
|
||||
pub(super) fn parse_query(
|
||||
raw: Option<&str>,
|
||||
policy: &WebDebugConfig,
|
||||
) -> Result<StatusQuery, String> {
|
||||
let mut query = StatusQuery {
|
||||
window_secs: policy.default_window_secs,
|
||||
ip: None,
|
||||
session: None,
|
||||
user_agent: None,
|
||||
key: None,
|
||||
group_by: Vec::new(),
|
||||
limit: DEFAULT_LIMIT,
|
||||
before_seq: None,
|
||||
record: None,
|
||||
};
|
||||
let mut seen = BTreeSet::new();
|
||||
for (name, value) in url::form_urlencoded::parse(raw.unwrap_or_default().as_bytes()) {
|
||||
let name = name.as_ref();
|
||||
let value = value.as_ref();
|
||||
if name != "group_by" && !seen.insert(name.to_string()) {
|
||||
return Err(format!("{name} must not repeat"));
|
||||
}
|
||||
match name {
|
||||
"window_secs" => {
|
||||
query.window_secs = parse_positive_u64(value, "window_secs")?;
|
||||
}
|
||||
"ip" => {
|
||||
let parsed = value
|
||||
.parse::<IpAddr>()
|
||||
.map_err(|_| "ip must be a canonical IP address".to_string())?;
|
||||
if parsed.to_string() != value {
|
||||
return Err("ip must use canonical formatting".to_string());
|
||||
}
|
||||
query.ip = Some(parsed);
|
||||
}
|
||||
"session" => query.session = Some(parse_positive_u64(value, "session")?),
|
||||
"user_agent" => {
|
||||
if value.is_empty() || value.len() > 512 {
|
||||
return Err("user_agent must contain 1..512 bytes".to_string());
|
||||
}
|
||||
query.user_agent = Some(value.to_string());
|
||||
}
|
||||
"key" => {
|
||||
if value.is_empty() || value.len() > 64 {
|
||||
return Err("key must contain 1..64 bytes".to_string());
|
||||
}
|
||||
query.key = Some(value.to_string());
|
||||
}
|
||||
"group_by" => {
|
||||
let group = GroupBy::parse(value).ok_or_else(|| {
|
||||
"group_by must be ip, session, user_agent, or key".to_string()
|
||||
})?;
|
||||
if query.group_by.contains(&group) {
|
||||
return Err("group_by values must not repeat".to_string());
|
||||
}
|
||||
query.group_by.push(group);
|
||||
}
|
||||
"limit" => {
|
||||
query.limit = value
|
||||
.parse::<usize>()
|
||||
.ok()
|
||||
.filter(|value| (1..=MAX_LIMIT).contains(value))
|
||||
.ok_or_else(|| "limit must be within 1..1000".to_string())?;
|
||||
}
|
||||
"before_seq" => {
|
||||
query.before_seq = Some(parse_positive_u64(value, "before_seq")?);
|
||||
}
|
||||
"record" => query.record = Some(parse_positive_u64(value, "record")?),
|
||||
_ => return Err(format!("unknown query field `{name}`")),
|
||||
}
|
||||
}
|
||||
if query.window_secs > policy.max_window_secs {
|
||||
return Err(format!(
|
||||
"window_secs must not exceed {}",
|
||||
policy.max_window_secs
|
||||
));
|
||||
}
|
||||
Ok(query)
|
||||
}
|
||||
|
||||
fn parse_positive_u64(value: &str, field: &str) -> Result<u64, String> {
|
||||
value
|
||||
.parse::<u64>()
|
||||
.ok()
|
||||
.filter(|value| *value != 0)
|
||||
.ok_or_else(|| format!("{field} must be a positive integer"))
|
||||
}
|
||||
|
||||
/// Applies the complete filter predicate to one immutable record.
|
||||
pub(super) fn record_matches(record: &TraceRecord, query: &StatusQuery, since_millis: u64) -> bool {
|
||||
!(record.epoch_millis < since_millis
|
||||
|| query.before_seq.is_some_and(|before| record.seq >= before)
|
||||
|| query.record.is_some_and(|seq| record.seq != seq)
|
||||
|| query.ip.is_some_and(|ip| client_ip(record) != Some(ip))
|
||||
|| query
|
||||
.session
|
||||
.is_some_and(|session| record.identity.session_id != Some(session))
|
||||
|| query.user_agent.as_ref().is_some_and(|needle| {
|
||||
record
|
||||
.user_agent
|
||||
.as_deref()
|
||||
.is_none_or(|value| !contains_ascii_case_insensitive(value, needle))
|
||||
})
|
||||
|| query.key.as_ref().is_some_and(|key| {
|
||||
record.identity.user.as_deref() != Some(key)
|
||||
&& record.identity.key_fingerprint.as_deref() != Some(key)
|
||||
}))
|
||||
}
|
||||
|
||||
fn contains_ascii_case_insensitive(value: &str, needle: &str) -> bool {
|
||||
let needle = needle.as_bytes();
|
||||
needle.is_empty()
|
||||
|| value
|
||||
.as_bytes()
|
||||
.windows(needle.len())
|
||||
.any(|window| window.eq_ignore_ascii_case(needle))
|
||||
}
|
||||
|
||||
/// Returns the trusted effective address or direct peer fallback.
|
||||
pub(super) fn client_ip(record: &TraceRecord) -> Option<IpAddr> {
|
||||
record.effective_ip.or(record.peer_ip)
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
use http_body_util::BodyExt as _;
|
||||
|
||||
use super::*;
|
||||
use crate::web::trace::{TraceIdentity, TraceLifecycleEvent};
|
||||
|
||||
#[test]
|
||||
fn query_rejects_noncanonical_ip_and_excessive_window() {
|
||||
let policy = WebDebugConfig::default();
|
||||
assert!(parse_query(Some("ip=2001%3A0db8%3A%3A1"), &policy).is_err());
|
||||
assert!(parse_query(Some("window_secs=3601"), &policy).is_err());
|
||||
assert!(parse_query(Some("session=1&session=2"), &policy).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn html_escaping_covers_active_markup_characters() {
|
||||
let mut output = String::new();
|
||||
escape(&mut output, "<script a='\"'>&");
|
||||
assert_eq!(output, "<script a='"'>&");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renderer_filters_groups_and_sets_control_plane_security_headers() {
|
||||
let mut policy = WebDebugConfig::default();
|
||||
policy.enabled = true;
|
||||
let mut limits = crate::config::WebLimitsConfig::default();
|
||||
limits.debug_records_capacity = 8;
|
||||
limits.debug_bytes_global = 16 * 1024;
|
||||
let store = WebTraceStore::new(policy.clone(), &limits);
|
||||
store.record_lifecycle(
|
||||
None,
|
||||
Some("192.0.2.40".parse().unwrap()),
|
||||
TraceIdentity {
|
||||
session_id: Some(42),
|
||||
user: Some("alice".to_string()),
|
||||
key_fingerprint: Some("0123456789abcdef".to_string()),
|
||||
},
|
||||
TraceLifecycleEvent::SessionCreated,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
let response = render(
|
||||
Some("ip=192.0.2.40&session=42&key=0123456789abcdef&group_by=ip&group_by=key"),
|
||||
&store,
|
||||
&policy,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store");
|
||||
assert!(
|
||||
response
|
||||
.headers()
|
||||
.contains_key(header::CONTENT_SECURITY_POLICY)
|
||||
);
|
||||
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||
let body = std::str::from_utf8(&body).unwrap();
|
||||
assert!(body.contains("session_created"));
|
||||
assert!(body.contains("0123456789abcdef"));
|
||||
assert!(body.contains("192.0.2.40"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn render_permits_remain_owned_by_inflight_response_bodies() {
|
||||
let policy = WebDebugConfig::default();
|
||||
let limits = crate::config::WebLimitsConfig::default();
|
||||
let store = WebTraceStore::new(policy.clone(), &limits);
|
||||
|
||||
let first = render(None, &store, &policy).await;
|
||||
let second = render(None, &store, &policy).await;
|
||||
let busy = render(None, &store, &policy).await;
|
||||
assert_eq!(busy.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
|
||||
drop(first);
|
||||
let admitted = render(None, &store, &policy).await;
|
||||
assert_eq!(admitted.status(), StatusCode::OK);
|
||||
drop(second);
|
||||
drop(admitted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_truncation_preserves_utf8_boundary_and_cap() {
|
||||
let mut html = "я".repeat(MAX_PAGE_BYTES);
|
||||
truncate_page(&mut html);
|
||||
assert!(html.len() <= MAX_PAGE_BYTES);
|
||||
assert!(html.ends_with("[page output truncated]"));
|
||||
}
|
||||
@@ -36,7 +36,7 @@ use super::load::{LoadedConfig, ProxyConfig};
|
||||
#[allow(unused_imports)]
|
||||
use crate::config::{
|
||||
CidrRateLimitKey, LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel,
|
||||
MeWriterPickMode,
|
||||
MeWriterPickMode, WebDebugConfig, web_debug_fits_limits,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::config::{ListenerConfig, SynLimitMode};
|
||||
|
||||
@@ -89,6 +89,7 @@ pub struct HotFields {
|
||||
pub user_max_unique_ips_global_each: usize,
|
||||
pub user_max_unique_ips_mode: crate::config::UserMaxUniqueIpsMode,
|
||||
pub user_max_unique_ips_window_secs: u64,
|
||||
pub web_debug: WebDebugConfig,
|
||||
}
|
||||
|
||||
impl HotFields {
|
||||
@@ -218,6 +219,7 @@ impl HotFields {
|
||||
user_max_unique_ips_global_each: cfg.access.user_max_unique_ips_global_each,
|
||||
user_max_unique_ips_mode: cfg.access.user_max_unique_ips_mode,
|
||||
user_max_unique_ips_window_secs: cfg.access.user_max_unique_ips_window_secs,
|
||||
web_debug: cfg.web.debug.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -340,6 +342,9 @@ pub(super) fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyC
|
||||
let process_limits = cfg.web.limits.clone();
|
||||
cfg.web = new.web.clone();
|
||||
cfg.web.limits = process_limits;
|
||||
if !web_debug_fits_limits(&cfg.web.debug, &cfg.web.limits) {
|
||||
cfg.web.debug = old.web.debug.clone();
|
||||
}
|
||||
if cfg.rebuild_runtime_user_auth().is_err() {
|
||||
cfg.runtime_user_auth = None;
|
||||
}
|
||||
|
||||
@@ -496,4 +496,13 @@ pub(super) fn log_changes(
|
||||
new_hot.user_max_unique_ips_window_secs
|
||||
);
|
||||
}
|
||||
if old_hot.web_debug != new_hot.web_debug {
|
||||
info!(
|
||||
"config reload: web.debug updated: enabled={} body_capture={:?} window={}..={}s",
|
||||
new_hot.web_debug.enabled,
|
||||
new_hot.web_debug.body_capture,
|
||||
new_hot.web_debug.default_window_secs,
|
||||
new_hot.web_debug.max_window_secs,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,6 +85,45 @@ fn bind_stale_mode_is_hot() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_debug_policy_is_hot_while_debug_capacity_is_process_owned() {
|
||||
let old = sample_config();
|
||||
let mut new = old.clone();
|
||||
new.web.debug.enabled = true;
|
||||
new.web.debug.default_window_secs = 60;
|
||||
new.web.limits.debug_records_capacity += 1;
|
||||
|
||||
let applied = overlay_hot_fields(&old, &new);
|
||||
assert!(applied.web.debug.enabled);
|
||||
assert_eq!(applied.web.debug.default_window_secs, 60);
|
||||
assert_eq!(
|
||||
applied.web.limits.debug_records_capacity,
|
||||
old.web.limits.debug_records_capacity
|
||||
);
|
||||
assert_ne!(
|
||||
HotFields::from_config(&old),
|
||||
HotFields::from_config(&applied)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_debug_prefix_requiring_deferred_capacity_is_not_hot_applied() {
|
||||
let old = sample_config();
|
||||
let mut new = old.clone();
|
||||
new.web.limits.max_body_bytes = 4 * 1024 * 1024;
|
||||
new.web.debug.body_prefix_bytes = 3 * 1024 * 1024;
|
||||
|
||||
let applied = overlay_hot_fields(&old, &new);
|
||||
assert_eq!(
|
||||
applied.web.limits.max_body_bytes,
|
||||
old.web.limits.max_body_bytes
|
||||
);
|
||||
assert_eq!(
|
||||
applied.web.debug.body_prefix_bytes,
|
||||
old.web.debug.body_prefix_bytes
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keepalive_is_not_hot() {
|
||||
let old = sample_config();
|
||||
|
||||
@@ -14,6 +14,7 @@ use sha2::{Digest, Sha256};
|
||||
use super::*;
|
||||
|
||||
const WEB_CAPABILITY_CONTEXT: &[u8] = b"tdesktop-web-proxy-bridge-v1\n";
|
||||
const WEB_DEBUG_FINGERPRINT_CONTEXT: &[u8] = b"telemt-web-debug-key-fingerprint-v1\0";
|
||||
const MAX_WEB_STATIC_DEPTH: usize = 64;
|
||||
|
||||
/// Builds the immutable WEB routing and decoy snapshot for one generation.
|
||||
@@ -49,6 +50,7 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
||||
client_secret(auth_entry.secret, profile.secret_mode);
|
||||
let capability =
|
||||
derive_web_capability(&client_secret[..client_secret_len], vhost.host.as_bytes())?;
|
||||
let key_fingerprint = debug_key_fingerprint(&client_secret[..client_secret_len]);
|
||||
if !capabilities.insert(capability) {
|
||||
return Err(ProxyError::Config(format!(
|
||||
"WEB vhost `{}` contains profiles with the same client capability",
|
||||
@@ -62,6 +64,7 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
||||
secret_mode: profile.secret_mode,
|
||||
carrier: config.web.carrier,
|
||||
capability,
|
||||
key_fingerprint,
|
||||
max_sessions: profile
|
||||
.max_sessions
|
||||
.unwrap_or(config.web.limits.max_sessions_global),
|
||||
@@ -93,6 +96,13 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn debug_key_fingerprint(secret: &[u8]) -> String {
|
||||
let mut digest = Sha256::new();
|
||||
digest.update(WEB_DEBUG_FINGERPRINT_CONTEXT);
|
||||
digest.update(secret);
|
||||
hex::encode(&digest.finalize()[..8])
|
||||
}
|
||||
|
||||
/// Derives the Telegram Desktop WEB capability for one exact secret and host.
|
||||
pub(crate) fn derive_web_capability(secret: &[u8], host: &[u8]) -> Result<[u8; 32]> {
|
||||
let mut mac = Hmac::<Sha256>::new_from_slice(secret)
|
||||
|
||||
@@ -259,7 +259,9 @@ const LISTENER_CONFIG_KEYS: &[&str] = &[
|
||||
"web_trusted_proxy_cidrs",
|
||||
];
|
||||
|
||||
const WEB_CONFIG_KEYS: &[&str] = &["enabled", "carrier", "limits", "timeouts", "vhosts"];
|
||||
const WEB_CONFIG_KEYS: &[&str] = &[
|
||||
"enabled", "carrier", "debug", "limits", "timeouts", "vhosts",
|
||||
];
|
||||
|
||||
const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
||||
"max_header_bytes",
|
||||
@@ -269,6 +271,10 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
||||
"max_frames_per_body",
|
||||
"max_http_connections",
|
||||
"max_http_handlers",
|
||||
"websocket_bytes_global",
|
||||
"websocket_admission_watermark_pct",
|
||||
"websocket_eviction_watermark_pct",
|
||||
"websocket_http_connection_reserve",
|
||||
"max_body_readers",
|
||||
"max_body_bytes_global",
|
||||
"max_sessions_global",
|
||||
@@ -290,6 +296,8 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
||||
"max_static_files",
|
||||
"max_static_file_bytes",
|
||||
"max_static_bytes",
|
||||
"debug_records_capacity",
|
||||
"debug_bytes_global",
|
||||
"memory_envelope_bytes",
|
||||
"new_bootstraps_per_minute",
|
||||
"new_bootstraps_burst",
|
||||
@@ -299,11 +307,27 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
||||
"new_streams_burst",
|
||||
];
|
||||
|
||||
const WEB_DEBUG_CONFIG_KEYS: &[&str] = &[
|
||||
"enabled",
|
||||
"capture_lifecycle",
|
||||
"capture_headers",
|
||||
"capture_timings",
|
||||
"capture_frames",
|
||||
"body_capture",
|
||||
"body_prefix_bytes",
|
||||
"decoy_body_prefix_bytes",
|
||||
"default_window_secs",
|
||||
"max_window_secs",
|
||||
];
|
||||
|
||||
const WEB_TIMEOUTS_CONFIG_KEYS: &[&str] = &[
|
||||
"header_secs",
|
||||
"body_secs",
|
||||
"stream_handshake_secs",
|
||||
"long_poll_secs",
|
||||
"websocket_write_secs",
|
||||
"websocket_backpressure_secs",
|
||||
"websocket_eviction_secs",
|
||||
"bootstrap_lifetime_secs",
|
||||
"reconnect_grace_secs",
|
||||
"http_idle_secs",
|
||||
|
||||
@@ -37,6 +37,7 @@ fn known_config_keys_for_suggestion() -> Vec<&'static str> {
|
||||
LISTENER_CONFIG_KEYS,
|
||||
WEB_CONFIG_KEYS,
|
||||
WEB_LIMITS_CONFIG_KEYS,
|
||||
WEB_DEBUG_CONFIG_KEYS,
|
||||
WEB_TIMEOUTS_CONFIG_KEYS,
|
||||
WEB_VHOST_CONFIG_KEYS,
|
||||
WEB_DECOY_CONFIG_KEYS,
|
||||
@@ -241,6 +242,13 @@ pub(super) fn collect_unknown_config_keys(parsed_toml: &toml::Value) -> Vec<Unkn
|
||||
&["web", "limits"],
|
||||
WEB_LIMITS_CONFIG_KEYS,
|
||||
);
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
&["web", "debug"],
|
||||
WEB_DEBUG_CONFIG_KEYS,
|
||||
);
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
|
||||
@@ -2,6 +2,13 @@ use std::collections::HashSet;
|
||||
|
||||
use super::*;
|
||||
|
||||
// Debug capture validation is independent from restart-only storage limits.
|
||||
mod debug;
|
||||
// Memory-envelope arithmetic remains isolated from protocol validation.
|
||||
mod memory;
|
||||
// WebSocket transport policy is validated independently from HTTP body policy.
|
||||
mod websocket;
|
||||
|
||||
const WEB_FRAME_HEADER_BYTES: usize = 8;
|
||||
const WEB_QUEUE_ITEM_COST: usize = 256;
|
||||
const WEB_CONTROL_EXTRA_ITEMS: usize = 16;
|
||||
@@ -59,10 +66,12 @@ pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
|
||||
}
|
||||
|
||||
validate_limits(&config.web.limits)?;
|
||||
debug::validate(&config.web.debug, &config.web.limits)?;
|
||||
if config.web.carrier == WebCarrier::HttpsLanes && config.web.limits.max_http_handlers < 2 {
|
||||
return config_error("web.carrier=https-lanes requires web.limits.max_http_handlers >= 2");
|
||||
}
|
||||
validate_timeouts(&config.web.timeouts)?;
|
||||
websocket::validate(config.web.carrier, &config.web.limits, &config.web.timeouts)?;
|
||||
validate_vhosts(config)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -175,6 +184,8 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
||||
("max_static_files", limits.max_static_files),
|
||||
("max_static_file_bytes", limits.max_static_file_bytes),
|
||||
("max_static_bytes", limits.max_static_bytes),
|
||||
("debug_records_capacity", limits.debug_records_capacity),
|
||||
("debug_bytes_global", limits.debug_bytes_global),
|
||||
("memory_envelope_bytes", limits.memory_envelope_bytes),
|
||||
];
|
||||
if let Some((field, _)) = positive.into_iter().find(|(_, value)| *value == 0) {
|
||||
@@ -309,38 +320,7 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
||||
"web.limits pending ceilings must preserve one uplink batch and downlink progress",
|
||||
);
|
||||
}
|
||||
let body_reservation = limits
|
||||
.max_body_readers
|
||||
.checked_mul(limits.max_body_bytes)
|
||||
.ok_or_else(|| {
|
||||
ProxyError::Config("web.limits body reader reservation overflowed usize".to_string())
|
||||
})?;
|
||||
if body_reservation > limits.max_body_bytes_global
|
||||
|| limits.max_body_bytes_global > u32::MAX as usize
|
||||
{
|
||||
return config_error(
|
||||
"web.limits max_body_readers * max_body_bytes must fit max_body_bytes_global and u32",
|
||||
);
|
||||
}
|
||||
let http_header_reservation = limits
|
||||
.max_http_connections
|
||||
.checked_mul(limits.max_header_bytes)
|
||||
.ok_or_else(|| {
|
||||
ProxyError::Config("web.limits HTTP header reservations overflow usize".to_string())
|
||||
})?;
|
||||
let reserved = limits
|
||||
.pending_bytes_global
|
||||
.checked_add(limits.max_body_bytes_global)
|
||||
.and_then(|value| value.checked_add(limits.max_static_bytes))
|
||||
.and_then(|value| value.checked_add(http_header_reservation))
|
||||
.ok_or_else(|| ProxyError::Config("web.limits byte ceilings overflow usize".to_string()))?;
|
||||
if reserved > limits.memory_envelope_bytes
|
||||
|| limits.memory_envelope_bytes > MAX_WEB_MEMORY_ENVELOPE_BYTES
|
||||
{
|
||||
return config_error(
|
||||
"web.limits memory reservations must fit memory_envelope_bytes within 4 GiB",
|
||||
);
|
||||
}
|
||||
memory::validate(limits)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -350,6 +330,12 @@ fn validate_timeouts(timeouts: &WebTimeoutsConfig) -> Result<()> {
|
||||
("body_secs", timeouts.body_secs),
|
||||
("stream_handshake_secs", timeouts.stream_handshake_secs),
|
||||
("long_poll_secs", timeouts.long_poll_secs),
|
||||
("websocket_write_secs", timeouts.websocket_write_secs),
|
||||
(
|
||||
"websocket_backpressure_secs",
|
||||
timeouts.websocket_backpressure_secs,
|
||||
),
|
||||
("websocket_eviction_secs", timeouts.websocket_eviction_secs),
|
||||
("bootstrap_lifetime_secs", timeouts.bootstrap_lifetime_secs),
|
||||
("reconnect_grace_secs", timeouts.reconnect_grace_secs),
|
||||
("http_idle_secs", timeouts.http_idle_secs),
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
use super::*;
|
||||
|
||||
const MAX_WEB_TRACE_WINDOW_SECS: u64 = 86_400;
|
||||
const MIN_WEB_DEBUG_BYTES_GLOBAL: usize = 4096;
|
||||
|
||||
/// Validates hot debug policy independently from process-owned storage limits.
|
||||
pub(super) fn validate(policy: &WebDebugConfig, limits: &WebLimitsConfig) -> Result<()> {
|
||||
if limits.debug_bytes_global < MIN_WEB_DEBUG_BYTES_GLOBAL {
|
||||
return config_error("web.limits.debug_bytes_global must be at least 4096 bytes");
|
||||
}
|
||||
if policy.default_window_secs == 0
|
||||
|| policy.max_window_secs == 0
|
||||
|| policy.default_window_secs > policy.max_window_secs
|
||||
|| policy.max_window_secs > MAX_WEB_TRACE_WINDOW_SECS
|
||||
{
|
||||
return config_error(
|
||||
"web.debug windows must be non-zero, ordered, and no greater than 86400 seconds",
|
||||
);
|
||||
}
|
||||
if policy.body_prefix_bytes > limits.max_body_bytes {
|
||||
return config_error(
|
||||
"web.debug.body_prefix_bytes must not exceed web.limits.max_body_bytes",
|
||||
);
|
||||
}
|
||||
if policy.body_prefix_bytes > limits.debug_bytes_global
|
||||
|| policy.decoy_body_prefix_bytes > limits.debug_bytes_global
|
||||
{
|
||||
return config_error(
|
||||
"web.debug body prefixes must not exceed web.limits.debug_bytes_global",
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
use super::*;
|
||||
|
||||
const WEB_DEBUG_RENDERERS: usize = 2;
|
||||
const WEB_DEBUG_STATUS_PAGE_BYTES: usize = 8 * 1024 * 1024;
|
||||
const WEB_DEBUG_GROUP_SCRATCH_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// Validates process-wide body, header, queue, static, and debug reservations.
|
||||
pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
|
||||
let body_reservation = limits
|
||||
.max_body_readers
|
||||
.checked_mul(limits.max_body_bytes)
|
||||
.ok_or_else(|| {
|
||||
ProxyError::Config("web.limits body reader reservation overflowed usize".to_string())
|
||||
})?;
|
||||
if body_reservation > limits.max_body_bytes_global
|
||||
|| limits.max_body_bytes_global > u32::MAX as usize
|
||||
{
|
||||
return config_error(
|
||||
"web.limits max_body_readers * max_body_bytes must fit max_body_bytes_global and u32",
|
||||
);
|
||||
}
|
||||
let http_header_reservation = limits
|
||||
.max_http_connections
|
||||
.checked_mul(limits.max_header_bytes)
|
||||
.ok_or_else(|| {
|
||||
ProxyError::Config("web.limits HTTP header reservations overflow usize".to_string())
|
||||
})?;
|
||||
let debug_ring_index = limits
|
||||
.debug_records_capacity
|
||||
.checked_mul(std::mem::size_of::<usize>())
|
||||
.ok_or_else(|| ProxyError::Config("web.limits debug index overflowed usize".to_string()))?;
|
||||
let status_pages = WEB_DEBUG_RENDERERS
|
||||
.checked_mul(WEB_DEBUG_STATUS_PAGE_BYTES)
|
||||
.ok_or_else(|| ProxyError::Config("web.debug status pages overflowed usize".to_string()))?;
|
||||
let debug_reservation = limits
|
||||
.debug_bytes_global
|
||||
.checked_add(
|
||||
debug_ring_index
|
||||
.checked_mul(WEB_DEBUG_RENDERERS)
|
||||
.ok_or_else(|| {
|
||||
ProxyError::Config("web.debug snapshot indexes overflowed usize".to_string())
|
||||
})?,
|
||||
)
|
||||
.and_then(|value| {
|
||||
WEB_DEBUG_RENDERERS
|
||||
.checked_mul(WEB_DEBUG_GROUP_SCRATCH_BYTES)
|
||||
.and_then(|scratch| value.checked_add(scratch))
|
||||
})
|
||||
.ok_or_else(|| ProxyError::Config("web.debug reservations overflowed usize".to_string()))?;
|
||||
let reserved = limits
|
||||
.pending_bytes_global
|
||||
.checked_add(limits.max_body_bytes_global)
|
||||
.and_then(|value| value.checked_add(limits.max_static_bytes))
|
||||
.and_then(|value| value.checked_add(debug_ring_index))
|
||||
.and_then(|value| value.checked_add(status_pages))
|
||||
.and_then(|value| value.checked_add(debug_reservation))
|
||||
.and_then(|value| value.checked_add(http_header_reservation))
|
||||
.ok_or_else(|| ProxyError::Config("web.limits byte ceilings overflow usize".to_string()))?;
|
||||
if reserved > limits.memory_envelope_bytes
|
||||
|| limits.memory_envelope_bytes > MAX_WEB_MEMORY_ENVELOPE_BYTES
|
||||
{
|
||||
return config_error(
|
||||
"web.limits memory reservations must fit memory_envelope_bytes within 4 GiB",
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
use super::*;
|
||||
|
||||
const MAX_WEBSOCKET_BATCH_BYTES: usize = 2 * 1024 * 1024;
|
||||
const WEBSOCKET_IO_BUFFER_BYTES: usize = 64 * 1024;
|
||||
const WEBSOCKET_DRIVER_OVERHEAD_BYTES: usize = 4 * 1024;
|
||||
const WEBSOCKET_FRAME_OVERHEAD_BYTES: usize = 14;
|
||||
|
||||
/// Validates WebSocket admission, memory, and deadline invariants.
|
||||
pub(super) fn validate(
|
||||
carrier: WebCarrier,
|
||||
limits: &WebLimitsConfig,
|
||||
timeouts: &WebTimeoutsConfig,
|
||||
) -> Result<()> {
|
||||
if !(1..100).contains(&limits.websocket_admission_watermark_pct)
|
||||
|| !(1..100).contains(&limits.websocket_eviction_watermark_pct)
|
||||
|| limits.websocket_admission_watermark_pct >= limits.websocket_eviction_watermark_pct
|
||||
{
|
||||
return config_error(
|
||||
"web.limits WebSocket watermarks must satisfy 1 <= admission < eviction < 100",
|
||||
);
|
||||
}
|
||||
if limits.websocket_bytes_global == 0 {
|
||||
return config_error("web.limits.websocket_bytes_global must be > 0");
|
||||
}
|
||||
if timeouts.websocket_eviction_secs > timeouts.websocket_write_secs {
|
||||
return config_error(
|
||||
"web.timeouts.websocket_eviction_secs must not exceed websocket_write_secs",
|
||||
);
|
||||
}
|
||||
if !carrier.uses_websocket() {
|
||||
return Ok(());
|
||||
}
|
||||
if limits.carrier_batch_bytes > MAX_WEBSOCKET_BATCH_BYTES {
|
||||
return config_error(
|
||||
"WebSocket carriers require web.limits.carrier_batch_bytes <= 2097152",
|
||||
);
|
||||
}
|
||||
if limits.websocket_http_connection_reserve == 0
|
||||
|| limits.websocket_http_connection_reserve >= limits.max_http_connections
|
||||
{
|
||||
return config_error(
|
||||
"WebSocket carriers require websocket_http_connection_reserve within [1, max_http_connections)",
|
||||
);
|
||||
}
|
||||
let socket_base = WEBSOCKET_IO_BUFFER_BYTES
|
||||
.checked_mul(2)
|
||||
.and_then(|value| value.checked_add(WEBSOCKET_DRIVER_OVERHEAD_BYTES))
|
||||
.ok_or_else(|| ProxyError::Config("WebSocket base reservation overflowed usize".into()))?;
|
||||
let minimum_websocket_progress = limits
|
||||
.carrier_batch_bytes
|
||||
.checked_add(WEBSOCKET_FRAME_OVERHEAD_BYTES)
|
||||
.and_then(|value| value.checked_mul(2))
|
||||
.and_then(|value| value.checked_add(socket_base))
|
||||
.ok_or_else(|| {
|
||||
ProxyError::Config("WebSocket progress reservation overflowed usize".into())
|
||||
})?;
|
||||
if limits.websocket_bytes_global < minimum_websocket_progress {
|
||||
return config_error(
|
||||
"web.limits.websocket_bytes_global must preserve one socket read and write",
|
||||
);
|
||||
}
|
||||
let data_bytes = limits
|
||||
.pending_bytes_global
|
||||
.saturating_sub(limits.control_bytes_global);
|
||||
let queue_progress = limits
|
||||
.max_body_bytes
|
||||
.checked_add(
|
||||
limits
|
||||
.max_frames_per_body
|
||||
.checked_mul(WEB_QUEUE_ITEM_COST)
|
||||
.ok_or_else(|| {
|
||||
ProxyError::Config("WEB queue progress reservation overflowed usize".into())
|
||||
})?,
|
||||
)
|
||||
.and_then(|value| value.checked_add(limits.carrier_batch_bytes))
|
||||
.ok_or_else(|| {
|
||||
ProxyError::Config("WEB queue progress reservation overflowed usize".into())
|
||||
})?;
|
||||
if limits.websocket_bytes_global > data_bytes.saturating_sub(queue_progress) {
|
||||
return config_error("web.limits.websocket_bytes_global must leave bounded queue progress");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -47,6 +47,56 @@ fn web_config_builds_canonical_runtime_snapshot() {
|
||||
assert_eq!(vhost.profiles[0].max_sessions, 4);
|
||||
assert_eq!(vhost.profiles[0].max_streams, 64);
|
||||
assert_eq!(vhost.profiles[0].max_streams_per_session, 16);
|
||||
assert_eq!(vhost.profiles[0].key_fingerprint.len(), 16);
|
||||
assert_ne!(vhost.profiles[0].key_fingerprint, "0001020304050607");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_debug_table_uses_debug_name_and_bounded_defaults() {
|
||||
let configured = WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
"[web.debug]\nenabled = true\nbody_capture = \"prefix\"\nbody_prefix_bytes = 2048\ndefault_window_secs = 180\nmax_window_secs = 900\n\n[[web.vhosts]]",
|
||||
);
|
||||
let config = load_config_from_temp_toml(&configured);
|
||||
assert!(config.web.debug.enabled);
|
||||
assert_eq!(config.web.debug.body_capture, WebDebugBodyCapture::Prefix);
|
||||
assert_eq!(config.web.debug.body_prefix_bytes, 2048);
|
||||
assert_eq!(config.web.debug.default_window_secs, 180);
|
||||
assert_eq!(config.web.debug.max_window_secs, 900);
|
||||
|
||||
let old_name = format!(
|
||||
"[general]\nconfig_strict = true\n{}",
|
||||
WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
"[web.trace]\nenabled = true\n\n[[web.vhosts]]",
|
||||
)
|
||||
);
|
||||
let error = load_config_error_from_temp_toml(&old_name);
|
||||
assert!(error.contains("web.trace"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_debug_prefix_and_window_validation_fail_closed() {
|
||||
let oversized_prefix = WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
"[web.debug]\nenabled = true\nbody_prefix_bytes = 2097153\n\n[[web.vhosts]]",
|
||||
);
|
||||
let error = load_config_error_from_temp_toml(&oversized_prefix);
|
||||
assert!(error.contains("web.debug.body_prefix_bytes"));
|
||||
|
||||
let reversed_window = WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
"[web.debug]\nenabled = true\ndefault_window_secs = 181\nmax_window_secs = 180\n\n[[web.vhosts]]",
|
||||
);
|
||||
let error = load_config_error_from_temp_toml(&reversed_window);
|
||||
assert!(error.contains("web.debug windows"));
|
||||
|
||||
let undersized_store = WEB_CONFIG.replace(
|
||||
"carrier = \"https-lanes\"",
|
||||
"carrier = \"https-lanes\"\n\n[web.limits]\ndebug_bytes_global = 4095",
|
||||
);
|
||||
let error = load_config_error_from_temp_toml(&undersized_store);
|
||||
assert!(error.contains("debug_bytes_global must be at least 4096"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -103,3 +153,52 @@ fn web_ipv6_decoy_uses_a_valid_http_authority() {
|
||||
};
|
||||
assert_eq!(authority, "[::1]:18081");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn websocket_carriers_build_runtime_profiles_with_bounded_defaults() {
|
||||
for (name, carrier) in [
|
||||
("websocket", WebCarrier::Websocket),
|
||||
("websocket-lanes", WebCarrier::WebsocketLanes),
|
||||
] {
|
||||
let configured = WEB_CONFIG.replace("https-lanes", name);
|
||||
let config = load_config_from_temp_toml(&configured);
|
||||
let profile = &config.web.runtime.unwrap().profiles[0];
|
||||
assert_eq!(profile.carrier, carrier);
|
||||
assert_eq!(config.web.limits.websocket_bytes_global, 256 * 1024 * 1024);
|
||||
assert_eq!(config.web.limits.websocket_admission_watermark_pct, 75);
|
||||
assert_eq!(config.web.limits.websocket_eviction_watermark_pct, 90);
|
||||
assert_eq!(config.web.limits.websocket_http_connection_reserve, 64);
|
||||
assert_eq!(config.web.timeouts.websocket_write_secs, 30);
|
||||
assert_eq!(config.web.timeouts.websocket_backpressure_secs, 30);
|
||||
assert_eq!(config.web.timeouts.websocket_eviction_secs, 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn websocket_limits_reject_ambiguous_or_nonprogressing_policy() {
|
||||
let reversed_watermarks = WEB_CONFIG.replace(
|
||||
"carrier = \"https-lanes\"",
|
||||
"carrier = \"websocket\"\n\n[web.limits]\nwebsocket_admission_watermark_pct = 90\nwebsocket_eviction_watermark_pct = 75",
|
||||
);
|
||||
assert!(
|
||||
load_config_error_from_temp_toml(&reversed_watermarks).contains("WebSocket watermarks")
|
||||
);
|
||||
|
||||
let no_http_reserve = WEB_CONFIG.replace(
|
||||
"carrier = \"https-lanes\"",
|
||||
"carrier = \"websocket\"\n\n[web.limits]\nwebsocket_http_connection_reserve = 0",
|
||||
);
|
||||
assert!(
|
||||
load_config_error_from_temp_toml(&no_http_reserve)
|
||||
.contains("websocket_http_connection_reserve")
|
||||
);
|
||||
|
||||
let oversized_batch = WEB_CONFIG.replace(
|
||||
"carrier = \"https-lanes\"",
|
||||
"carrier = \"websocket\"\n\n[web.limits]\nmax_body_bytes = 4194304\ncarrier_batch_bytes = 4194304\nmax_body_readers = 16",
|
||||
);
|
||||
assert!(
|
||||
load_config_error_from_temp_toml(&oversized_batch)
|
||||
.contains("carrier_batch_bytes <= 2097152")
|
||||
);
|
||||
}
|
||||
|
||||
@@ -24,6 +24,8 @@ mod network;
|
||||
mod policies;
|
||||
mod server;
|
||||
mod web;
|
||||
// WEB debug capture policy is reusable by config reload and process storage.
|
||||
mod web_debug;
|
||||
|
||||
pub use access::{AccessConfig, CidrRateLimitKey, RateLimitBps};
|
||||
#[allow(unused_imports)]
|
||||
@@ -56,6 +58,8 @@ pub(crate) use web::{
|
||||
WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset,
|
||||
WebStaticSite,
|
||||
};
|
||||
pub(crate) use web_debug::web_debug_fits_limits;
|
||||
pub use web_debug::{WebDebugBodyCapture, WebDebugConfig};
|
||||
|
||||
fn default_quota_state_path() -> PathBuf {
|
||||
PathBuf::from("telemt.limit.json")
|
||||
|
||||
+84
-3
@@ -6,6 +6,8 @@ use std::sync::Arc;
|
||||
use bytes::Bytes;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::web_debug::WebDebugConfig;
|
||||
|
||||
/// Client-facing secret representation used to derive a WEB capability.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
@@ -16,7 +18,7 @@ pub enum WebSecretMode {
|
||||
Dd,
|
||||
}
|
||||
|
||||
/// HTTP carrier selected for newly issued WEB bridge sessions.
|
||||
/// Carrier selected for newly issued WEB bridge sessions.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum WebCarrier {
|
||||
@@ -25,6 +27,10 @@ pub enum WebCarrier {
|
||||
Https,
|
||||
/// Give every logical stream independent HTTPS sequencing and polling state.
|
||||
HttpsLanes,
|
||||
/// Multiplex all logical streams over one ordered WebSocket.
|
||||
Websocket,
|
||||
/// Give every logical stream an independently owned WebSocket lane.
|
||||
WebsocketLanes,
|
||||
}
|
||||
|
||||
impl WebCarrier {
|
||||
@@ -33,8 +39,25 @@ impl WebCarrier {
|
||||
match self {
|
||||
Self::Https => "https",
|
||||
Self::HttpsLanes => "https-lanes",
|
||||
Self::Websocket => "websocket",
|
||||
Self::WebsocketLanes => "websocket-lanes",
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether one carrier owns independent state per logical stream.
|
||||
pub(crate) const fn uses_lanes(self) -> bool {
|
||||
matches!(self, Self::HttpsLanes | Self::WebsocketLanes)
|
||||
}
|
||||
|
||||
/// Returns whether carrier messages use RFC 6455 instead of HTTP bodies.
|
||||
pub(crate) const fn uses_websocket(self) -> bool {
|
||||
matches!(self, Self::Websocket | Self::WebsocketLanes)
|
||||
}
|
||||
|
||||
/// Returns whether all logical streams share one carrier state machine.
|
||||
pub(crate) const fn is_multiplexed(self) -> bool {
|
||||
matches!(self, Self::Https | Self::Websocket)
|
||||
}
|
||||
}
|
||||
|
||||
/// One access user explicitly exposed through a WEB virtual host.
|
||||
@@ -112,6 +135,18 @@ pub struct WebLimitsConfig {
|
||||
/// Process-wide concurrently executing HTTP handler ceiling.
|
||||
#[serde(default = "default_web_max_http_handlers")]
|
||||
pub max_http_handlers: usize,
|
||||
/// Process-wide transient WebSocket byte sub-budget inside pending bytes.
|
||||
#[serde(default = "default_web_websocket_bytes_global")]
|
||||
pub websocket_bytes_global: usize,
|
||||
/// WebSocket usage percentage above which ordinary admission uses replacement.
|
||||
#[serde(default = "default_web_websocket_admission_watermark_pct")]
|
||||
pub websocket_admission_watermark_pct: u8,
|
||||
/// WebSocket usage percentage that triggers pressure eviction.
|
||||
#[serde(default = "default_web_websocket_eviction_watermark_pct")]
|
||||
pub websocket_eviction_watermark_pct: u8,
|
||||
/// Accepted HTTP connections that WebSocket upgrades must leave available.
|
||||
#[serde(default = "default_web_websocket_http_connection_reserve")]
|
||||
pub websocket_http_connection_reserve: usize,
|
||||
/// Process-wide concurrently collected request body ceiling.
|
||||
#[serde(default = "default_web_max_body_readers")]
|
||||
pub max_body_readers: usize,
|
||||
@@ -130,7 +165,7 @@ pub struct WebLimitsConfig {
|
||||
/// Process-wide live logical-stream ceiling.
|
||||
#[serde(default = "default_web_max_streams_global")]
|
||||
pub max_streams_global: usize,
|
||||
/// Process-wide concurrent inner MTProxy handshake ceiling.
|
||||
/// Process-wide ceiling for inner MTProxy handshakes that received a first byte.
|
||||
#[serde(default = "default_web_max_stream_handshakes")]
|
||||
pub max_stream_handshakes: usize,
|
||||
/// Closed stream identifiers retained by one session.
|
||||
@@ -175,6 +210,12 @@ pub struct WebLimitsConfig {
|
||||
/// Maximum static snapshot bytes across all virtual hosts.
|
||||
#[serde(default = "default_web_max_static_bytes")]
|
||||
pub max_static_bytes: usize,
|
||||
/// Maximum retained WEB debug record count.
|
||||
#[serde(default = "default_web_debug_records_capacity")]
|
||||
pub debug_records_capacity: usize,
|
||||
/// Process-wide retained and in-flight WEB debug byte ceiling.
|
||||
#[serde(default = "default_web_debug_bytes_global")]
|
||||
pub debug_bytes_global: usize,
|
||||
/// Declared process envelope for HTTP heads, bodies, queues, and static snapshots.
|
||||
#[serde(default = "default_web_memory_envelope_bytes")]
|
||||
pub memory_envelope_bytes: usize,
|
||||
@@ -208,6 +249,10 @@ impl Default for WebLimitsConfig {
|
||||
max_frames_per_body: default_web_max_frames_per_body(),
|
||||
max_http_connections: default_web_max_http_connections(),
|
||||
max_http_handlers: default_web_max_http_handlers(),
|
||||
websocket_bytes_global: default_web_websocket_bytes_global(),
|
||||
websocket_admission_watermark_pct: default_web_websocket_admission_watermark_pct(),
|
||||
websocket_eviction_watermark_pct: default_web_websocket_eviction_watermark_pct(),
|
||||
websocket_http_connection_reserve: default_web_websocket_http_connection_reserve(),
|
||||
max_body_readers: default_web_max_body_readers(),
|
||||
max_body_bytes_global: default_web_max_body_bytes_global(),
|
||||
max_sessions_global: default_web_max_sessions_global(),
|
||||
@@ -229,6 +274,8 @@ impl Default for WebLimitsConfig {
|
||||
max_static_files: default_web_max_static_files(),
|
||||
max_static_file_bytes: default_web_max_static_file_bytes(),
|
||||
max_static_bytes: default_web_max_static_bytes(),
|
||||
debug_records_capacity: default_web_debug_records_capacity(),
|
||||
debug_bytes_global: default_web_debug_bytes_global(),
|
||||
memory_envelope_bytes: default_web_memory_envelope_bytes(),
|
||||
new_bootstraps_per_minute: default_web_new_bootstraps_per_minute(),
|
||||
new_bootstraps_burst: default_web_new_bootstraps_burst(),
|
||||
@@ -249,12 +296,21 @@ pub struct WebTimeoutsConfig {
|
||||
/// Deadline for collecting one authenticated carrier request body.
|
||||
#[serde(default = "default_web_body_timeout_secs")]
|
||||
pub body_secs: u64,
|
||||
/// Deadline for the inner MTProxy handshake on one logical stream.
|
||||
/// Deadline from the first inner byte through MTProxy authentication.
|
||||
#[serde(default = "default_web_stream_handshake_timeout_secs")]
|
||||
pub stream_handshake_secs: u64,
|
||||
/// Maximum wait for one empty downlink long poll.
|
||||
#[serde(default = "default_web_long_poll_timeout_secs")]
|
||||
pub long_poll_secs: u64,
|
||||
/// Maximum wait for one WebSocket write to complete.
|
||||
#[serde(default = "default_web_websocket_write_secs")]
|
||||
pub websocket_write_secs: u64,
|
||||
/// Maximum wait for WebSocket queue or byte-budget progress.
|
||||
#[serde(default = "default_web_websocket_backpressure_secs")]
|
||||
pub websocket_backpressure_secs: u64,
|
||||
/// Maximum graceful close wait for an evicted WebSocket.
|
||||
#[serde(default = "default_web_websocket_eviction_secs")]
|
||||
pub websocket_eviction_secs: u64,
|
||||
/// Lifetime of an unused bootstrap credential and closed-token replay marker.
|
||||
#[serde(default = "default_web_bootstrap_lifetime_secs")]
|
||||
pub bootstrap_lifetime_secs: u64,
|
||||
@@ -279,6 +335,9 @@ impl Default for WebTimeoutsConfig {
|
||||
body_secs: default_web_body_timeout_secs(),
|
||||
stream_handshake_secs: default_web_stream_handshake_timeout_secs(),
|
||||
long_poll_secs: default_web_long_poll_timeout_secs(),
|
||||
websocket_write_secs: default_web_websocket_write_secs(),
|
||||
websocket_backpressure_secs: default_web_websocket_backpressure_secs(),
|
||||
websocket_eviction_secs: default_web_websocket_eviction_secs(),
|
||||
bootstrap_lifetime_secs: default_web_bootstrap_lifetime_secs(),
|
||||
reconnect_grace_secs: default_web_reconnect_grace_secs(),
|
||||
http_idle_secs: default_web_http_idle_secs(),
|
||||
@@ -300,6 +359,9 @@ pub struct WebConfig {
|
||||
/// Hard process and protocol limits.
|
||||
#[serde(default)]
|
||||
pub limits: WebLimitsConfig,
|
||||
/// Hot-reloadable bounded server-side debug policy.
|
||||
#[serde(default)]
|
||||
pub debug: WebDebugConfig,
|
||||
/// WEB lifecycle deadlines.
|
||||
#[serde(default)]
|
||||
pub timeouts: WebTimeoutsConfig,
|
||||
@@ -348,6 +410,8 @@ pub(crate) struct WebRuntimeProfile {
|
||||
pub(crate) carrier: WebCarrier,
|
||||
/// HMAC-derived bridge capability.
|
||||
pub(crate) capability: [u8; 32],
|
||||
/// Non-secret domain-separated client-secret fingerprint for debugging.
|
||||
pub(crate) key_fingerprint: String,
|
||||
/// Per-profile live session ceiling.
|
||||
pub(crate) max_sessions: usize,
|
||||
/// Per-profile live logical-stream ceiling.
|
||||
@@ -403,6 +467,14 @@ macro_rules! u32_default {
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! u8_default {
|
||||
($name:ident, $value:expr) => {
|
||||
fn $name() -> u8 {
|
||||
$value
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! u64_default {
|
||||
($name:ident, $value:expr) => {
|
||||
fn $name() -> u64 {
|
||||
@@ -418,6 +490,10 @@ usize_default!(default_web_carrier_batch_bytes, 2 * 1024 * 1024);
|
||||
usize_default!(default_web_max_frames_per_body, 4096);
|
||||
usize_default!(default_web_max_http_connections, 1024);
|
||||
usize_default!(default_web_max_http_handlers, 512);
|
||||
usize_default!(default_web_websocket_bytes_global, 256 * 1024 * 1024);
|
||||
u8_default!(default_web_websocket_admission_watermark_pct, 75);
|
||||
u8_default!(default_web_websocket_eviction_watermark_pct, 90);
|
||||
usize_default!(default_web_websocket_http_connection_reserve, 64);
|
||||
usize_default!(default_web_max_body_readers, 32);
|
||||
usize_default!(default_web_max_body_bytes_global, 64 * 1024 * 1024);
|
||||
usize_default!(default_web_max_sessions_global, 128);
|
||||
@@ -439,6 +515,8 @@ usize_default!(default_web_max_profiles, 32);
|
||||
usize_default!(default_web_max_static_files, 4096);
|
||||
usize_default!(default_web_max_static_file_bytes, 8 * 1024 * 1024);
|
||||
usize_default!(default_web_max_static_bytes, 64 * 1024 * 1024);
|
||||
usize_default!(default_web_debug_records_capacity, 65_536);
|
||||
usize_default!(default_web_debug_bytes_global, 64 * 1024 * 1024);
|
||||
usize_default!(default_web_memory_envelope_bytes, 768 * 1024 * 1024);
|
||||
u32_default!(default_web_new_bootstraps_per_minute, 1200);
|
||||
u32_default!(default_web_new_bootstraps_burst, 256);
|
||||
@@ -450,6 +528,9 @@ u64_default!(default_web_header_timeout_secs, 10);
|
||||
u64_default!(default_web_body_timeout_secs, 30);
|
||||
u64_default!(default_web_stream_handshake_timeout_secs, 10);
|
||||
u64_default!(default_web_long_poll_timeout_secs, 25);
|
||||
u64_default!(default_web_websocket_write_secs, 30);
|
||||
u64_default!(default_web_websocket_backpressure_secs, 30);
|
||||
u64_default!(default_web_websocket_eviction_secs, 1);
|
||||
u64_default!(default_web_bootstrap_lifetime_secs, 120);
|
||||
u64_default!(default_web_reconnect_grace_secs, 120);
|
||||
u64_default!(default_web_http_idle_secs, 75);
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::web::WebLimitsConfig;
|
||||
|
||||
/// Request and response body retention policy for WEB debugging.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum WebDebugBodyCapture {
|
||||
/// Omits body snapshots entirely.
|
||||
Off,
|
||||
/// Retains body lengths and completion state without payload bytes.
|
||||
#[default]
|
||||
Metadata,
|
||||
/// Retains a bounded prefix of each body.
|
||||
Prefix,
|
||||
/// Retains complete bounded carrier bodies and bounded decoy prefixes.
|
||||
Full,
|
||||
}
|
||||
|
||||
/// Hot-reloadable WEB server-side debugging policy.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct WebDebugConfig {
|
||||
/// Enables process-owned WEB debug collection.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
/// Records typed bridge, session, stream, handshake, and relay events.
|
||||
#[serde(default = "default_true")]
|
||||
pub capture_lifecycle: bool,
|
||||
/// Retains allowlisted header values and names of all other headers.
|
||||
#[serde(default = "default_true")]
|
||||
pub capture_headers: bool,
|
||||
/// Retains request service and body-consumption timing points.
|
||||
#[serde(default = "default_true")]
|
||||
pub capture_timings: bool,
|
||||
/// Parses carrier bodies into bounded frame metadata.
|
||||
#[serde(default = "default_true")]
|
||||
pub capture_frames: bool,
|
||||
/// Controls request and response body byte retention.
|
||||
#[serde(default)]
|
||||
pub body_capture: WebDebugBodyCapture,
|
||||
/// Maximum retained body prefix for recognized WEB requests.
|
||||
#[serde(default = "default_body_prefix_bytes")]
|
||||
pub body_prefix_bytes: usize,
|
||||
/// Maximum retained body prefix for ordinary decoy traffic.
|
||||
#[serde(default = "default_decoy_body_prefix_bytes")]
|
||||
pub decoy_body_prefix_bytes: usize,
|
||||
/// Default observation window presented by the status page.
|
||||
#[serde(default = "default_window_secs")]
|
||||
pub default_window_secs: u64,
|
||||
/// Largest observation window accepted by the status page.
|
||||
#[serde(default = "default_max_window_secs")]
|
||||
pub max_window_secs: u64,
|
||||
}
|
||||
|
||||
impl Default for WebDebugConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
capture_lifecycle: true,
|
||||
capture_headers: true,
|
||||
capture_timings: true,
|
||||
capture_frames: true,
|
||||
body_capture: WebDebugBodyCapture::Metadata,
|
||||
body_prefix_bytes: default_body_prefix_bytes(),
|
||||
decoy_body_prefix_bytes: default_decoy_body_prefix_bytes(),
|
||||
default_window_secs: default_window_secs(),
|
||||
max_window_secs: default_max_window_secs(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn default_body_prefix_bytes() -> usize {
|
||||
4096
|
||||
}
|
||||
|
||||
fn default_decoy_body_prefix_bytes() -> usize {
|
||||
4096
|
||||
}
|
||||
|
||||
fn default_window_secs() -> u64 {
|
||||
180
|
||||
}
|
||||
|
||||
fn default_max_window_secs() -> u64 {
|
||||
3600
|
||||
}
|
||||
|
||||
/// Checks whether a hot debug policy fits restart-frozen process capacities.
|
||||
pub(crate) fn web_debug_fits_limits(policy: &WebDebugConfig, limits: &WebLimitsConfig) -> bool {
|
||||
policy.body_prefix_bytes <= limits.max_body_bytes
|
||||
&& policy.body_prefix_bytes <= limits.debug_bytes_global
|
||||
&& policy.decoy_body_prefix_bytes <= limits.debug_bytes_global
|
||||
}
|
||||
@@ -246,11 +246,19 @@ impl RuntimeGeneration {
|
||||
|
||||
/// Registers a session only while admission remains open.
|
||||
pub(crate) fn spawn_session<F>(&self, future: F) -> bool
|
||||
where
|
||||
F: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
self.try_spawn_session(future).is_ok()
|
||||
}
|
||||
|
||||
/// Registers one session or returns its unpolled future to the caller.
|
||||
pub(crate) fn try_spawn_session<F>(&self, future: F) -> Result<(), F>
|
||||
where
|
||||
F: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let Some(_registration) = self.session_admission.try_register() else {
|
||||
return false;
|
||||
return Err(future);
|
||||
};
|
||||
let cancel = self.session_cancel.clone();
|
||||
self.sessions.spawn(async move {
|
||||
@@ -259,7 +267,7 @@ impl RuntimeGeneration {
|
||||
_ = future => {}
|
||||
}
|
||||
});
|
||||
true
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Closes admission while preserving already registered sessions.
|
||||
@@ -305,8 +313,18 @@ impl RuntimeGeneration {
|
||||
#[cfg(test)]
|
||||
/// Builds a lightweight runtime generation without network startup tasks.
|
||||
pub(crate) fn test_runtime_generation(id: u64, config: ProxyConfig) -> Arc<RuntimeGeneration> {
|
||||
let (config_tx, config_rx) = watch::channel(Arc::new(config.clone()));
|
||||
let (_admission_tx, admission_rx) = watch::channel(true);
|
||||
test_runtime_generation_with_admission(id, config, admission_rx)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
/// Builds a lightweight runtime generation with a controllable admission gate.
|
||||
pub(crate) fn test_runtime_generation_with_admission(
|
||||
id: u64,
|
||||
config: ProxyConfig,
|
||||
admission_rx: watch::Receiver<bool>,
|
||||
) -> Arc<RuntimeGeneration> {
|
||||
let (config_tx, config_rx) = watch::channel(Arc::new(config.clone()));
|
||||
let stats = Arc::new(Stats::new());
|
||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
||||
config.upstreams,
|
||||
|
||||
@@ -14,6 +14,7 @@ use super::plan::{ListenerBindSpec, listener_bind_plan};
|
||||
#[cfg(unix)]
|
||||
use super::unix::UnixAcceptHandle;
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
use crate::web::trace::WebTraceStore;
|
||||
|
||||
/// Process-owned listener inventory and accept-task lifecycle controller.
|
||||
pub(crate) struct ListenerManager {
|
||||
@@ -43,12 +44,14 @@ impl ListenerManager {
|
||||
pub(crate) fn start(
|
||||
bound: BoundListeners,
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
trace: Arc<WebTraceStore>,
|
||||
) -> Self {
|
||||
let has_web = bound
|
||||
.listeners
|
||||
.iter()
|
||||
.any(|listener| listener.spec.transport == ListenerTransport::Web);
|
||||
let web_runtime = has_web.then(|| WebProcessRuntime::start(active_runtime.clone()));
|
||||
let web_runtime =
|
||||
has_web.then(|| WebProcessRuntime::start_with_trace(active_runtime.clone(), trace));
|
||||
let mut slots = BTreeMap::new();
|
||||
for listener in bound.listeners {
|
||||
let addr = listener.spec.addr;
|
||||
@@ -307,7 +310,11 @@ mod tests {
|
||||
#[cfg(unix)]
|
||||
unix_listener: None,
|
||||
};
|
||||
let mut manager = ListenerManager::start(bound, active_runtime);
|
||||
let trace = WebTraceStore::new(
|
||||
runtime.config().web.debug.clone(),
|
||||
&runtime.config().web.limits,
|
||||
);
|
||||
let mut manager = ListenerManager::start(bound, active_runtime, trace);
|
||||
let blocker = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let blocked_addr = blocker.local_addr().unwrap();
|
||||
let mut desired = ProxyConfig::default();
|
||||
@@ -330,7 +337,11 @@ mod tests {
|
||||
#[cfg(unix)]
|
||||
unix_listener: None,
|
||||
};
|
||||
let mut manager = ListenerManager::start(bound, active_runtime);
|
||||
let trace = WebTraceStore::new(
|
||||
runtime.config().web.debug.clone(),
|
||||
&runtime.config().web.limits,
|
||||
);
|
||||
let mut manager = ListenerManager::start(bound, active_runtime, trace);
|
||||
let reservation = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let new_addr = reservation.local_addr().unwrap();
|
||||
drop(reservation);
|
||||
|
||||
@@ -17,6 +17,7 @@ use crate::stats::{QuotaStore, Stats};
|
||||
use crate::synlimit_control;
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
use crate::web::trace::WebTraceStore;
|
||||
|
||||
use super::{
|
||||
bootstrap, generation, listeners, reload, reload_supervisor, runtime_startup, runtime_tasks,
|
||||
@@ -104,6 +105,7 @@ pub(super) async fn run_telemt_core(
|
||||
config.access.user_rate_limits.clone(),
|
||||
config.access.cidr_rate_limits.clone(),
|
||||
);
|
||||
let web_trace = WebTraceStore::new(config.web.debug.clone(), &config.web.limits);
|
||||
|
||||
let (detected_ips_tx, detected_ips_rx) = watch::channel((None::<IpAddr>, None::<IpAddr>));
|
||||
let initial_direct_first = config.general.use_middle_proxy && config.general.me2dc_fallback;
|
||||
@@ -154,6 +156,7 @@ pub(super) async fn run_telemt_core(
|
||||
let reload_control_api = reload_control.clone();
|
||||
let active_runtime_rx_api = active_runtime_rx.clone();
|
||||
let runtime_watch_rx_api = runtime_watch_rx.clone();
|
||||
let web_trace_api = web_trace.clone();
|
||||
tokio::spawn(async move {
|
||||
api::serve(
|
||||
listen,
|
||||
@@ -171,6 +174,7 @@ pub(super) async fn run_telemt_core(
|
||||
reload_control_api,
|
||||
active_runtime_rx_api,
|
||||
runtime_watch_rx_api,
|
||||
web_trace_api,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
@@ -314,7 +318,8 @@ pub(super) async fn run_telemt_core(
|
||||
active_runtime_tx.send_replace(Some(active_runtime.clone()));
|
||||
runtime_tasks::mark_runtime_ready(&startup_tracker).await;
|
||||
|
||||
let listener_manager = listeners::ListenerManager::start(bound, active_runtime.clone());
|
||||
let listener_manager =
|
||||
listeners::ListenerManager::start(bound, active_runtime.clone(), web_trace.clone());
|
||||
let reload_supervisor = reload_supervisor::ReloadSupervisor::spawn(
|
||||
active_runtime.clone(),
|
||||
reload_control,
|
||||
@@ -325,6 +330,7 @@ pub(super) async fn run_telemt_core(
|
||||
runtime_log_filter,
|
||||
runtime_watch_tx,
|
||||
listener_manager,
|
||||
web_trace,
|
||||
);
|
||||
|
||||
shutdown::spawn_signal_handlers(active_runtime.clone(), process_started_at);
|
||||
|
||||
@@ -9,6 +9,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::stats::QuotaStore;
|
||||
use crate::web::trace::WebTraceStore;
|
||||
|
||||
use super::generation::{RuntimeGeneration, RuntimeWatchState};
|
||||
use super::listeners::{ListenerManager, PreparedListenerTransition};
|
||||
@@ -29,6 +30,7 @@ pub(crate) struct ReloadSupervisor {
|
||||
runtime_log_filter: RuntimeLogFilter,
|
||||
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
||||
listener_manager: Arc<Mutex<ListenerManager>>,
|
||||
web_trace: Arc<WebTraceStore>,
|
||||
}
|
||||
|
||||
/// Process-owned handle that quiesces reloads before shutdown snapshots the runtime.
|
||||
@@ -105,6 +107,7 @@ impl ReloadSupervisor {
|
||||
runtime_log_filter: RuntimeLogFilter,
|
||||
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
||||
listener_manager: ListenerManager,
|
||||
web_trace: Arc<WebTraceStore>,
|
||||
) -> ReloadSupervisorHandle {
|
||||
let listener_manager = Arc::new(Mutex::new(listener_manager));
|
||||
let supervisor = Self {
|
||||
@@ -117,6 +120,7 @@ impl ReloadSupervisor {
|
||||
runtime_log_filter,
|
||||
runtime_watch_tx,
|
||||
listener_manager: listener_manager.clone(),
|
||||
web_trace,
|
||||
};
|
||||
let control = supervisor.control.clone();
|
||||
let shutdown = CancellationToken::new();
|
||||
@@ -309,6 +313,7 @@ impl ReloadSupervisor {
|
||||
};
|
||||
old_runtime.stop_accepting_sessions();
|
||||
let replaced = self.active_runtime.swap(new_runtime.clone());
|
||||
self.web_trace.apply_policy(&new_runtime.config().web.debug);
|
||||
if let Some(pending) = pending_listener_transition {
|
||||
self.listener_manager
|
||||
.lock()
|
||||
|
||||
@@ -34,6 +34,10 @@ async fn fixture(request: ReloadRequest) -> ReloadFixture {
|
||||
let (detected_ips_tx, _detected_ips_rx) = watch::channel((None, None));
|
||||
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(old_runtime.watch_state()));
|
||||
let listener_manager = Arc::new(Mutex::new(ListenerManager::empty(active_runtime.clone())));
|
||||
let web_trace = crate::web::trace::WebTraceStore::new(
|
||||
old_runtime.config().web.debug.clone(),
|
||||
&old_runtime.config().web.limits,
|
||||
);
|
||||
let supervisor = Arc::new(ReloadSupervisor {
|
||||
active_runtime,
|
||||
control: control.clone(),
|
||||
@@ -44,6 +48,7 @@ async fn fixture(request: ReloadRequest) -> ReloadFixture {
|
||||
runtime_log_filter: runtime_log_filter(),
|
||||
runtime_watch_tx,
|
||||
listener_manager,
|
||||
web_trace,
|
||||
});
|
||||
let command = ReloadCommand {
|
||||
reload_id: accepted.reload_id,
|
||||
@@ -306,6 +311,10 @@ async fn quiesce_joins_idle_supervisor_and_rejects_later_submissions() {
|
||||
runtime_log_filter(),
|
||||
runtime_watch_tx,
|
||||
listener_manager,
|
||||
crate::web::trace::WebTraceStore::new(
|
||||
runtime.config().web.debug.clone(),
|
||||
&runtime.config().web.limits,
|
||||
),
|
||||
);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), handle.quiesce())
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tokio::sync::{RwLock, Semaphore, watch};
|
||||
|
||||
use crate::config::{ProxyConfig, ServerConfig};
|
||||
use crate::config::{ProxyConfig, ServerConfig, web_debug_fits_limits};
|
||||
use crate::crypto::SecureRandom;
|
||||
use crate::ip_tracker::UserIpTracker;
|
||||
use crate::network::probe::{decide_network_capabilities, run_probe};
|
||||
@@ -415,6 +415,10 @@ pub(crate) fn resolve_reload_config(
|
||||
effective.web = old.web.clone();
|
||||
}
|
||||
}
|
||||
if !web_debug_fits_limits(&effective.web.debug, &effective.web.limits) {
|
||||
fields.push("web.debug".to_string());
|
||||
effective.web.debug = old.web.debug.clone();
|
||||
}
|
||||
let runtime_changed = !configs_equal(old, &effective);
|
||||
ResolvedReloadConfig {
|
||||
effective,
|
||||
|
||||
@@ -170,6 +170,27 @@ fn web_allocation_limits_are_deferred_until_restart() {
|
||||
assert!(!resolved.runtime_changed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_debug_prefix_dependent_on_new_capacity_is_deferred_with_limits() {
|
||||
let mut old = ProxyConfig::default();
|
||||
old.rebuild_runtime_user_auth().unwrap();
|
||||
old.rebuild_runtime_web().unwrap();
|
||||
let mut desired = old.clone();
|
||||
desired.web.limits.max_body_bytes = 4 * 1024 * 1024;
|
||||
desired.web.debug.body_prefix_bytes = 3 * 1024 * 1024;
|
||||
|
||||
let resolved = resolve_reload_config(&old, &desired);
|
||||
|
||||
assert_eq!(
|
||||
resolved.deferred_process_fields,
|
||||
vec!["web.limits".to_string(), "web.debug".to_string()]
|
||||
);
|
||||
assert_eq!(
|
||||
resolved.effective.web.debug.body_prefix_bytes,
|
||||
old.web.debug.body_prefix_bytes
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_middle_proxy_requires_a_prepared_pool() {
|
||||
assert!(strict_middle_proxy_unavailable(true, false, false));
|
||||
|
||||
@@ -16,19 +16,37 @@ impl MePool {
|
||||
include_warm: bool,
|
||||
) -> Vec<usize> {
|
||||
let preferred_snapshot = self.preferred_endpoints_by_dc.load();
|
||||
let Some(preferred) = preferred_snapshot.get(&routed_dc) else {
|
||||
return Vec::new();
|
||||
};
|
||||
if preferred.is_empty() {
|
||||
return Vec::new();
|
||||
let mut out = Vec::new();
|
||||
if let Some(preferred) = preferred_snapshot
|
||||
.get(&routed_dc)
|
||||
.filter(|preferred| !preferred.is_empty())
|
||||
{
|
||||
for (idx, w) in writers.iter().enumerate() {
|
||||
if !self.writer_eligible_for_selection(w, include_warm) {
|
||||
continue;
|
||||
}
|
||||
if w.writer_dc == routed_dc && preferred.binary_search(&w.addr).is_ok() {
|
||||
out.push(idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
if !out.is_empty() || !include_warm {
|
||||
return out;
|
||||
}
|
||||
|
||||
let mut out = Vec::new();
|
||||
// A map update publishes desired endpoints before replacement coverage is
|
||||
// guaranteed. Preserve the existing same-DC writer as the final tier so
|
||||
// the data plane remains available while the pool-owned reinit converges.
|
||||
for (idx, w) in writers.iter().enumerate() {
|
||||
if !self.writer_eligible_for_selection(w, include_warm) {
|
||||
continue;
|
||||
}
|
||||
if w.writer_dc == routed_dc && preferred.binary_search(&w.addr).is_ok() {
|
||||
let family_enabled = if w.addr.is_ipv4() {
|
||||
self.decision.ipv4_me
|
||||
} else {
|
||||
self.decision.ipv6_me
|
||||
};
|
||||
if family_enabled
|
||||
&& w.writer_dc == routed_dc
|
||||
&& self.writer_eligible_for_selection(w, true)
|
||||
{
|
||||
out.push(idx);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,10 @@ use crate::network::probe::NetworkDecision;
|
||||
use crate::stats::Stats;
|
||||
|
||||
async fn make_pool() -> (Arc<MePool>, Arc<SecureRandom>) {
|
||||
make_pool_with_decision(NetworkDecision::default()).await
|
||||
}
|
||||
|
||||
async fn make_pool_with_decision(decision: NetworkDecision) -> (Arc<MePool>, Arc<SecureRandom>) {
|
||||
let general = GeneralConfig {
|
||||
me_route_no_writer_mode: MeRouteNoWriterMode::AsyncRecoveryFailfast,
|
||||
me_route_no_writer_wait_ms: 50,
|
||||
@@ -40,7 +44,7 @@ async fn make_pool() -> (Arc<MePool>, Arc<SecureRandom>) {
|
||||
HashMap::new(),
|
||||
HashMap::new(),
|
||||
None,
|
||||
NetworkDecision::default(),
|
||||
decision,
|
||||
None,
|
||||
rng.clone(),
|
||||
Arc::new(Stats::default()),
|
||||
@@ -213,6 +217,61 @@ fn proxy_req_our_addr_from_payload(payload: &[u8]) -> SocketAddr {
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn send_proxy_req_uses_live_same_dc_writer_while_preferred_endpoint_refills() {
|
||||
let decision = NetworkDecision {
|
||||
ipv4_dc: true,
|
||||
ipv4_me: true,
|
||||
effective_prefer: 4,
|
||||
..NetworkDecision::default()
|
||||
};
|
||||
let (pool, _rng) = make_pool_with_decision(decision).await;
|
||||
let old_positive_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 3, 2)), 443);
|
||||
let old_negative_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 3, 3)), 443);
|
||||
let mut old_positive_rx = insert_writer(&pool, 41, 2, old_positive_addr, true).await;
|
||||
let _old_negative_rx = insert_writer(&pool, 42, -2, old_negative_addr, true).await;
|
||||
|
||||
let new_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(192, 0, 2, 42)), 443);
|
||||
pool.update_proxy_maps(
|
||||
HashMap::from([(2, vec![(new_addr.ip(), new_addr.port())])]),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(pool.admission_ready_conditional_cast().await);
|
||||
assert_eq!(
|
||||
pool.preferred_endpoints_by_dc
|
||||
.load()
|
||||
.get(&2)
|
||||
.cloned()
|
||||
.unwrap_or_default(),
|
||||
vec![new_addr]
|
||||
);
|
||||
|
||||
let (conn_id, _rx) = pool.registry.register().await;
|
||||
let result = pool
|
||||
.send_proxy_req(
|
||||
conn_id,
|
||||
2,
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 30004),
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443),
|
||||
b"cold-route",
|
||||
0,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"a live same-DC writer must bridge preferred-endpoint refill: {result:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
recv_data_count(&mut old_positive_rx, Duration::from_millis(50)).await,
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn send_proxy_req_does_not_replay_when_first_bind_commit_fails() {
|
||||
let (pool, _rng) = make_pool().await;
|
||||
|
||||
+135
-11
@@ -54,23 +54,25 @@ const DOCUMENT: &str = r##"<!doctype html>
|
||||
<script nonce="__NONCE__">
|
||||
(()=>{
|
||||
'use strict';
|
||||
const relayOrigin='https://__HOST__',bootstrap='__BOOTSTRAP__',carrier='__CARRIER__';
|
||||
const relayOrigin='https://__HOST__',bootstrap="__BOOTSTRAP__",carrier='__CARRIER__';
|
||||
const batchLimit=__BATCH_LIMIT__,queueLimit=__QUEUE_LIMIT__,queueItemLimit=__QUEUE_ITEMS__;
|
||||
const laneQueueLimit=Math.min(queueLimit,8388608),laneItemLimit=Math.min(queueItemLimit,1024),closedLaneLimit=4096;
|
||||
const fragment=location.hash,androidNonce=/^#android=([A-Za-z0-9_-]{43})$/.exec(fragment)?.[1]||'';
|
||||
history.replaceState(null,'',location.pathname);
|
||||
let initialized=false,closed=false,port=null,sessionToken='',createStarted=false;
|
||||
let initialized=false,closed=false,port=null,sessionToken='',createStarted=false,socket=null,socketReady=false;
|
||||
let queuedBytes=0,queuedItems=0,upSequence=1,downCursor='0',upRunning=false,pollController=null;
|
||||
const pending=[],upPending=[],lanes=new Map(),closedLanes=new Set(),closedLaneOrder=[];
|
||||
const status=state=>{if(port&&!closed)port.postMessage({t:'status',state})};
|
||||
const pause=milliseconds=>new Promise(resolve=>setTimeout(resolve,milliseconds));
|
||||
const socketURL=()=>relayOrigin.replace(/^https:/,'wss:')+'/api/v1/ws';
|
||||
const options=(method,token,body,headers,signal,keepalive)=>({
|
||||
method,body,signal,keepalive:!!keepalive,mode:'same-origin',credentials:'omit',cache:'no-store',redirect:'error',referrerPolicy:'no-referrer',
|
||||
headers:Object.assign(token?{Authorization:'Bearer '+token}:{},body?{'Content-Type':'application/octet-stream'}:{},headers||{})
|
||||
});
|
||||
function reserve(data,lane){
|
||||
if(!data.byteLength||data.byteLength>queueLimit-queuedBytes||queuedItems>=queueItemLimit)return false;
|
||||
if(lane&&(data.byteLength>laneQueueLimit-lane.bytes||lane.items>=laneItemLimit))return false;
|
||||
let buffered=socket?socket.bufferedAmount:0;for(const value of lanes.values())if(value.socket)buffered+=value.socket.bufferedAmount;
|
||||
if(!data.byteLength||data.byteLength>queueLimit-queuedBytes-buffered||queuedItems>=queueItemLimit)return false;
|
||||
if(lane&&(data.byteLength>laneQueueLimit-lane.bytes-(lane.socket?lane.socket.bufferedAmount:0)||lane.items>=laneItemLimit))return false;
|
||||
queuedBytes+=data.byteLength;queuedItems++;if(lane){lane.bytes+=data.byteLength;lane.items++}return true;
|
||||
}
|
||||
function release(bytes,items,lane){queuedBytes-=bytes;queuedItems-=items;if(lane){lane.bytes-=bytes;lane.items-=items}}
|
||||
@@ -154,12 +156,17 @@ async function createSession(first){
|
||||
const welcome=await response.arrayBuffer();
|
||||
port.postMessage(welcome,[welcome]);status('connected');
|
||||
if(carrier==='https-lanes')ensureLane(0);
|
||||
if(carrier==='websocket')openSocket();
|
||||
for(const data of pending.splice(0)){release(data.byteLength,1,null);queueCarrier(data)}
|
||||
if(carrier==='https')poll();else pollLane(lanes.get(0));
|
||||
if(carrier==='https')poll();else if(carrier==='https-lanes')pollLane(lanes.get(0));
|
||||
}catch(error){fail()}
|
||||
}
|
||||
function queueCarrier(data){
|
||||
try{if(carrier==='https')queueUp(data);else for(const value of splitFrames(data))queueLane(value)}catch(error){fail()}
|
||||
try{
|
||||
if(carrier==='https')queueUp(data);
|
||||
else if(carrier==='websocket')queueSocket(data);
|
||||
else for(const value of splitFrames(data))queueLane(value);
|
||||
}catch(error){fail()}
|
||||
}
|
||||
function queueUp(data){if(!reserve(data,null)){fail();return}upPending.push(data);runUp()}
|
||||
async function runUp(){
|
||||
@@ -174,6 +181,31 @@ async function runUp(){
|
||||
}catch(error){fail()}
|
||||
finally{upRunning=false;if(!closed&&sessionToken&&upPending.length)runUp()}
|
||||
}
|
||||
function openSocket(){
|
||||
if(socket||closed)return;socket=new WebSocket(socketURL(),'tproxy-v1.'+sessionToken);socket.binaryType='arraybuffer';
|
||||
socket.onopen=()=>{if(closed)return;socketReady=true;status('connected');runSocketUp()};
|
||||
socket.onmessage=event=>{
|
||||
if(closed||!(event.data instanceof ArrayBuffer)){fail();return}
|
||||
try{const bound=frameBound(event.data,4096,batchLimit);if(bound.bytes!==event.data.byteLength)throw new Error('invalid frame batch')}catch(error){fail();return}
|
||||
port.postMessage({t:'traffic',up:0,down:event.data.byteLength});port.postMessage(event.data,[event.data]);status('connected');
|
||||
};
|
||||
socket.onerror=()=>{};socket.onclose=()=>{socketReady=false;if(!closed)fail()};
|
||||
}
|
||||
function queueSocket(data){if(!reserve(data,null)){fail();return}upPending.push(data);runSocketUp()}
|
||||
async function waitSocket(next,size,limit){
|
||||
while(!closed&&next.readyState===WebSocket.OPEN&&next.bufferedAmount>limit-size)await pause(10);
|
||||
if(closed||next.readyState!==WebSocket.OPEN)throw new Error('websocket closed');
|
||||
}
|
||||
async function runSocketUp(){
|
||||
if(upRunning||!socketReady)return;upRunning=true;
|
||||
try{
|
||||
while(!closed&&socketReady&&upPending.length){
|
||||
const batch=joinPending(upPending,null);await waitSocket(socket,batch.total,queueLimit);socket.send(batch.body);
|
||||
release(batch.total,batch.count,null);port.postMessage({t:'traffic',up:batch.total,down:0});
|
||||
}
|
||||
}catch(error){if(!closed)fail()}
|
||||
finally{upRunning=false;if(!closed&&socketReady&&upPending.length)runSocketUp()}
|
||||
}
|
||||
async function poll(){
|
||||
while(!closed&&sessionToken){
|
||||
try{
|
||||
@@ -190,7 +222,7 @@ async function poll(){
|
||||
}
|
||||
function ensureLane(id){
|
||||
let lane=lanes.get(id);
|
||||
if(!lane){lane={id,sequence:1,cursor:'0',pending:[],bytes:0,items:0,running:false,polling:false,controller:null};lanes.set(id,lane)}
|
||||
if(!lane){lane={id,sequence:1,cursor:'0',pending:[],bytes:0,items:0,running:false,polling:false,controller:null,socket:null,ready:false,remoteClosed:false};lanes.set(id,lane)}
|
||||
return lane;
|
||||
}
|
||||
function rememberLaneClosed(id){
|
||||
@@ -198,10 +230,13 @@ function rememberLaneClosed(id){
|
||||
if(closedLaneOrder.length===closedLaneLimit)closedLanes.delete(closedLaneOrder.shift());
|
||||
closedLanes.add(id);closedLaneOrder.push(id);
|
||||
}
|
||||
function finishLane(lane){
|
||||
function closeFrame(id){const value=new Uint8Array(8);value[0]=3;value[1]=(id>>>16)&255;value[2]=(id>>>8)&255;value[3]=id&255;return value.buffer}
|
||||
function finishLane(lane,notifyClient){
|
||||
if(lanes.get(lane.id)!==lane)return;
|
||||
if(lane.socket&&lane.socket.readyState<WebSocket.CLOSING)lane.socket.close();
|
||||
if(lane.bytes||lane.items)release(lane.bytes,lane.items,lane);
|
||||
lane.pending.length=0;lanes.delete(lane.id);rememberLaneClosed(lane.id);
|
||||
if(notifyClient&&!lane.remoteClosed&&port){const frame=closeFrame(lane.id);port.postMessage(frame,[frame])}
|
||||
}
|
||||
function queueLane(value){
|
||||
let lane=lanes.get(value.id);
|
||||
@@ -210,7 +245,29 @@ function queueLane(value){
|
||||
if(!lane&&value.type!==1)throw new Error('lane did not begin with OPEN');
|
||||
lane=lane||ensureLane(value.id);
|
||||
if(!reserve(value.data,lane)){fail();return}
|
||||
lane.pending.push(value.data);runLaneUp(lane);
|
||||
lane.pending.push(value.data);
|
||||
if(carrier==='websocket-lanes'){openLaneSocket(lane);runLaneSocketUp(lane)}else runLaneUp(lane);
|
||||
}
|
||||
function openLaneSocket(lane){
|
||||
if(lane.socket||closed)return;lane.socket=new WebSocket(socketURL(),'tproxy-lane-v1.'+sessionToken+'.'+String(lane.id));lane.socket.binaryType='arraybuffer';
|
||||
lane.socket.onopen=()=>{if(closed||lanes.get(lane.id)!==lane)return;lane.ready=true;status('connected');runLaneSocketUp(lane)};
|
||||
lane.socket.onmessage=event=>{
|
||||
if(closed||lanes.get(lane.id)!==lane||!(event.data instanceof ArrayBuffer)){finishLane(lane,true);return}
|
||||
let values;try{values=splitFrames(event.data);for(const value of values)if(value.id!==lane.id)throw new Error('cross-lane frame')}catch(error){finishLane(lane,true);return}
|
||||
if(values.some(value=>value.type===3))lane.remoteClosed=true;
|
||||
port.postMessage({t:'traffic',up:0,down:event.data.byteLength});port.postMessage(event.data,[event.data]);status('connected');
|
||||
};
|
||||
lane.socket.onerror=()=>{};lane.socket.onclose=()=>{lane.ready=false;lane.socket=null;if(!closed)finishLane(lane,true)};
|
||||
}
|
||||
async function runLaneSocketUp(lane){
|
||||
if(lane.running||!lane.ready)return;lane.running=true;
|
||||
try{
|
||||
while(!closed&&lane.ready&&lanes.get(lane.id)===lane&&lane.pending.length){
|
||||
const batch=joinPending(lane.pending,lane);await waitSocket(lane.socket,batch.total,laneQueueLimit);lane.socket.send(batch.body);
|
||||
release(batch.total,batch.count,lane);port.postMessage({t:'traffic',up:batch.total,down:0});
|
||||
}
|
||||
}catch(error){if(!closed)finishLane(lane,true)}
|
||||
finally{lane.running=false;if(!closed&&lane.ready&&lane.pending.length)runLaneSocketUp(lane)}
|
||||
}
|
||||
async function runLaneUp(lane){
|
||||
if(lane.running)return;lane.running=true;
|
||||
@@ -232,7 +289,7 @@ async function pollLane(lane){
|
||||
const controller=new AbortController(),laneID=String(lane.id);lane.controller=controller;
|
||||
const response=await request('/api/v1/down',()=>options('POST',sessionToken,null,{'X-Down-Cursor':lane.cursor,'X-Lane-ID':laneID},controller.signal));
|
||||
if(response.status===204){
|
||||
if(response.headers.get('X-Lane-Closed')==='1'){finishLane(lane);return}
|
||||
if(response.headers.get('X-Lane-Closed')==='1'){finishLane(lane,false);return}
|
||||
status('connected');continue;
|
||||
}
|
||||
if(response.status!==200)throw new Error('lane downlink rejected');
|
||||
@@ -250,7 +307,7 @@ function deleteSession(){
|
||||
}
|
||||
function close(notifyServer){
|
||||
if(closed)return;closed=true;if(pollController)pollController.abort();
|
||||
for(const lane of lanes.values())if(lane.controller)lane.controller.abort();
|
||||
if(socket)socket.close();for(const lane of lanes.values()){if(lane.controller)lane.controller.abort();if(lane.socket)lane.socket.close()}
|
||||
if(notifyServer)deleteSession();pending.length=0;upPending.length=0;
|
||||
for(const lane of lanes.values())lane.pending.length=0;lanes.clear();queuedBytes=0;queuedItems=0;if(port)port.close();
|
||||
}
|
||||
@@ -331,4 +388,71 @@ mod tests {
|
||||
.contains("frame-ancestors http://127.0.0.1:*")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rendered_page_is_parseable_by_ios_native_carrier() {
|
||||
let bootstrap = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB";
|
||||
let page = render(
|
||||
"proxy.example.com",
|
||||
bootstrap,
|
||||
2 * 1024 * 1024,
|
||||
32 * 1024 * 1024,
|
||||
16 * 1024,
|
||||
WebCarrier::Https,
|
||||
&SecureRandom::new(),
|
||||
);
|
||||
let accepted_shapes = [
|
||||
format!("const bootstrap=\"{bootstrap}\""),
|
||||
format!("const bootstrap='{bootstrap}'"),
|
||||
format!("bootstrap=\"{bootstrap}\""),
|
||||
];
|
||||
|
||||
assert!(
|
||||
accepted_shapes
|
||||
.iter()
|
||||
.any(|shape| page.body.contains(shape)),
|
||||
"the iOS native carrier cannot parse a comma-declared single-quoted bootstrap"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rendered_page_advertises_exact_websocket_carriers() {
|
||||
let websocket = render(
|
||||
"proxy.example.com",
|
||||
"CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC",
|
||||
2 * 1024 * 1024,
|
||||
32 * 1024 * 1024,
|
||||
16 * 1024,
|
||||
WebCarrier::Websocket,
|
||||
&SecureRandom::new(),
|
||||
);
|
||||
assert!(websocket.body.contains("carrier='websocket'"));
|
||||
assert!(
|
||||
websocket
|
||||
.body
|
||||
.contains("new WebSocket(socketURL(),'tproxy-v1.'+sessionToken)")
|
||||
);
|
||||
assert!(
|
||||
websocket
|
||||
.content_security_policy
|
||||
.contains("connect-src 'self' wss://proxy.example.com")
|
||||
);
|
||||
|
||||
let lanes = render(
|
||||
"proxy.example.com",
|
||||
"DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD",
|
||||
2 * 1024 * 1024,
|
||||
32 * 1024 * 1024,
|
||||
16 * 1024,
|
||||
WebCarrier::WebsocketLanes,
|
||||
&SecureRandom::new(),
|
||||
);
|
||||
assert!(lanes.body.contains("carrier='websocket-lanes'"));
|
||||
assert!(
|
||||
lanes
|
||||
.body
|
||||
.contains("'tproxy-lane-v1.'+sessionToken+'.'+String(lane.id)")
|
||||
);
|
||||
assert!(!lanes.body.contains("__"));
|
||||
}
|
||||
}
|
||||
|
||||
+169
-165
@@ -5,9 +5,8 @@ use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bytes::Bytes;
|
||||
use http_body_util::BodyExt;
|
||||
use http_body_util::combinators::UnsyncBoxBody;
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use hyper::body::Incoming;
|
||||
use hyper::header::{self, HeaderName, HeaderValue};
|
||||
use hyper::server::conn::http1;
|
||||
use hyper::service::service_fn;
|
||||
@@ -18,7 +17,7 @@ use parking_lot::Mutex;
|
||||
use tokio::net::TcpStream;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::config::{WebCarrier, WebClientIpSource, WebRuntimeVhost};
|
||||
use crate::config::{WebClientIpSource, WebRuntimeVhost};
|
||||
use crate::web::bridge;
|
||||
use crate::web::frame::{self, FrameType};
|
||||
use crate::web::manager::{ManagerError, WebProcessRuntime};
|
||||
@@ -29,18 +28,34 @@ mod activity;
|
||||
mod body;
|
||||
// Decoy routing and upstream proxying are isolated from carrier authentication.
|
||||
mod decoy;
|
||||
// Downlink long-poll handling remains isolated from request routing.
|
||||
mod down;
|
||||
// Canonical request parsing rejects ambiguous credentials before routing.
|
||||
mod request;
|
||||
// Carrier response construction and lane-header helpers are shared by handlers.
|
||||
mod response;
|
||||
// RFC 6455 upgrade validation and carrier drivers remain isolated from HTTP routing.
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
mod websocket;
|
||||
// Enabled-debug integration coverage remains separate from carrier behavior tests.
|
||||
#[cfg(test)]
|
||||
#[path = "http/trace_tests.rs"]
|
||||
mod trace_tests;
|
||||
|
||||
use crate::web::trace::{HttpTraceExchange, TraceDirection, TraceLifecycleEvent, TraceRoute};
|
||||
use activity::{ActivityBody, RequestActivity};
|
||||
use body::{CollectBodyError, CollectedBody, collect_body};
|
||||
use body::{CollectBodyError, CollectedBody, RequestBody, collect_body};
|
||||
use decoy::serve_decoy;
|
||||
use down::handle_down;
|
||||
use request::{
|
||||
bearer_token_hash, binary_content_type, bridge_candidate, canonical_request_host,
|
||||
canonical_u64_header, client_ip, compatible_cookie_header, match_profile,
|
||||
};
|
||||
use response::{
|
||||
bad_gateway, carrier_empty, carrier_headers, carrier_lane, full_response, generic_not_found,
|
||||
insert_header, service_unavailable,
|
||||
};
|
||||
|
||||
type BoxError = Box<dyn Error + Send + Sync>;
|
||||
type HttpBody = UnsyncBoxBody<Bytes, BoxError>;
|
||||
@@ -48,6 +63,7 @@ type HttpResponse = Response<HttpBody>;
|
||||
|
||||
const CREATE_BODY_LIMIT: usize = 64;
|
||||
const TRANSPORT_PATHS: [&str; 3] = ["/api/v1/session", "/api/v1/up", "/api/v1/down"];
|
||||
const WEBSOCKET_PATH: &str = "/api/v1/ws";
|
||||
|
||||
/// Serves one bounded HTTP/1.1 connection accepted from an external TLS terminator.
|
||||
pub(crate) async fn serve_connection(
|
||||
@@ -65,13 +81,18 @@ pub(crate) async fn serve_connection(
|
||||
let idle_timeout = Duration::from_secs(config.web.timeouts.http_idle_secs);
|
||||
let last_activity = Arc::new(Mutex::new(Instant::now()));
|
||||
let service_last_activity = Arc::clone(&last_activity);
|
||||
let service = service_fn(move |request| {
|
||||
let service = service_fn(move |mut request| {
|
||||
let runtime = Arc::clone(&runtime);
|
||||
let trusted_proxy_cidrs = Arc::clone(&trusted_proxy_cidrs);
|
||||
let last_activity = Arc::clone(&service_last_activity);
|
||||
let client_ip_source = client_ip_source;
|
||||
async move {
|
||||
let activity = RequestActivity::begin(last_activity);
|
||||
let trace = runtime.trace().begin_http(&request, peer.ip());
|
||||
if let Some(trace) = &trace {
|
||||
request.extensions_mut().insert(Arc::clone(trace));
|
||||
}
|
||||
let request = request.map(|body| RequestBody::new(body, trace.clone()));
|
||||
let response = if let Some(_handler_permit) = runtime.try_http_handler() {
|
||||
handle_request(
|
||||
request,
|
||||
@@ -84,7 +105,11 @@ pub(crate) async fn serve_connection(
|
||||
} else {
|
||||
service_unavailable()
|
||||
};
|
||||
let response = response.map(|body| ActivityBody::new(body, activity).boxed_unsync());
|
||||
if let Some(trace) = &trace {
|
||||
trace.response_ready(&response);
|
||||
}
|
||||
let response =
|
||||
response.map(|body| ActivityBody::new(body, activity, trace).boxed_unsync());
|
||||
Ok::<_, Infallible>(response)
|
||||
}
|
||||
});
|
||||
@@ -93,7 +118,11 @@ pub(crate) async fn serve_connection(
|
||||
.header_read_timeout(header_timeout)
|
||||
.max_buf_size(max_header_bytes)
|
||||
.keep_alive(true)
|
||||
.serve_connection(TokioIo::new(stream), service);
|
||||
.serve_connection(
|
||||
TokioIo::new(websocket::ConnectionIo::new(stream, connection_permit)),
|
||||
service,
|
||||
)
|
||||
.with_upgrades();
|
||||
tokio::pin!(connection);
|
||||
let mut idle_check = tokio::time::interval((idle_timeout / 2).max(Duration::from_secs(1)));
|
||||
idle_check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
@@ -111,16 +140,21 @@ pub(crate) async fn serve_connection(
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(connection_permit);
|
||||
}
|
||||
|
||||
async fn handle_request(
|
||||
request: Request<Incoming>,
|
||||
request: Request<RequestBody>,
|
||||
peer: SocketAddr,
|
||||
client_ip_source: WebClientIpSource,
|
||||
trusted_proxy_cidrs: &[IpNetwork],
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
) -> HttpResponse {
|
||||
set_trace_route(&request, TraceRoute::Decoy);
|
||||
if let Some(trace) = request_trace(&request)
|
||||
&& let Some(client_ip) = client_ip(&request, peer, client_ip_source, trusted_proxy_cidrs)
|
||||
{
|
||||
trace.set_effective_ip(client_ip);
|
||||
}
|
||||
let generation = runtime.active_generation();
|
||||
let config = generation.config();
|
||||
let Some(web_runtime) = config.web.runtime.as_ref() else {
|
||||
@@ -133,6 +167,17 @@ async fn handle_request(
|
||||
return generic_not_found();
|
||||
};
|
||||
let path = request.uri().path();
|
||||
if path == WEBSOCKET_PATH {
|
||||
return websocket::handle(
|
||||
request,
|
||||
peer,
|
||||
client_ip_source,
|
||||
trusted_proxy_cidrs,
|
||||
runtime,
|
||||
vhost,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
if TRANSPORT_PATHS.contains(&path) {
|
||||
return handle_api(
|
||||
request,
|
||||
@@ -159,7 +204,7 @@ async fn handle_request(
|
||||
}
|
||||
|
||||
async fn handle_root(
|
||||
mut request: Request<Incoming>,
|
||||
mut request: Request<RequestBody>,
|
||||
peer: SocketAddr,
|
||||
client_ip_source: WebClientIpSource,
|
||||
trusted_proxy_cidrs: &[IpNetwork],
|
||||
@@ -175,15 +220,34 @@ async fn handle_root(
|
||||
strip_query(&mut request);
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.set_route(TraceRoute::Bridge);
|
||||
trace.set_effective_ip(client_ip);
|
||||
}
|
||||
let carrier = profile.carrier;
|
||||
let Ok(bootstrap) = runtime.issue_bootstrap(profile, client_ip) else {
|
||||
strip_query(&mut request);
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
let bootstrap = match runtime.issue_bootstrap(Arc::clone(&profile), client_ip) {
|
||||
Ok(bootstrap) => bootstrap,
|
||||
Err(error) => {
|
||||
runtime.trace().record_profile_lifecycle(
|
||||
client_ip,
|
||||
None,
|
||||
&profile,
|
||||
TraceLifecycleEvent::BootstrapRejected,
|
||||
None,
|
||||
Some(manager_error_reason(error)),
|
||||
);
|
||||
strip_query(&mut request);
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
};
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.bind_profile(&profile, bootstrap.trace_session_id);
|
||||
trace.register_redaction(bootstrap.token.as_bytes());
|
||||
}
|
||||
let generation = runtime.active_generation();
|
||||
let page = bridge::render(
|
||||
&vhost.host,
|
||||
&bootstrap,
|
||||
&bootstrap.token,
|
||||
generation.config().web.limits.carrier_batch_bytes,
|
||||
generation.config().web.limits.pending_bytes_per_session,
|
||||
generation.config().web.limits.pending_items_per_session,
|
||||
@@ -224,7 +288,7 @@ async fn handle_root(
|
||||
}
|
||||
|
||||
async fn handle_api(
|
||||
request: Request<Incoming>,
|
||||
request: Request<RequestBody>,
|
||||
peer: SocketAddr,
|
||||
client_ip_source: WebClientIpSource,
|
||||
trusted_proxy_cidrs: &[IpNetwork],
|
||||
@@ -237,6 +301,9 @@ async fn handle_api(
|
||||
let Some(client_ip) = client_ip(&request, peer, client_ip_source, trusted_proxy_cidrs) else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.set_effective_ip(client_ip);
|
||||
}
|
||||
let Some(token_hash) = bearer_token_hash(&request) else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
@@ -249,7 +316,7 @@ async fn handle_api(
|
||||
}
|
||||
|
||||
async fn handle_session(
|
||||
request: Request<Incoming>,
|
||||
request: Request<RequestBody>,
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
vhost: Arc<WebRuntimeVhost>,
|
||||
token_hash: crate::web::manager::TokenHash,
|
||||
@@ -262,6 +329,12 @@ async fn handle_session(
|
||||
if request.headers().contains_key(header::CONTENT_TYPE) {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
if let Some(trace) = request_trace(&request)
|
||||
&& let Ok(session) = runtime.get_session(token_hash, &vhost.host)
|
||||
{
|
||||
trace.set_route(TraceRoute::Session);
|
||||
trace.bind_identity(session.trace_identity());
|
||||
}
|
||||
let CollectedBody {
|
||||
request,
|
||||
body,
|
||||
@@ -281,8 +354,14 @@ async fn handle_session(
|
||||
if request.method() != Method::POST || !binary_content_type(&request) {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
if !runtime.has_bootstrap(token_hash, &vhost.host) {
|
||||
let Some((trace_session_id, profile)) =
|
||||
runtime.bootstrap_trace_identity(token_hash, &vhost.host)
|
||||
else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.set_route(TraceRoute::Session);
|
||||
trace.bind_profile(&profile, trace_session_id);
|
||||
}
|
||||
let CollectedBody {
|
||||
request,
|
||||
@@ -295,9 +374,24 @@ async fn handle_session(
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
};
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.record_frames(
|
||||
TraceDirection::Request,
|
||||
&body,
|
||||
&runtime.active_generation().config().web.limits,
|
||||
);
|
||||
}
|
||||
match runtime.create_session(token_hash, &vhost.host, client_ip, &body) {
|
||||
Ok(result) => {
|
||||
let welcome = frame::encode(FrameType::Welcome, 0, &[]);
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.register_redaction(result.token.as_bytes());
|
||||
trace.record_frames(
|
||||
TraceDirection::Response,
|
||||
&welcome,
|
||||
&runtime.active_generation().config().web.limits,
|
||||
);
|
||||
}
|
||||
let mut response = full_response(StatusCode::OK, welcome);
|
||||
carrier_headers(&mut response);
|
||||
insert_header(
|
||||
@@ -315,15 +409,35 @@ async fn handle_session(
|
||||
);
|
||||
response
|
||||
}
|
||||
Err(ManagerError::Limit | ManagerError::Backpressure | ManagerError::Concurrent) => {
|
||||
Err(
|
||||
error @ (ManagerError::Limit | ManagerError::Backpressure | ManagerError::Concurrent),
|
||||
) => {
|
||||
runtime.trace().record_profile_lifecycle(
|
||||
client_ip,
|
||||
Some(trace_session_id),
|
||||
&profile,
|
||||
TraceLifecycleEvent::SessionRejected,
|
||||
None,
|
||||
Some(manager_error_reason(error)),
|
||||
);
|
||||
service_unavailable()
|
||||
}
|
||||
Err(_) => serve_decoy(request, vhost, true, &runtime).await,
|
||||
Err(error) => {
|
||||
runtime.trace().record_profile_lifecycle(
|
||||
client_ip,
|
||||
Some(trace_session_id),
|
||||
&profile,
|
||||
TraceLifecycleEvent::SessionRejected,
|
||||
None,
|
||||
Some(manager_error_reason(error)),
|
||||
);
|
||||
serve_decoy(request, vhost, true, &runtime).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_up(
|
||||
request: Request<Incoming>,
|
||||
request: Request<RequestBody>,
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
vhost: Arc<WebRuntimeVhost>,
|
||||
token_hash: crate::web::manager::TokenHash,
|
||||
@@ -338,6 +452,13 @@ async fn handle_up(
|
||||
let Ok(session) = runtime.get_session(token_hash, &vhost.host) else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
if session.carrier().uses_websocket() {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.set_route(TraceRoute::Uplink);
|
||||
trace.bind_identity(session.trace_identity());
|
||||
}
|
||||
let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
@@ -358,6 +479,13 @@ async fn handle_up(
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
};
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.record_frames(
|
||||
TraceDirection::Request,
|
||||
&body,
|
||||
&runtime.active_generation().config().web.limits,
|
||||
);
|
||||
}
|
||||
let result = match lane_id {
|
||||
Some(lane_id) => session.process_up_lane(lane_id, sequence, &body),
|
||||
None => session.process_up(sequence, &body),
|
||||
@@ -379,151 +507,6 @@ async fn handle_up(
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_down(
|
||||
request: Request<Incoming>,
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
vhost: Arc<WebRuntimeVhost>,
|
||||
token_hash: crate::web::manager::TokenHash,
|
||||
) -> HttpResponse {
|
||||
if request.method() != Method::POST || request.headers().contains_key(header::CONTENT_TYPE) {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
let Some(cursor) = canonical_u64_header(&request, "x-down-cursor") else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
let Ok(session) = runtime.get_session(token_hash, &vhost.host) else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
let CollectedBody {
|
||||
request,
|
||||
body,
|
||||
_body_budget,
|
||||
} = match collect_body(request, &runtime, 1, true).await {
|
||||
Ok(result) => result,
|
||||
Err(CollectBodyError::Limit) => return service_unavailable(),
|
||||
Err(CollectBodyError::Invalid(request)) => {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
};
|
||||
if !body.is_empty() {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
let _lane_poll = if lane_id.is_some() {
|
||||
let Some(permit) = runtime.try_lane_poll() else {
|
||||
return service_unavailable();
|
||||
};
|
||||
Some(permit)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let result = match lane_id {
|
||||
Some(lane_id) => session.poll_down_lane(lane_id, cursor).await,
|
||||
None => session.poll_down(cursor).await,
|
||||
};
|
||||
match result {
|
||||
Ok(result) if result.body.is_empty() => {
|
||||
let mut response = carrier_empty(StatusCode::NO_CONTENT);
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("x-down-cursor"),
|
||||
&result.next_cursor.to_string(),
|
||||
);
|
||||
if result.lane_closed {
|
||||
response.headers_mut().insert(
|
||||
HeaderName::from_static("x-lane-closed"),
|
||||
HeaderValue::from_static("1"),
|
||||
);
|
||||
}
|
||||
response
|
||||
}
|
||||
Ok(result) => {
|
||||
let mut response = full_response(StatusCode::OK, result.body);
|
||||
carrier_headers(&mut response);
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("x-down-cursor"),
|
||||
&result.next_cursor.to_string(),
|
||||
);
|
||||
response
|
||||
}
|
||||
Err(ManagerError::Concurrent | ManagerError::Backpressure | ManagerError::Limit) => {
|
||||
service_unavailable()
|
||||
}
|
||||
Err(_) => serve_decoy(request, vhost, true, &runtime).await,
|
||||
}
|
||||
}
|
||||
|
||||
fn carrier_lane<B>(request: &Request<B>, carrier: WebCarrier) -> Option<Option<u32>> {
|
||||
match carrier {
|
||||
WebCarrier::Https => (!request.headers().contains_key("x-lane-id")).then_some(None),
|
||||
WebCarrier::HttpsLanes => canonical_u64_header(request, "x-lane-id")
|
||||
.and_then(|value| u32::try_from(value).ok())
|
||||
.filter(|value| *value <= frame::MAX_STREAM_ID)
|
||||
.map(Some),
|
||||
}
|
||||
}
|
||||
|
||||
fn carrier_headers(response: &mut HttpResponse) {
|
||||
response.headers_mut().insert(
|
||||
header::CONTENT_TYPE,
|
||||
HeaderValue::from_static("application/octet-stream"),
|
||||
);
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
||||
}
|
||||
|
||||
fn carrier_empty(status: StatusCode) -> HttpResponse {
|
||||
let mut response = empty_response(status);
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
||||
response
|
||||
}
|
||||
|
||||
fn service_unavailable() -> HttpResponse {
|
||||
let mut response = carrier_empty(StatusCode::SERVICE_UNAVAILABLE);
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::RETRY_AFTER, HeaderValue::from_static("1"));
|
||||
response
|
||||
}
|
||||
|
||||
fn bad_gateway() -> HttpResponse {
|
||||
full_response(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
Bytes::from_static(b"site unavailable\n"),
|
||||
)
|
||||
}
|
||||
|
||||
fn generic_not_found() -> HttpResponse {
|
||||
full_response(StatusCode::NOT_FOUND, Bytes::from_static(b"not found\n"))
|
||||
}
|
||||
|
||||
fn full_response(status: StatusCode, body: Bytes) -> HttpResponse {
|
||||
let length = body.len();
|
||||
let body = Full::new(body)
|
||||
.map_err(|never| -> BoxError { match never {} })
|
||||
.boxed_unsync();
|
||||
let mut response = Response::new(body);
|
||||
*response.status_mut() = status;
|
||||
insert_header(&mut response, header::CONTENT_LENGTH, &length.to_string());
|
||||
response
|
||||
}
|
||||
|
||||
fn empty_response(status: StatusCode) -> HttpResponse {
|
||||
full_response(status, Bytes::new())
|
||||
}
|
||||
|
||||
fn insert_header(response: &mut HttpResponse, name: HeaderName, value: &str) {
|
||||
if let Ok(value) = HeaderValue::from_str(value) {
|
||||
response.headers_mut().insert(name, value);
|
||||
}
|
||||
}
|
||||
|
||||
fn strip_query<B>(request: &mut Request<B>) {
|
||||
if request.uri().query().is_some()
|
||||
&& let Ok(uri) = request.uri().path().parse()
|
||||
@@ -531,3 +514,24 @@ fn strip_query<B>(request: &mut Request<B>) {
|
||||
*request.uri_mut() = uri;
|
||||
}
|
||||
}
|
||||
|
||||
fn request_trace<B>(request: &Request<B>) -> Option<&Arc<HttpTraceExchange>> {
|
||||
request.extensions().get::<Arc<HttpTraceExchange>>()
|
||||
}
|
||||
|
||||
fn set_trace_route<B>(request: &Request<B>, route: TraceRoute) {
|
||||
if let Some(trace) = request_trace(request) {
|
||||
trace.set_route(route);
|
||||
}
|
||||
}
|
||||
|
||||
fn manager_error_reason(error: ManagerError) -> &'static str {
|
||||
match error {
|
||||
ManagerError::Authentication => "authentication",
|
||||
ManagerError::Backpressure => "backpressure",
|
||||
ManagerError::Limit => "limit",
|
||||
ManagerError::Protocol => "protocol",
|
||||
ManagerError::Concurrent => "concurrent",
|
||||
ManagerError::Closed => "closed",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ use hyper::body::{Body, Frame, SizeHint};
|
||||
use parking_lot::Mutex;
|
||||
|
||||
use super::{BoxError, HttpBody};
|
||||
use crate::web::trace::{HttpTraceExchange, TraceBodyState, TraceDirection};
|
||||
|
||||
/// Request lifecycle guard that refreshes HTTP connection activity on completion.
|
||||
pub(super) struct RequestActivity {
|
||||
@@ -32,12 +33,33 @@ impl Drop for RequestActivity {
|
||||
pub(super) struct ActivityBody {
|
||||
inner: HttpBody,
|
||||
activity: RequestActivity,
|
||||
trace: Option<Arc<HttpTraceExchange>>,
|
||||
terminal: bool,
|
||||
}
|
||||
|
||||
impl ActivityBody {
|
||||
/// Binds one response body to its request activity guard.
|
||||
pub(super) fn new(inner: HttpBody, activity: RequestActivity) -> Self {
|
||||
Self { inner, activity }
|
||||
pub(super) fn new(
|
||||
inner: HttpBody,
|
||||
activity: RequestActivity,
|
||||
trace: Option<Arc<HttpTraceExchange>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
activity,
|
||||
trace,
|
||||
terminal: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(&mut self, state: TraceBodyState) {
|
||||
if self.terminal {
|
||||
return;
|
||||
}
|
||||
self.terminal = true;
|
||||
if let Some(trace) = &self.trace {
|
||||
trace.body_finished(TraceDirection::Response, state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,6 +72,21 @@ impl Body for ActivityBody {
|
||||
context: &mut Context<'_>,
|
||||
) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
|
||||
let result = Pin::new(&mut self.inner).poll_frame(context);
|
||||
match &result {
|
||||
Poll::Ready(Some(Ok(frame))) => {
|
||||
if let Some(data) = frame.data_ref()
|
||||
&& let Some(trace) = &self.trace
|
||||
{
|
||||
trace.body_data(TraceDirection::Response, data);
|
||||
}
|
||||
if self.inner.is_end_stream() {
|
||||
self.finish(TraceBodyState::Complete);
|
||||
}
|
||||
}
|
||||
Poll::Ready(Some(Err(_))) => self.finish(TraceBodyState::Error),
|
||||
Poll::Ready(None) => self.finish(TraceBodyState::Complete),
|
||||
Poll::Pending => {}
|
||||
}
|
||||
if result.is_ready() {
|
||||
*self.activity.last_activity.lock() = Instant::now();
|
||||
}
|
||||
@@ -64,3 +101,9 @@ impl Body for ActivityBody {
|
||||
self.inner.size_hint()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ActivityBody {
|
||||
fn drop(&mut self) {
|
||||
self.finish(TraceBodyState::Aborted);
|
||||
}
|
||||
}
|
||||
|
||||
+85
-2
@@ -1,11 +1,94 @@
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use http_body_util::{BodyExt, Empty, Limited};
|
||||
use hyper::Request;
|
||||
use hyper::body::{Body as _, Incoming};
|
||||
use hyper::body::{Body, Frame, Incoming, SizeHint};
|
||||
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
use crate::web::trace::{HttpTraceExchange, TraceBodyState, TraceDirection};
|
||||
|
||||
/// Incoming request body wrapper that observes frames without changing streaming semantics.
|
||||
pub(super) struct RequestBody {
|
||||
inner: Incoming,
|
||||
trace: Option<Arc<HttpTraceExchange>>,
|
||||
terminal: bool,
|
||||
}
|
||||
|
||||
impl RequestBody {
|
||||
/// Wraps one Hyper request body with optional enabled-only capture state.
|
||||
pub(super) fn new(inner: Incoming, trace: Option<Arc<HttpTraceExchange>>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
trace,
|
||||
terminal: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Completes observation for a request whose Hyper body is already empty.
|
||||
pub(super) fn finish_empty(&mut self) -> bool {
|
||||
if !self.inner.is_end_stream() {
|
||||
return false;
|
||||
}
|
||||
self.finish(TraceBodyState::Complete);
|
||||
true
|
||||
}
|
||||
|
||||
fn finish(&mut self, state: TraceBodyState) {
|
||||
if self.terminal {
|
||||
return;
|
||||
}
|
||||
self.terminal = true;
|
||||
if let Some(trace) = &self.trace {
|
||||
trace.body_finished(TraceDirection::Request, state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Body for RequestBody {
|
||||
type Data = Bytes;
|
||||
type Error = hyper::Error;
|
||||
|
||||
fn poll_frame(
|
||||
mut self: Pin<&mut Self>,
|
||||
context: &mut Context<'_>,
|
||||
) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
|
||||
let result = Pin::new(&mut self.inner).poll_frame(context);
|
||||
match &result {
|
||||
Poll::Ready(Some(Ok(frame))) => {
|
||||
if let Some(data) = frame.data_ref()
|
||||
&& let Some(trace) = &self.trace
|
||||
{
|
||||
trace.body_data(TraceDirection::Request, data);
|
||||
}
|
||||
if self.inner.is_end_stream() {
|
||||
self.finish(TraceBodyState::Complete);
|
||||
}
|
||||
}
|
||||
Poll::Ready(Some(Err(_))) => self.finish(TraceBodyState::Error),
|
||||
Poll::Ready(None) => self.finish(TraceBodyState::Complete),
|
||||
Poll::Pending => {}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn is_end_stream(&self) -> bool {
|
||||
self.inner.is_end_stream()
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> SizeHint {
|
||||
self.inner.size_hint()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RequestBody {
|
||||
fn drop(&mut self) {
|
||||
self.finish(TraceBodyState::Aborted);
|
||||
}
|
||||
}
|
||||
|
||||
/// Collected carrier request retaining its process-wide body reservation.
|
||||
pub(super) struct CollectedBody {
|
||||
@@ -29,7 +112,7 @@ pub(super) enum CollectBodyError {
|
||||
|
||||
/// Collects one bounded carrier body under reader, byte, and deadline ownership.
|
||||
pub(super) async fn collect_body(
|
||||
request: Request<Incoming>,
|
||||
request: Request<RequestBody>,
|
||||
runtime: &WebProcessRuntime,
|
||||
limit: usize,
|
||||
allow_empty: bool,
|
||||
|
||||
+12
-1
@@ -17,6 +17,7 @@ use crate::config::{WebRuntimeDecoy, WebRuntimeVhost};
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
|
||||
/// Serves the configured ordinary site after optionally removing carrier material.
|
||||
/// Transport-sanitized static fallbacks remain uncacheable after query removal.
|
||||
pub(super) async fn serve_decoy<B>(
|
||||
mut request: Request<B>,
|
||||
vhost: Arc<WebRuntimeVhost>,
|
||||
@@ -27,6 +28,7 @@ where
|
||||
B: hyper::body::Body<Data = Bytes> + Send + 'static,
|
||||
B::Error: Error + Send + Sync + 'static,
|
||||
{
|
||||
super::set_trace_route(&request, crate::web::trace::TraceRoute::Decoy);
|
||||
if sanitize_transport {
|
||||
sanitize_transport_request(&mut request);
|
||||
}
|
||||
@@ -41,7 +43,15 @@ where
|
||||
};
|
||||
let request = Request::from_parts(parts, body);
|
||||
match &vhost.decoy {
|
||||
WebRuntimeDecoy::StaticDirectory(site) => serve_static(request, site),
|
||||
WebRuntimeDecoy::StaticDirectory(site) => {
|
||||
let mut response = serve_static(request, site);
|
||||
if sanitize_transport {
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
||||
}
|
||||
response
|
||||
}
|
||||
WebRuntimeDecoy::HttpUpstream { addr, authority } => {
|
||||
proxy_to_upstream(
|
||||
request,
|
||||
@@ -226,6 +236,7 @@ fn sanitize_transport_request<B>(request: &mut Request<B>) {
|
||||
header::CONTENT_TYPE,
|
||||
header::UPGRADE,
|
||||
HeaderName::from_static("sec-websocket-key"),
|
||||
HeaderName::from_static("sec-websocket-extensions"),
|
||||
HeaderName::from_static("sec-websocket-protocol"),
|
||||
HeaderName::from_static("sec-websocket-version"),
|
||||
HeaderName::from_static("x-down-cursor"),
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use hyper::header::{self, HeaderName, HeaderValue};
|
||||
use hyper::{Request, StatusCode};
|
||||
|
||||
use super::body::{CollectBodyError, CollectedBody, RequestBody, collect_body};
|
||||
use super::decoy::serve_decoy;
|
||||
use super::request::canonical_u64_header;
|
||||
use super::{
|
||||
HttpResponse, carrier_empty, carrier_headers, carrier_lane, full_response, insert_header,
|
||||
request_trace, service_unavailable,
|
||||
};
|
||||
use crate::config::WebRuntimeVhost;
|
||||
use crate::web::manager::{ManagerError, TokenHash, WebProcessRuntime};
|
||||
use crate::web::trace::{TraceDirection, TraceRoute};
|
||||
|
||||
/// Handles one authenticated long-poll downlink exchange.
|
||||
pub(super) async fn handle_down(
|
||||
request: Request<RequestBody>,
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
vhost: Arc<WebRuntimeVhost>,
|
||||
token_hash: TokenHash,
|
||||
) -> HttpResponse {
|
||||
if request.method() != hyper::Method::POST
|
||||
|| request.headers().contains_key(header::CONTENT_TYPE)
|
||||
{
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
let Some(cursor) = canonical_u64_header(&request, "x-down-cursor") else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
let Ok(session) = runtime.get_session(token_hash, &vhost.host) else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
if session.carrier().uses_websocket() {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.set_route(TraceRoute::Downlink);
|
||||
trace.bind_identity(session.trace_identity());
|
||||
}
|
||||
let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
let CollectedBody {
|
||||
request,
|
||||
body,
|
||||
_body_budget,
|
||||
} = match collect_body(request, &runtime, 1, true).await {
|
||||
Ok(result) => result,
|
||||
Err(CollectBodyError::Limit) => return service_unavailable(),
|
||||
Err(CollectBodyError::Invalid(request)) => {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
};
|
||||
if !body.is_empty() {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
let _lane_poll = if lane_id.is_some() {
|
||||
let Some(permit) = runtime.try_lane_poll() else {
|
||||
return service_unavailable();
|
||||
};
|
||||
Some(permit)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let result = match lane_id {
|
||||
Some(lane_id) => session.poll_down_lane(lane_id, cursor).await,
|
||||
None => session.poll_down(cursor).await,
|
||||
};
|
||||
match result {
|
||||
Ok(result) if result.body.is_empty() => {
|
||||
let mut response = carrier_empty(StatusCode::NO_CONTENT);
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("x-down-cursor"),
|
||||
&result.next_cursor.to_string(),
|
||||
);
|
||||
if result.lane_closed {
|
||||
response.headers_mut().insert(
|
||||
HeaderName::from_static("x-lane-closed"),
|
||||
HeaderValue::from_static("1"),
|
||||
);
|
||||
}
|
||||
response
|
||||
}
|
||||
Ok(result) => {
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.record_frames(
|
||||
TraceDirection::Response,
|
||||
&result.body,
|
||||
&runtime.active_generation().config().web.limits,
|
||||
);
|
||||
}
|
||||
let mut response = full_response(StatusCode::OK, result.body);
|
||||
carrier_headers(&mut response);
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("x-down-cursor"),
|
||||
&result.next_cursor.to_string(),
|
||||
);
|
||||
response
|
||||
}
|
||||
Err(ManagerError::Concurrent | ManagerError::Backpressure | ManagerError::Limit) => {
|
||||
service_unavailable()
|
||||
}
|
||||
Err(_) => serve_decoy(request, vhost, true, &runtime).await,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
use bytes::Bytes;
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use hyper::header::{self, HeaderName, HeaderValue};
|
||||
use hyper::{Request, Response, StatusCode};
|
||||
|
||||
use super::request::canonical_u64_header;
|
||||
use super::{BoxError, HttpResponse};
|
||||
use crate::config::WebCarrier;
|
||||
use crate::web::frame;
|
||||
|
||||
/// Validates and resolves the optional carrier lane header.
|
||||
pub(super) fn carrier_lane<B>(request: &Request<B>, carrier: WebCarrier) -> Option<Option<u32>> {
|
||||
match carrier {
|
||||
WebCarrier::Https => (!request.headers().contains_key("x-lane-id")).then_some(None),
|
||||
WebCarrier::HttpsLanes => canonical_u64_header(request, "x-lane-id")
|
||||
.and_then(|value| u32::try_from(value).ok())
|
||||
.filter(|value| *value <= frame::MAX_STREAM_ID)
|
||||
.map(Some),
|
||||
WebCarrier::Websocket | WebCarrier::WebsocketLanes => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies common binary carrier response headers.
|
||||
pub(super) fn carrier_headers(response: &mut HttpResponse) {
|
||||
response.headers_mut().insert(
|
||||
header::CONTENT_TYPE,
|
||||
HeaderValue::from_static("application/octet-stream"),
|
||||
);
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
||||
}
|
||||
|
||||
/// Builds an empty no-store carrier response.
|
||||
pub(super) fn carrier_empty(status: StatusCode) -> HttpResponse {
|
||||
let mut response = empty_response(status);
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
||||
response
|
||||
}
|
||||
|
||||
/// Builds the bounded retryable carrier saturation response.
|
||||
pub(super) fn service_unavailable() -> HttpResponse {
|
||||
let mut response = carrier_empty(StatusCode::SERVICE_UNAVAILABLE);
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::RETRY_AFTER, HeaderValue::from_static("1"));
|
||||
response
|
||||
}
|
||||
|
||||
/// Builds the ordinary decoy upstream failure response.
|
||||
pub(super) fn bad_gateway() -> HttpResponse {
|
||||
full_response(
|
||||
StatusCode::BAD_GATEWAY,
|
||||
Bytes::from_static(b"site unavailable\n"),
|
||||
)
|
||||
}
|
||||
|
||||
/// Builds the ordinary unmatched-host response.
|
||||
pub(super) fn generic_not_found() -> HttpResponse {
|
||||
full_response(StatusCode::NOT_FOUND, Bytes::from_static(b"not found\n"))
|
||||
}
|
||||
|
||||
/// Builds one in-memory response with an exact content length.
|
||||
pub(super) fn full_response(status: StatusCode, body: Bytes) -> HttpResponse {
|
||||
let length = body.len();
|
||||
let body = Full::new(body)
|
||||
.map_err(|never| -> BoxError { match never {} })
|
||||
.boxed_unsync();
|
||||
let mut response = Response::new(body);
|
||||
*response.status_mut() = status;
|
||||
insert_header(&mut response, header::CONTENT_LENGTH, &length.to_string());
|
||||
response
|
||||
}
|
||||
|
||||
fn empty_response(status: StatusCode) -> HttpResponse {
|
||||
full_response(status, Bytes::new())
|
||||
}
|
||||
|
||||
/// Inserts one validated dynamic response header value.
|
||||
pub(super) fn insert_header(response: &mut HttpResponse, name: HeaderName, value: &str) {
|
||||
if let Ok(value) = HeaderValue::from_str(value) {
|
||||
response.headers_mut().insert(name, value);
|
||||
}
|
||||
}
|
||||
+50
-16
@@ -17,7 +17,7 @@ use crate::maestro::generation::test_runtime_generation;
|
||||
use crate::web::frame::{self, FrameType};
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
|
||||
fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
|
||||
pub(super) fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
|
||||
let profile = Arc::new(WebRuntimeProfile {
|
||||
host: "proxy.example.com".to_string(),
|
||||
public_addr: "203.0.113.10:443".parse().unwrap(),
|
||||
@@ -25,6 +25,7 @@ fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
|
||||
secret_mode: WebSecretMode::Plain,
|
||||
carrier,
|
||||
capability,
|
||||
key_fingerprint: "0000000000000000".to_string(),
|
||||
max_sessions: 4,
|
||||
max_streams: 16,
|
||||
max_streams_per_session: 4,
|
||||
@@ -71,7 +72,7 @@ fn runtime_config(capability: [u8; 32], carrier: WebCarrier) -> ProxyConfig {
|
||||
config
|
||||
}
|
||||
|
||||
async fn request(
|
||||
pub(super) async fn request(
|
||||
listener: &TcpListener,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
request: Vec<u8>,
|
||||
@@ -97,7 +98,7 @@ async fn request(
|
||||
response
|
||||
}
|
||||
|
||||
fn split_response(response: &[u8]) -> (&[u8], &[u8]) {
|
||||
pub(super) fn split_response(response: &[u8]) -> (&[u8], &[u8]) {
|
||||
let separator = response
|
||||
.windows(4)
|
||||
.position(|window| window == b"\r\n\r\n")
|
||||
@@ -131,8 +132,8 @@ async fn https_carrier_bootstraps_and_closes_one_session() {
|
||||
assert!(root_headers.starts_with(b"HTTP/1.1 200"));
|
||||
let root_body = std::str::from_utf8(root_body).unwrap();
|
||||
let bootstrap = root_body
|
||||
.split_once("bootstrap='")
|
||||
.and_then(|(_, suffix)| suffix.split_once('\''))
|
||||
.split_once("bootstrap=\"")
|
||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||
.map(|(token, _)| token)
|
||||
.unwrap();
|
||||
assert_eq!(bootstrap.len(), 43);
|
||||
@@ -184,7 +185,7 @@ async fn https_carrier_bootstraps_and_closes_one_session() {
|
||||
assert!(
|
||||
next_root_body
|
||||
.windows(11)
|
||||
.any(|value| value == b"bootstrap='")
|
||||
.any(|value| value == b"bootstrap=\"")
|
||||
);
|
||||
assert!(
|
||||
next_root_body
|
||||
@@ -209,6 +210,39 @@ async fn https_carrier_bootstraps_and_closes_one_session() {
|
||||
replacement.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_bridge_bootstrap_falls_back_to_uncacheable_static_index() {
|
||||
let capability = [15u8; 32];
|
||||
let generation = test_runtime_generation(1, runtime_config(capability, WebCarrier::Https));
|
||||
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||
let runtime = WebProcessRuntime::start(active_runtime);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||
let bridge_request = || {
|
||||
format!(
|
||||
"GET /?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes()
|
||||
};
|
||||
|
||||
let first_response = request(&listener, &runtime, bridge_request()).await;
|
||||
let (_, first_body) = split_response(&first_response);
|
||||
assert!(first_body.windows(11).any(|value| value == b"bootstrap=\""));
|
||||
|
||||
let fallback_response = request(&listener, &runtime, bridge_request()).await;
|
||||
let (fallback_headers, fallback_body) = split_response(&fallback_response);
|
||||
assert!(fallback_headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(
|
||||
response_header(fallback_headers, "cache-control"),
|
||||
"no-store"
|
||||
);
|
||||
assert_eq!(fallback_body, b"<!doctype html><title>decoy</title>");
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_survives_client_address_family_change() {
|
||||
let capability = [8u8; 32];
|
||||
@@ -225,8 +259,8 @@ async fn bootstrap_survives_client_address_family_change() {
|
||||
let (_, root_body) = split_response(&root_response);
|
||||
let root_body = std::str::from_utf8(root_body).unwrap();
|
||||
let bootstrap = root_body
|
||||
.split_once("bootstrap='")
|
||||
.and_then(|(_, suffix)| suffix.split_once('\''))
|
||||
.split_once("bootstrap=\"")
|
||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||
.map(|(token, _)| token)
|
||||
.unwrap();
|
||||
|
||||
@@ -261,8 +295,8 @@ async fn unused_bootstrap_survives_equivalent_runtime_generation_swap() {
|
||||
let (_, root_body) = split_response(&root_response);
|
||||
let root_body = std::str::from_utf8(root_body).unwrap();
|
||||
let bootstrap = root_body
|
||||
.split_once("bootstrap='")
|
||||
.and_then(|(_, suffix)| suffix.split_once('\''))
|
||||
.split_once("bootstrap=\"")
|
||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||
.map(|(token, _)| token)
|
||||
.unwrap();
|
||||
|
||||
@@ -301,8 +335,8 @@ async fn unused_bootstrap_is_rejected_after_profile_identity_change() {
|
||||
let (_, root_body) = split_response(&root_response);
|
||||
let root_body = std::str::from_utf8(root_body).unwrap();
|
||||
let bootstrap = root_body
|
||||
.split_once("bootstrap='")
|
||||
.and_then(|(_, suffix)| suffix.split_once('\''))
|
||||
.split_once("bootstrap=\"")
|
||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||
.map(|(token, _)| token)
|
||||
.unwrap();
|
||||
|
||||
@@ -343,8 +377,8 @@ async fn https_lanes_is_advertised_and_requires_canonical_lane_headers() {
|
||||
let root_body = std::str::from_utf8(root_body).unwrap();
|
||||
assert!(root_body.contains("carrier='https-lanes'"));
|
||||
let bootstrap = root_body
|
||||
.split_once("bootstrap='")
|
||||
.and_then(|(_, suffix)| suffix.split_once('\''))
|
||||
.split_once("bootstrap=\"")
|
||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||
.map(|(token, _)| token)
|
||||
.unwrap();
|
||||
|
||||
@@ -427,8 +461,8 @@ async fn windows_restricted_webview_empty_cookie_preserves_the_carrier_flow() {
|
||||
assert!(root_headers.starts_with(b"HTTP/1.1 200"));
|
||||
let root_body = std::str::from_utf8(root_body).unwrap();
|
||||
let bootstrap = root_body
|
||||
.split_once("bootstrap='")
|
||||
.and_then(|(_, suffix)| suffix.split_once('\''))
|
||||
.split_once("bootstrap=\"")
|
||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||
.map(|(token, _)| token)
|
||||
.unwrap();
|
||||
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use base64::Engine as _;
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
use super::tests::{request, runtime_config, split_response};
|
||||
use crate::config::{WebCarrier, WebDebugBodyCapture};
|
||||
use crate::maestro::generation::test_runtime_generation;
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
use crate::web::trace::{TraceBodyState, TraceRecordKind, TraceRoute};
|
||||
|
||||
#[tokio::test]
|
||||
async fn enabled_debug_records_bridge_request_response_without_credentials() {
|
||||
let capability = [18u8; 32];
|
||||
let mut config = runtime_config(capability, WebCarrier::Https);
|
||||
config.web.debug.enabled = true;
|
||||
config.web.debug.body_capture = WebDebugBodyCapture::Prefix;
|
||||
config.web.debug.body_prefix_bytes = 4096;
|
||||
let generation = test_runtime_generation(1, config);
|
||||
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||
let runtime = WebProcessRuntime::start(active_runtime);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||
let root = format!(
|
||||
"GET /?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.50\r\nUser-Agent: debug-client/1\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
|
||||
let response = request(&listener, &runtime, root).await;
|
||||
let (_, body) = split_response(&response);
|
||||
let body = std::str::from_utf8(body).unwrap();
|
||||
let bootstrap = body
|
||||
.split_once("bootstrap=\"")
|
||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||
.map(|(token, _)| token)
|
||||
.unwrap();
|
||||
let records = runtime
|
||||
.trace()
|
||||
.snapshot_matching(|record| matches!(&record.kind, TraceRecordKind::Http(_)));
|
||||
assert_eq!(records.len(), 1);
|
||||
let record = &records[0].record;
|
||||
assert_eq!(record.effective_ip, Some("192.0.2.50".parse().unwrap()));
|
||||
assert_eq!(record.user_agent.as_deref(), Some("debug-client/1"));
|
||||
assert!(record.identity.session_id.is_some());
|
||||
assert_eq!(record.identity.user.as_deref(), Some("alice"));
|
||||
let TraceRecordKind::Http(http) = &record.kind else {
|
||||
panic!("expected HTTP debug record");
|
||||
};
|
||||
assert_eq!(http.method, "GET");
|
||||
assert_eq!(http.path, "/");
|
||||
assert_eq!(http.route, TraceRoute::Bridge);
|
||||
assert_eq!(http.status, Some(200));
|
||||
assert_eq!(
|
||||
http.response_body.as_ref().unwrap().state,
|
||||
TraceBodyState::Complete
|
||||
);
|
||||
let captured_response = &http.response_body.as_ref().unwrap().captured;
|
||||
assert!(
|
||||
!captured_response
|
||||
.windows(bootstrap.len())
|
||||
.any(|value| value == bootstrap.as_bytes())
|
||||
);
|
||||
assert!(
|
||||
captured_response
|
||||
.windows(bootstrap.len())
|
||||
.any(|value| value.iter().all(|byte| *byte == b'*'))
|
||||
);
|
||||
assert!(http.timings.as_ref().unwrap().response_ready_us.is_some());
|
||||
assert!(http.timings.as_ref().unwrap().response_body_us.is_some());
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
@@ -0,0 +1,461 @@
|
||||
use std::net::SocketAddr;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
|
||||
use base64::Engine as _;
|
||||
use bytes::Bytes;
|
||||
use hyper::header::{self, HeaderName, HeaderValue};
|
||||
use hyper::{Method, Request, StatusCode};
|
||||
use ipnetwork::IpNetwork;
|
||||
use sha1::{Digest as _, Sha1};
|
||||
use sha2::Sha256;
|
||||
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::sync::OwnedSemaphorePermit;
|
||||
|
||||
use super::body::RequestBody;
|
||||
use super::decoy::serve_decoy;
|
||||
use super::request::client_ip;
|
||||
use super::response::{full_response, insert_header};
|
||||
use super::{HttpResponse, request_trace, set_trace_route};
|
||||
use crate::config::{WebCarrier, WebClientIpSource, WebRuntimeVhost};
|
||||
use crate::web::manager::{TokenHash, WebProcessRuntime, WebSocketKind};
|
||||
use crate::web::trace::TraceRoute;
|
||||
|
||||
// Codec buffers and fixed driver state are charged before HTTP 101 commits.
|
||||
const BASE_BUDGET_BYTES: usize = 132 * 1024;
|
||||
const WEBSOCKET_GUID: &[u8] = b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11";
|
||||
|
||||
/// Accepted connection IO retains the process HTTP slot after an upgrade.
|
||||
pub(super) struct ConnectionIo {
|
||||
stream: TcpStream,
|
||||
_connection_permit: OwnedSemaphorePermit,
|
||||
websocket_read: Option<WebSocketReadBoundary>,
|
||||
}
|
||||
|
||||
impl ConnectionIo {
|
||||
pub(super) fn new(stream: TcpStream, connection_permit: OwnedSemaphorePermit) -> Self {
|
||||
Self {
|
||||
stream,
|
||||
_connection_permit: connection_permit,
|
||||
websocket_read: None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn readable(&self) -> std::io::Result<()> {
|
||||
if self
|
||||
.websocket_read
|
||||
.as_ref()
|
||||
.is_some_and(WebSocketReadBoundary::has_buffered)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
self.stream.readable().await
|
||||
}
|
||||
|
||||
fn enable_websocket(&mut self, buffered: Bytes) {
|
||||
self.websocket_read = Some(WebSocketReadBoundary::new(buffered));
|
||||
}
|
||||
|
||||
fn websocket_fragmented_message(&self) -> bool {
|
||||
self.websocket_read
|
||||
.as_ref()
|
||||
.is_some_and(WebSocketReadBoundary::fragmented_message)
|
||||
}
|
||||
}
|
||||
|
||||
// Frame-boundary reads keep the kernel readiness gate authoritative even when
|
||||
// Hyper read-ahead or one TCP packet contains several WebSocket messages.
|
||||
enum WebSocketReadState {
|
||||
Header {
|
||||
bytes: [u8; 14],
|
||||
filled: usize,
|
||||
target: usize,
|
||||
},
|
||||
Payload {
|
||||
remaining: usize,
|
||||
},
|
||||
}
|
||||
|
||||
struct WebSocketReadBoundary {
|
||||
buffered: Bytes,
|
||||
state: WebSocketReadState,
|
||||
fragmented_message: bool,
|
||||
}
|
||||
|
||||
impl WebSocketReadBoundary {
|
||||
fn new(buffered: Bytes) -> Self {
|
||||
Self {
|
||||
buffered,
|
||||
state: Self::new_header(),
|
||||
fragmented_message: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn has_buffered(&self) -> bool {
|
||||
!self.buffered.is_empty()
|
||||
}
|
||||
|
||||
fn fragmented_message(&self) -> bool {
|
||||
self.fragmented_message
|
||||
}
|
||||
|
||||
fn maximum_read(&self, requested: usize) -> usize {
|
||||
let boundary = match self.state {
|
||||
WebSocketReadState::Header { filled, target, .. } => target.saturating_sub(filled),
|
||||
WebSocketReadState::Payload { remaining } => remaining,
|
||||
};
|
||||
requested.min(boundary)
|
||||
}
|
||||
|
||||
fn observe(&mut self, bytes: &[u8]) {
|
||||
match &mut self.state {
|
||||
WebSocketReadState::Header {
|
||||
bytes: header,
|
||||
filled,
|
||||
target,
|
||||
} => {
|
||||
debug_assert!(bytes.len() <= target.saturating_sub(*filled));
|
||||
header[*filled..*filled + bytes.len()].copy_from_slice(bytes);
|
||||
*filled += bytes.len();
|
||||
if *filled == 2 && *target == 2 {
|
||||
let extended = match header[1] & 0x7f {
|
||||
126 => 2,
|
||||
127 => 8,
|
||||
_ => 0,
|
||||
};
|
||||
let mask = usize::from(header[1] & 0x80 != 0) * 4;
|
||||
*target = 2 + extended + mask;
|
||||
}
|
||||
if *filled == *target {
|
||||
let finished = header[0] & 0x80 != 0;
|
||||
match header[0] & 0x0f {
|
||||
0 if finished => self.fragmented_message = false,
|
||||
1 | 2 if !finished => self.fragmented_message = true,
|
||||
_ => {}
|
||||
}
|
||||
let payload = match header[1] & 0x7f {
|
||||
value @ 0..=125 => usize::from(value),
|
||||
126 => usize::from(u16::from_be_bytes([header[2], header[3]])),
|
||||
127 => usize::try_from(u64::from_be_bytes([
|
||||
header[2], header[3], header[4], header[5], header[6], header[7],
|
||||
header[8], header[9],
|
||||
]))
|
||||
.unwrap_or(usize::MAX),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
self.state = if payload == 0 {
|
||||
Self::new_header()
|
||||
} else {
|
||||
WebSocketReadState::Payload { remaining: payload }
|
||||
};
|
||||
}
|
||||
}
|
||||
WebSocketReadState::Payload { remaining } => {
|
||||
debug_assert!(bytes.len() <= *remaining);
|
||||
*remaining -= bytes.len();
|
||||
if *remaining == 0 {
|
||||
self.state = Self::new_header();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn new_header() -> WebSocketReadState {
|
||||
WebSocketReadState::Header {
|
||||
bytes: [0; 14],
|
||||
filled: 0,
|
||||
target: 2,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for ConnectionIo {
|
||||
fn poll_read(
|
||||
self: Pin<&mut Self>,
|
||||
context: &mut Context<'_>,
|
||||
buffer: &mut ReadBuf<'_>,
|
||||
) -> Poll<std::io::Result<()>> {
|
||||
let this = self.get_mut();
|
||||
let Some(boundary) = this.websocket_read.as_mut() else {
|
||||
return Pin::new(&mut this.stream).poll_read(context, buffer);
|
||||
};
|
||||
let maximum = boundary.maximum_read(buffer.remaining());
|
||||
if maximum == 0 {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
if !boundary.buffered.is_empty() {
|
||||
let bytes = boundary
|
||||
.buffered
|
||||
.split_to(maximum.min(boundary.buffered.len()));
|
||||
boundary.observe(&bytes);
|
||||
buffer.put_slice(&bytes);
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
let unfilled = buffer.initialize_unfilled_to(maximum);
|
||||
let mut limited = ReadBuf::new(unfilled);
|
||||
match Pin::new(&mut this.stream).poll_read(context, &mut limited) {
|
||||
Poll::Ready(Ok(())) => {
|
||||
let filled = limited.filled().len();
|
||||
boundary.observe(limited.filled());
|
||||
drop(limited);
|
||||
buffer.advance(filled);
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for ConnectionIo {
|
||||
fn poll_write(
|
||||
mut self: Pin<&mut Self>,
|
||||
context: &mut Context<'_>,
|
||||
bytes: &[u8],
|
||||
) -> Poll<std::io::Result<usize>> {
|
||||
Pin::new(&mut self.stream).poll_write(context, bytes)
|
||||
}
|
||||
|
||||
fn poll_flush(
|
||||
mut self: Pin<&mut Self>,
|
||||
context: &mut Context<'_>,
|
||||
) -> Poll<std::io::Result<()>> {
|
||||
Pin::new(&mut self.stream).poll_flush(context)
|
||||
}
|
||||
|
||||
fn poll_shutdown(
|
||||
mut self: Pin<&mut Self>,
|
||||
context: &mut Context<'_>,
|
||||
) -> Poll<std::io::Result<()>> {
|
||||
Pin::new(&mut self.stream).poll_shutdown(context)
|
||||
}
|
||||
}
|
||||
|
||||
enum ParsedCarrier {
|
||||
Multiplex,
|
||||
Lane(u32),
|
||||
}
|
||||
|
||||
struct ParsedUpgrade {
|
||||
token_hash: TokenHash,
|
||||
protocol: String,
|
||||
accept: String,
|
||||
carrier: ParsedCarrier,
|
||||
}
|
||||
|
||||
pub(super) async fn handle(
|
||||
mut request: Request<RequestBody>,
|
||||
peer: SocketAddr,
|
||||
client_ip_source: WebClientIpSource,
|
||||
trusted_proxy_cidrs: &[IpNetwork],
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
vhost: Arc<WebRuntimeVhost>,
|
||||
) -> HttpResponse {
|
||||
let Some(parsed) = parse_upgrade(&request) else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
if !request.body_mut().finish_empty() {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
let Some(effective_ip) = client_ip(&request, peer, client_ip_source, trusted_proxy_cidrs)
|
||||
else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
let Ok(session) = runtime.get_session(parsed.token_hash, &vhost.host) else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
let kind = match (parsed.carrier, session.carrier()) {
|
||||
(ParsedCarrier::Multiplex, WebCarrier::Websocket) => WebSocketKind::Multiplex,
|
||||
(ParsedCarrier::Lane(lane_id), WebCarrier::WebsocketLanes) => WebSocketKind::Lane(lane_id),
|
||||
_ => return serve_decoy(request, vhost, true, &runtime).await,
|
||||
};
|
||||
let mut lane_reservation = match kind {
|
||||
WebSocketKind::Multiplex => None,
|
||||
WebSocketKind::Lane(lane_id) => match session.reserve_websocket_lane(lane_id) {
|
||||
Ok(reservation) => Some(reservation),
|
||||
Err(_) => return serve_decoy(request, vhost, true, &runtime).await,
|
||||
},
|
||||
};
|
||||
let timeouts = runtime.active_generation().config().web.timeouts.clone();
|
||||
let connection = match runtime
|
||||
.admit_websocket(
|
||||
session.profile_key(),
|
||||
session.trace_session_id(),
|
||||
effective_ip,
|
||||
kind,
|
||||
BASE_BUDGET_BYTES,
|
||||
Duration::from_secs(timeouts.long_poll_secs),
|
||||
Duration::from_secs(timeouts.websocket_eviction_secs),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(connection) => connection,
|
||||
Err(_) => return serve_decoy(request, vhost, true, &runtime).await,
|
||||
};
|
||||
let trace_context = runtime.trace().websocket_context(
|
||||
&request,
|
||||
peer.ip(),
|
||||
effective_ip,
|
||||
connection.id(),
|
||||
match kind {
|
||||
WebSocketKind::Multiplex => None,
|
||||
WebSocketKind::Lane(lane_id) => Some(lane_id),
|
||||
},
|
||||
|| session.trace_identity(),
|
||||
);
|
||||
if let Some(trace) = request_trace(&request) {
|
||||
trace.set_effective_ip(effective_ip);
|
||||
trace.set_route(TraceRoute::Websocket);
|
||||
trace.bind_identity(session.trace_identity());
|
||||
trace.register_redaction(parsed.protocol.as_bytes());
|
||||
}
|
||||
let on_upgrade = hyper::upgrade::on(&mut request);
|
||||
let protocol = parsed.protocol;
|
||||
let accept = parsed.accept;
|
||||
let driver_runtime = Arc::clone(&runtime);
|
||||
let driver_session = Arc::clone(&session);
|
||||
runtime.spawn_auxiliary(async move {
|
||||
driver::run_upgraded(
|
||||
on_upgrade,
|
||||
driver_runtime,
|
||||
driver_session,
|
||||
connection,
|
||||
lane_reservation.take(),
|
||||
trace_context,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
set_trace_route(&request, TraceRoute::Websocket);
|
||||
let mut response = full_response(StatusCode::SWITCHING_PROTOCOLS, Bytes::new());
|
||||
response.headers_mut().remove(header::CONTENT_LENGTH);
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::CONNECTION, HeaderValue::from_static("Upgrade"));
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::UPGRADE, HeaderValue::from_static("websocket"));
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("sec-websocket-accept"),
|
||||
&accept,
|
||||
);
|
||||
insert_header(
|
||||
&mut response,
|
||||
HeaderName::from_static("sec-websocket-protocol"),
|
||||
&protocol,
|
||||
);
|
||||
response
|
||||
}
|
||||
|
||||
fn parse_upgrade<B>(request: &Request<B>) -> Option<ParsedUpgrade> {
|
||||
if request.method() != Method::GET
|
||||
|| request.uri().query().is_some()
|
||||
|| request.headers().contains_key(header::AUTHORIZATION)
|
||||
|| request.headers().contains_key(header::CONTENT_LENGTH)
|
||||
|| request.headers().contains_key(header::TRANSFER_ENCODING)
|
||||
|| !single_header_eq(request, header::UPGRADE, "websocket")
|
||||
|| !single_header_eq(request, "sec-websocket-version", "13")
|
||||
|| !header_has_token(request, header::CONNECTION, "upgrade")
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let key = single_header(request, "sec-websocket-key")?;
|
||||
let decoded_key = base64::engine::general_purpose::STANDARD.decode(key).ok()?;
|
||||
if decoded_key.len() != 16
|
||||
|| base64::engine::general_purpose::STANDARD.encode(&decoded_key) != key
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let protocol = single_header(request, "sec-websocket-protocol")?;
|
||||
if protocol
|
||||
.bytes()
|
||||
.any(|value| value == b',' || value.is_ascii_whitespace())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let (token, carrier) = if let Some(token) = protocol.strip_prefix("tproxy-v1.") {
|
||||
(token, ParsedCarrier::Multiplex)
|
||||
} else if let Some(lane) = protocol.strip_prefix("tproxy-lane-v1.") {
|
||||
let (token, lane_id) = lane.split_once('.')?;
|
||||
if lane_id.is_empty()
|
||||
|| lane_id.starts_with('+')
|
||||
|| (lane_id.len() > 1 && lane_id.starts_with('0'))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let lane_id = lane_id
|
||||
.parse::<u32>()
|
||||
.ok()
|
||||
.filter(|value| (1..=crate::web::frame::MAX_STREAM_ID).contains(value))?;
|
||||
(token, ParsedCarrier::Lane(lane_id))
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
let raw_token = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
||||
.decode(token)
|
||||
.ok()?;
|
||||
if raw_token.len() != 32
|
||||
|| token.len() != 43
|
||||
|| base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(&raw_token) != token
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let token_hash = Sha256::digest(&raw_token).into();
|
||||
let mut accept = Sha1::new();
|
||||
accept.update(key.as_bytes());
|
||||
accept.update(WEBSOCKET_GUID);
|
||||
Some(ParsedUpgrade {
|
||||
token_hash,
|
||||
protocol: protocol.to_string(),
|
||||
accept: base64::engine::general_purpose::STANDARD.encode(accept.finalize()),
|
||||
carrier,
|
||||
})
|
||||
}
|
||||
|
||||
fn single_header<'a, B>(
|
||||
request: &'a Request<B>,
|
||||
name: impl hyper::header::AsHeaderName,
|
||||
) -> Option<&'a str> {
|
||||
let mut values = request.headers().get_all(name).iter();
|
||||
let value = values.next()?.to_str().ok()?;
|
||||
values.next().is_none().then_some(value)
|
||||
}
|
||||
|
||||
fn single_header_eq<B>(
|
||||
request: &Request<B>,
|
||||
name: impl hyper::header::AsHeaderName,
|
||||
expected: &str,
|
||||
) -> bool {
|
||||
single_header(request, name).is_some_and(|value| value.eq_ignore_ascii_case(expected))
|
||||
}
|
||||
|
||||
fn header_has_token<B>(
|
||||
request: &Request<B>,
|
||||
name: impl hyper::header::AsHeaderName,
|
||||
expected: &str,
|
||||
) -> bool {
|
||||
let mut found = false;
|
||||
for value in request.headers().get_all(name) {
|
||||
let Ok(value) = value.to_str() else {
|
||||
return false;
|
||||
};
|
||||
for token in value.split(',').map(str::trim) {
|
||||
if token.eq_ignore_ascii_case(expected) {
|
||||
if found {
|
||||
return false;
|
||||
}
|
||||
found = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
found
|
||||
}
|
||||
|
||||
// Ordered WebSocket message relay and deadlines are isolated from handshake parsing.
|
||||
mod driver;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,638 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use hyper_util::rt::TokioIo;
|
||||
use tokio_tungstenite::WebSocketStream;
|
||||
use tokio_tungstenite::tungstenite::protocol::{Message, Role, WebSocketConfig};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::ConnectionIo;
|
||||
use crate::web::manager::{
|
||||
ManagerError, WebProcessRuntime, WebSocketBudgetLease, WebSocketConnection,
|
||||
};
|
||||
use crate::web::session::{WebSession, WebSocketLaneReservation};
|
||||
use crate::web::trace::{TraceDirection, TraceWebSocketContext};
|
||||
|
||||
const READ_BUFFER_BYTES: usize = 64 * 1024;
|
||||
const WRITE_BUFFER_BYTES: usize = 64 * 1024;
|
||||
|
||||
pub(super) async fn run_upgraded(
|
||||
on_upgrade: hyper::upgrade::OnUpgrade,
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
session: Arc<WebSession>,
|
||||
connection: WebSocketConnection,
|
||||
mut lane_reservation: Option<WebSocketLaneReservation>,
|
||||
trace: Option<TraceWebSocketContext>,
|
||||
) {
|
||||
let Ok(upgraded) = on_upgrade.await else {
|
||||
return;
|
||||
};
|
||||
let Ok(parts) = upgraded.downcast::<TokioIo<ConnectionIo>>() else {
|
||||
return;
|
||||
};
|
||||
let mut io = parts.io.into_inner();
|
||||
io.enable_websocket(parts.read_buf);
|
||||
let limits = runtime.active_generation().config().web.limits.clone();
|
||||
let config = WebSocketConfig::default()
|
||||
.read_buffer_size(READ_BUFFER_BYTES)
|
||||
.write_buffer_size(WRITE_BUFFER_BYTES)
|
||||
.max_write_buffer_size(
|
||||
WRITE_BUFFER_BYTES
|
||||
.saturating_add(limits.carrier_batch_bytes)
|
||||
.saturating_add(1024),
|
||||
)
|
||||
.max_message_size(Some(limits.carrier_batch_bytes))
|
||||
.max_frame_size(Some(limits.carrier_batch_bytes));
|
||||
let mut socket = WebSocketStream::from_raw_socket(io, Role::Server, Some(config)).await;
|
||||
connection.mark_opened();
|
||||
let cancellation = connection.cancellation();
|
||||
if let Some(reservation) = lane_reservation.as_mut() {
|
||||
let _ = run_lane(
|
||||
&mut socket,
|
||||
&runtime,
|
||||
&session,
|
||||
&connection,
|
||||
reservation,
|
||||
cancellation.clone(),
|
||||
trace.as_ref(),
|
||||
)
|
||||
.await;
|
||||
} else {
|
||||
let _ = run_multiplex(
|
||||
&mut socket,
|
||||
&runtime,
|
||||
&session,
|
||||
&connection,
|
||||
cancellation.clone(),
|
||||
trace.as_ref(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let eviction = Duration::from_secs(
|
||||
runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.timeouts
|
||||
.websocket_eviction_secs,
|
||||
);
|
||||
let _ = tokio::time::timeout(eviction, socket.close(None)).await;
|
||||
if let Some(reservation) = lane_reservation {
|
||||
session.close_websocket_lane(reservation.lane_id());
|
||||
drop(reservation);
|
||||
} else {
|
||||
session.close();
|
||||
}
|
||||
}
|
||||
|
||||
type CarrierSocket = WebSocketStream<ConnectionIo>;
|
||||
|
||||
async fn run_multiplex(
|
||||
socket: &mut CarrierSocket,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
session: &Arc<WebSession>,
|
||||
connection: &WebSocketConnection,
|
||||
cancellation: CancellationToken,
|
||||
trace: Option<&TraceWebSocketContext>,
|
||||
) -> Result<(), ()> {
|
||||
let mut sequence = 1u64;
|
||||
let mut cursor = 0u64;
|
||||
// The lease survives cancelled select branches and control frames interleaved
|
||||
// inside one fragmented data message.
|
||||
let mut read_budget = None;
|
||||
let liveness_interval = connection.liveness_interval();
|
||||
let mut next_ping = Instant::now() + liveness_interval;
|
||||
loop {
|
||||
let down = session.poll_down(cursor);
|
||||
tokio::pin!(down);
|
||||
let event = tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
_ = tokio::time::sleep_until(next_ping.into()) => DriverEvent::Liveness,
|
||||
incoming = read_message(
|
||||
socket,
|
||||
runtime,
|
||||
session.profile_key(),
|
||||
&cancellation,
|
||||
&mut read_budget,
|
||||
) => {
|
||||
DriverEvent::Incoming(incoming?)
|
||||
}
|
||||
down = &mut down => DriverEvent::Down(down.map_err(|_| ())?),
|
||||
};
|
||||
match event {
|
||||
DriverEvent::Incoming((message, _budget)) => match message {
|
||||
Message::Binary(body) => {
|
||||
let started = Instant::now();
|
||||
let result =
|
||||
process_multiplex(runtime, session, sequence, &body, &cancellation).await;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"binary",
|
||||
&body,
|
||||
started,
|
||||
);
|
||||
result?;
|
||||
sequence = sequence.checked_add(1).ok_or(())?;
|
||||
connection.mark_peer_activity();
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
Message::Pong(payload) => {
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"pong",
|
||||
&payload,
|
||||
Instant::now(),
|
||||
);
|
||||
connection.mark_peer_activity();
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
Message::Ping(payload) => {
|
||||
let started = Instant::now();
|
||||
flush(socket, runtime).await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"ping",
|
||||
&payload,
|
||||
started,
|
||||
);
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"pong",
|
||||
&payload,
|
||||
started,
|
||||
);
|
||||
connection.mark_peer_activity();
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
Message::Close(_) => {
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"close",
|
||||
&[],
|
||||
Instant::now(),
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
Message::Text(text) => {
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"text",
|
||||
text.as_bytes(),
|
||||
Instant::now(),
|
||||
);
|
||||
return Err(());
|
||||
}
|
||||
Message::Frame(_) => return Err(()),
|
||||
},
|
||||
DriverEvent::Down(result) => {
|
||||
if result.body.is_empty() {
|
||||
let started = Instant::now();
|
||||
send(socket, runtime, Message::Ping(Bytes::new())).await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"ping",
|
||||
&[],
|
||||
started,
|
||||
);
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
} else {
|
||||
let _budget = reserve_data(
|
||||
runtime,
|
||||
session.profile_key(),
|
||||
result.body.len(),
|
||||
&cancellation,
|
||||
)
|
||||
.await?;
|
||||
let body = result.body;
|
||||
let started = Instant::now();
|
||||
if trace.is_some() {
|
||||
send(socket, runtime, Message::Binary(body.clone())).await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"binary",
|
||||
&body,
|
||||
started,
|
||||
);
|
||||
} else {
|
||||
send(socket, runtime, Message::Binary(body)).await?;
|
||||
}
|
||||
connection.mark_progress();
|
||||
}
|
||||
cursor = result.next_cursor;
|
||||
}
|
||||
DriverEvent::Liveness => {
|
||||
let started = Instant::now();
|
||||
send(socket, runtime, Message::Ping(Bytes::new())).await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"ping",
|
||||
&[],
|
||||
started,
|
||||
);
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_lane(
|
||||
socket: &mut CarrierSocket,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
session: &Arc<WebSession>,
|
||||
connection: &WebSocketConnection,
|
||||
reservation: &mut WebSocketLaneReservation,
|
||||
cancellation: CancellationToken,
|
||||
trace: Option<&TraceWebSocketContext>,
|
||||
) -> Result<(), ()> {
|
||||
let mut sequence = 1u64;
|
||||
let mut cursor = 0u64;
|
||||
// Lane reads use the same cancellation-safe fragmented-message ownership.
|
||||
let mut read_budget = None;
|
||||
let liveness_interval = connection.liveness_interval();
|
||||
let mut next_ping = Instant::now() + liveness_interval;
|
||||
loop {
|
||||
let down = session.poll_down_lane(reservation.lane_id(), cursor);
|
||||
tokio::pin!(down);
|
||||
let event = tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
_ = tokio::time::sleep_until(next_ping.into()) => DriverEvent::Liveness,
|
||||
incoming = read_message(
|
||||
socket,
|
||||
runtime,
|
||||
session.profile_key(),
|
||||
&cancellation,
|
||||
&mut read_budget,
|
||||
) => {
|
||||
DriverEvent::Incoming(incoming?)
|
||||
}
|
||||
down = &mut down => DriverEvent::Down(down.map_err(|_| ())?),
|
||||
};
|
||||
match event {
|
||||
DriverEvent::Incoming((message, _budget)) => match message {
|
||||
Message::Binary(body) => {
|
||||
let started = Instant::now();
|
||||
let result = process_lane(
|
||||
runtime,
|
||||
session,
|
||||
reservation,
|
||||
sequence,
|
||||
&body,
|
||||
&cancellation,
|
||||
)
|
||||
.await;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"binary",
|
||||
&body,
|
||||
started,
|
||||
);
|
||||
result?;
|
||||
sequence = sequence.checked_add(1).ok_or(())?;
|
||||
connection.mark_peer_activity();
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
Message::Pong(payload) => {
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"pong",
|
||||
&payload,
|
||||
Instant::now(),
|
||||
);
|
||||
connection.mark_peer_activity();
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
Message::Ping(payload) => {
|
||||
let started = Instant::now();
|
||||
flush(socket, runtime).await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"ping",
|
||||
&payload,
|
||||
started,
|
||||
);
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"pong",
|
||||
&payload,
|
||||
started,
|
||||
);
|
||||
connection.mark_peer_activity();
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
Message::Close(_) => {
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"close",
|
||||
&[],
|
||||
Instant::now(),
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
Message::Text(text) => {
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Request,
|
||||
"text",
|
||||
text.as_bytes(),
|
||||
Instant::now(),
|
||||
);
|
||||
return Err(());
|
||||
}
|
||||
Message::Frame(_) => return Err(()),
|
||||
},
|
||||
DriverEvent::Down(result) => {
|
||||
if result.lane_closed {
|
||||
return Ok(());
|
||||
}
|
||||
if result.body.is_empty() {
|
||||
let started = Instant::now();
|
||||
send(socket, runtime, Message::Ping(Bytes::new())).await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"ping",
|
||||
&[],
|
||||
started,
|
||||
);
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
} else {
|
||||
let _budget = reserve_data(
|
||||
runtime,
|
||||
session.profile_key(),
|
||||
result.body.len(),
|
||||
&cancellation,
|
||||
)
|
||||
.await?;
|
||||
let body = result.body;
|
||||
let started = Instant::now();
|
||||
if trace.is_some() {
|
||||
send(socket, runtime, Message::Binary(body.clone())).await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"binary",
|
||||
&body,
|
||||
started,
|
||||
);
|
||||
} else {
|
||||
send(socket, runtime, Message::Binary(body)).await?;
|
||||
}
|
||||
connection.mark_progress();
|
||||
}
|
||||
cursor = result.next_cursor;
|
||||
}
|
||||
DriverEvent::Liveness => {
|
||||
let started = Instant::now();
|
||||
send(socket, runtime, Message::Ping(Bytes::new())).await?;
|
||||
record_message(
|
||||
runtime,
|
||||
trace,
|
||||
TraceDirection::Response,
|
||||
"ping",
|
||||
&[],
|
||||
started,
|
||||
);
|
||||
next_ping = Instant::now() + liveness_interval;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum DriverEvent {
|
||||
Incoming((Message, Option<WebSocketBudgetLease>)),
|
||||
Down(crate::web::session::PollResult),
|
||||
Liveness,
|
||||
}
|
||||
|
||||
async fn read_message(
|
||||
socket: &mut CarrierSocket,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
owner: crate::web::manager::ProfileKey,
|
||||
cancellation: &CancellationToken,
|
||||
retained_budget: &mut Option<WebSocketBudgetLease>,
|
||||
) -> Result<(Message, Option<WebSocketBudgetLease>), ()> {
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
ready = socket.get_ref().readable() => ready.map_err(|_| ())?,
|
||||
}
|
||||
if retained_budget.is_none() {
|
||||
let maximum = runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.limits
|
||||
.carrier_batch_bytes;
|
||||
*retained_budget = Some(reserve_data(runtime, owner, maximum, cancellation).await?);
|
||||
}
|
||||
let message = tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
message = socket.next() => message.ok_or(())?.map_err(|_| ())?,
|
||||
};
|
||||
if socket.get_ref().websocket_fragmented_message() {
|
||||
return Ok((message, None));
|
||||
}
|
||||
let mut budget = retained_budget.take().ok_or(())?;
|
||||
budget.shrink_to(message.len());
|
||||
Ok((message, Some(budget)))
|
||||
}
|
||||
|
||||
async fn reserve_data(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
owner: crate::web::manager::ProfileKey,
|
||||
bytes: usize,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<WebSocketBudgetLease, ()> {
|
||||
let timeout = Duration::from_secs(
|
||||
runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.timeouts
|
||||
.websocket_backpressure_secs,
|
||||
);
|
||||
tokio::time::timeout(timeout, async {
|
||||
loop {
|
||||
let notify = runtime.budget_notify();
|
||||
let notified = notify.notified();
|
||||
if let Some(budget) = runtime.try_websocket_data_budget(owner, bytes.max(1)) {
|
||||
return Ok(budget);
|
||||
}
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
_ = notified => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ())?
|
||||
}
|
||||
|
||||
async fn process_multiplex(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
session: &Arc<WebSession>,
|
||||
sequence: u64,
|
||||
body: &[u8],
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), ()> {
|
||||
retry_backpressure(runtime, cancellation, || {
|
||||
session.process_up(sequence, body).map(|_| ())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn process_lane(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
session: &Arc<WebSession>,
|
||||
reservation: &mut WebSocketLaneReservation,
|
||||
sequence: u64,
|
||||
body: &[u8],
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<(), ()> {
|
||||
let timeout = Duration::from_secs(
|
||||
runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.timeouts
|
||||
.websocket_backpressure_secs,
|
||||
);
|
||||
tokio::time::timeout(timeout, async {
|
||||
loop {
|
||||
let notify = runtime.budget_notify();
|
||||
let notified = notify.notified();
|
||||
match session.process_websocket_lane(reservation, sequence, body) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(ManagerError::Backpressure) => {}
|
||||
Err(_) => return Err(()),
|
||||
}
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
_ = notified => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ())?
|
||||
}
|
||||
|
||||
async fn retry_backpressure<F>(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
cancellation: &CancellationToken,
|
||||
mut operation: F,
|
||||
) -> Result<(), ()>
|
||||
where
|
||||
F: FnMut() -> Result<(), ManagerError>,
|
||||
{
|
||||
let timeout = Duration::from_secs(
|
||||
runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.timeouts
|
||||
.websocket_backpressure_secs,
|
||||
);
|
||||
tokio::time::timeout(timeout, async {
|
||||
loop {
|
||||
let notify = runtime.budget_notify();
|
||||
let notified = notify.notified();
|
||||
match operation() {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(ManagerError::Backpressure) => {}
|
||||
Err(_) => return Err(()),
|
||||
}
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
_ = notified => {}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| ())?
|
||||
}
|
||||
|
||||
async fn send(
|
||||
socket: &mut CarrierSocket,
|
||||
runtime: &WebProcessRuntime,
|
||||
message: Message,
|
||||
) -> Result<(), ()> {
|
||||
let timeout = Duration::from_secs(
|
||||
runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.timeouts
|
||||
.websocket_write_secs,
|
||||
);
|
||||
tokio::time::timeout(timeout, socket.send(message))
|
||||
.await
|
||||
.map_err(|_| ())?
|
||||
.map_err(|_| ())
|
||||
}
|
||||
|
||||
async fn flush(socket: &mut CarrierSocket, runtime: &WebProcessRuntime) -> Result<(), ()> {
|
||||
let timeout = Duration::from_secs(
|
||||
runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.timeouts
|
||||
.websocket_write_secs,
|
||||
);
|
||||
tokio::time::timeout(timeout, socket.flush())
|
||||
.await
|
||||
.map_err(|_| ())?
|
||||
.map_err(|_| ())
|
||||
}
|
||||
|
||||
fn record_message(
|
||||
runtime: &WebProcessRuntime,
|
||||
trace: Option<&TraceWebSocketContext>,
|
||||
direction: TraceDirection,
|
||||
message_type: &'static str,
|
||||
payload: &[u8],
|
||||
started: Instant,
|
||||
) {
|
||||
let Some(trace) = trace else {
|
||||
return;
|
||||
};
|
||||
runtime.trace().record_websocket_message(
|
||||
trace,
|
||||
direction,
|
||||
message_type,
|
||||
payload,
|
||||
started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
use super::*;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use sha2::{Digest, Sha256};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
use tokio_tungstenite::WebSocketStream;
|
||||
use tokio_tungstenite::tungstenite::protocol::{Message, Role};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::maestro::generation::{RuntimeGeneration, test_runtime_generation};
|
||||
use crate::web::frame::{self, FrameType};
|
||||
use crate::web::http::tests::runtime_config;
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
|
||||
fn request(protocol: &str) -> Request<()> {
|
||||
Request::builder()
|
||||
.method(Method::GET)
|
||||
.uri("/api/v1/ws")
|
||||
.header(header::CONNECTION, "keep-alive, Upgrade")
|
||||
.header(header::UPGRADE, "websocket")
|
||||
.header("sec-websocket-version", "13")
|
||||
.header("sec-websocket-key", "dGhlIHNhbXBsZSBub25jZQ==")
|
||||
.header("sec-websocket-protocol", protocol)
|
||||
.body(())
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonical_multiplex_and_lane_protocols_are_accepted() {
|
||||
let token = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode([7u8; 32]);
|
||||
let multiplex = parse_upgrade(&request(&format!("tproxy-v1.{token}"))).unwrap();
|
||||
assert!(matches!(multiplex.carrier, ParsedCarrier::Multiplex));
|
||||
assert_eq!(multiplex.accept, "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=");
|
||||
|
||||
let lane = parse_upgrade(&request(&format!("tproxy-lane-v1.{token}.16777215"))).unwrap();
|
||||
assert!(matches!(lane.carrier, ParsedCarrier::Lane(16_777_215)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aliases_authorization_and_request_bodies_are_rejected_before_upgrade() {
|
||||
let token = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode([9u8; 32]);
|
||||
assert!(parse_upgrade(&request(&format!("tproxy-lane-v1.{token}.01"))).is_none());
|
||||
assert!(parse_upgrade(&request(&format!("tproxy-lane-v1.{token}.0"))).is_none());
|
||||
|
||||
let with_authorization = Request::builder()
|
||||
.method(Method::GET)
|
||||
.uri("/api/v1/ws")
|
||||
.header(header::CONNECTION, "Upgrade")
|
||||
.header(header::UPGRADE, "websocket")
|
||||
.header(header::AUTHORIZATION, "Bearer hidden")
|
||||
.header("sec-websocket-version", "13")
|
||||
.header("sec-websocket-key", "dGhlIHNhbXBsZSBub25jZQ==")
|
||||
.header("sec-websocket-protocol", format!("tproxy-v1.{token}"))
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(parse_upgrade(&with_authorization).is_none());
|
||||
|
||||
let with_body = Request::builder()
|
||||
.method(Method::GET)
|
||||
.uri("/api/v1/ws")
|
||||
.header(header::CONNECTION, "Upgrade")
|
||||
.header(header::UPGRADE, "websocket")
|
||||
.header(header::CONTENT_LENGTH, "1")
|
||||
.header("sec-websocket-version", "13")
|
||||
.header("sec-websocket-key", "dGhlIHNhbXBsZSBub25jZQ==")
|
||||
.header("sec-websocket-protocol", format!("tproxy-v1.{token}"))
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(parse_upgrade(&with_body).is_none());
|
||||
}
|
||||
|
||||
struct LiveRuntime {
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
generation: Arc<RuntimeGeneration>,
|
||||
}
|
||||
|
||||
impl LiveRuntime {
|
||||
async fn shutdown(self) {
|
||||
self.runtime.shutdown().await;
|
||||
self.generation.stop_sessions().await;
|
||||
self.generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
|
||||
fn live_runtime(carrier: WebCarrier) -> LiveRuntime {
|
||||
live_runtime_with_long_poll(carrier, 1)
|
||||
}
|
||||
|
||||
fn live_runtime_with_long_poll(carrier: WebCarrier, long_poll_secs: u64) -> LiveRuntime {
|
||||
let mut config = runtime_config([31; 32], carrier);
|
||||
config.web.timeouts.long_poll_secs = long_poll_secs;
|
||||
config.web.timeouts.websocket_write_secs = 2;
|
||||
config.web.timeouts.websocket_backpressure_secs = 2;
|
||||
config.web.timeouts.websocket_eviction_secs = 1;
|
||||
let generation = test_runtime_generation(1, config);
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
LiveRuntime {
|
||||
runtime,
|
||||
generation,
|
||||
}
|
||||
}
|
||||
|
||||
fn create_session(runtime: &Arc<WebProcessRuntime>) -> (String, TokenHash) {
|
||||
let profile = runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
.web
|
||||
.runtime
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.profiles[0]
|
||||
.clone();
|
||||
let client_ip = "192.0.2.10".parse().unwrap();
|
||||
let bootstrap = runtime.issue_bootstrap(profile, client_ip).unwrap().token;
|
||||
let raw = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
||||
.decode(&bootstrap)
|
||||
.unwrap();
|
||||
let bootstrap_hash = Sha256::digest(raw).into();
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
let session = runtime
|
||||
.create_session(bootstrap_hash, "proxy.example.com", client_ip, &hello)
|
||||
.unwrap()
|
||||
.token;
|
||||
let raw = base64::engine::general_purpose::URL_SAFE_NO_PAD
|
||||
.decode(&session)
|
||||
.unwrap();
|
||||
let session_hash = Sha256::digest(raw).into();
|
||||
(session, session_hash)
|
||||
}
|
||||
|
||||
async fn upgrade(
|
||||
listener: &TcpListener,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
protocol: &str,
|
||||
) -> WebSocketStream<TcpStream> {
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (accepted, client) = tokio::join!(listener.accept(), TcpStream::connect(addr));
|
||||
let (server, peer) = accepted.unwrap();
|
||||
let mut client = client.unwrap();
|
||||
let permit = runtime.try_http_connection().unwrap();
|
||||
tokio::spawn(super::super::serve_connection(
|
||||
server,
|
||||
peer,
|
||||
WebClientIpSource::XForwardedFor,
|
||||
Arc::from(["127.0.0.1/32".parse().unwrap()]),
|
||||
Arc::clone(runtime),
|
||||
CancellationToken::new(),
|
||||
permit,
|
||||
));
|
||||
let request = format!(
|
||||
"GET /api/v1/ws HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Protocol: {protocol}\r\nCookie: browser-state=allowed\r\n\r\n"
|
||||
);
|
||||
client.write_all(request.as_bytes()).await.unwrap();
|
||||
let mut response = Vec::new();
|
||||
while !response.ends_with(b"\r\n\r\n") {
|
||||
let byte = client.read_u8().await.unwrap();
|
||||
response.push(byte);
|
||||
assert!(response.len() <= 16 * 1024);
|
||||
}
|
||||
assert!(response.starts_with(b"HTTP/1.1 101"));
|
||||
assert!(
|
||||
std::str::from_utf8(&response)
|
||||
.unwrap()
|
||||
.contains(&format!("sec-websocket-protocol: {protocol}"))
|
||||
);
|
||||
WebSocketStream::from_raw_socket(client, Role::Client, None).await
|
||||
}
|
||||
|
||||
fn masked_message(opcode: u8, payload: &[u8], mask: [u8; 4]) -> Vec<u8> {
|
||||
masked_frame(true, opcode, payload, mask)
|
||||
}
|
||||
|
||||
fn masked_frame(finished: bool, opcode: u8, payload: &[u8], mask: [u8; 4]) -> Vec<u8> {
|
||||
assert!(payload.len() < 126);
|
||||
let mut encoded = Vec::with_capacity(payload.len() + 6);
|
||||
encoded.push(u8::from(finished) << 7 | opcode);
|
||||
encoded.push(0x80 | payload.len() as u8);
|
||||
encoded.extend_from_slice(&mask);
|
||||
encoded.extend(
|
||||
payload
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, value)| value ^ mask[index % mask.len()]),
|
||||
);
|
||||
encoded
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multiplex_upgrade_relays_binary_and_transport_control_messages() {
|
||||
let live = live_runtime(WebCarrier::Websocket);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let (session, _) = create_session(&live.runtime);
|
||||
let protocol = format!("tproxy-v1.{session}");
|
||||
let mut socket = upgrade(&listener, &live.runtime, &protocol).await;
|
||||
|
||||
socket
|
||||
.send(Message::Binary(frame::encode(FrameType::Pong, 0, &[])))
|
||||
.await
|
||||
.unwrap();
|
||||
socket
|
||||
.send(Message::Ping(Bytes::from_static(b"live")))
|
||||
.await
|
||||
.unwrap();
|
||||
let response = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(response, Message::Pong(Bytes::from_static(b"live")));
|
||||
|
||||
let _ = socket.close(None).await;
|
||||
live.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn coalesced_websocket_messages_do_not_wait_for_new_tcp_readiness() {
|
||||
let live = live_runtime_with_long_poll(WebCarrier::Websocket, 10);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let (session, _) = create_session(&live.runtime);
|
||||
let protocol = format!("tproxy-v1.{session}");
|
||||
let mut socket = upgrade(&listener, &live.runtime, &protocol).await;
|
||||
let mut wire = masked_message(0x02, &frame::encode(FrameType::Pong, 0, &[]), [1, 2, 3, 4]);
|
||||
wire.extend_from_slice(&masked_message(0x09, b"coalesced", [5, 6, 7, 8]));
|
||||
socket.get_mut().write_all(&wire).await.unwrap();
|
||||
|
||||
let response = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(response, Message::Pong(Bytes::from_static(b"coalesced")));
|
||||
|
||||
let _ = socket.close(None).await;
|
||||
live.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fragmented_message_budget_survives_interleaved_control_frames() {
|
||||
let live = live_runtime_with_long_poll(WebCarrier::Websocket, 10);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let (session, _) = create_session(&live.runtime);
|
||||
let protocol = format!("tproxy-v1.{session}");
|
||||
let mut socket = upgrade(&listener, &live.runtime, &protocol).await;
|
||||
let body = frame::encode(FrameType::Pong, 0, &[]);
|
||||
let mut wire = masked_frame(false, 0x02, &body[..4], [1, 2, 3, 4]);
|
||||
wire.extend_from_slice(&masked_message(0x09, b"mid", [5, 6, 7, 8]));
|
||||
wire.extend_from_slice(&masked_frame(true, 0x00, &body[4..], [9, 10, 11, 12]));
|
||||
wire.extend_from_slice(&masked_message(0x09, b"after", [13, 14, 15, 16]));
|
||||
socket.get_mut().write_all(&wire).await.unwrap();
|
||||
|
||||
for expected in [b"mid".as_slice(), b"after".as_slice()] {
|
||||
let response = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(response, Message::Pong(Bytes::copy_from_slice(expected)));
|
||||
}
|
||||
|
||||
let _ = socket.close(None).await;
|
||||
live.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_websocket_lane_closes_only_that_lane() {
|
||||
let live = live_runtime(WebCarrier::WebsocketLanes);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let (session, session_hash) = create_session(&live.runtime);
|
||||
let first_protocol = format!("tproxy-lane-v1.{session}.7");
|
||||
let mut first = upgrade(&listener, &live.runtime, &first_protocol).await;
|
||||
first
|
||||
.send(Message::Binary(frame::encode(FrameType::Data, 7, &[1])))
|
||||
.await
|
||||
.unwrap();
|
||||
let closed = tokio::time::timeout(Duration::from_secs(2), first.next())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(matches!(closed, Message::Close(_)));
|
||||
assert!(
|
||||
live.runtime
|
||||
.get_session(session_hash, "proxy.example.com")
|
||||
.is_ok()
|
||||
);
|
||||
|
||||
let second_protocol = format!("tproxy-lane-v1.{session}.8");
|
||||
let mut second = upgrade(&listener, &live.runtime, &second_protocol).await;
|
||||
second
|
||||
.send(Message::Binary(frame::encode(FrameType::Open, 8, &[])))
|
||||
.await
|
||||
.unwrap();
|
||||
second
|
||||
.send(Message::Ping(Bytes::from_static(b"lane")))
|
||||
.await
|
||||
.unwrap();
|
||||
let pong = tokio::time::timeout(Duration::from_secs(2), second.next())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(pong, Message::Pong(Bytes::from_static(b"lane")));
|
||||
|
||||
let _ = second.close(None).await;
|
||||
live.shutdown().await;
|
||||
}
|
||||
+115
-294
@@ -1,34 +1,35 @@
|
||||
use std::future::Future;
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use parking_lot::Mutex;
|
||||
use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq;
|
||||
use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::TaskTracker;
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use crate::config::{WebCarrier, WebLimitsConfig, WebRuntimeProfile};
|
||||
use crate::config::{WebCarrier, WebLimitsConfig};
|
||||
use crate::maestro::generation::RuntimeGeneration;
|
||||
use crate::web::frame;
|
||||
use crate::web::session::WebSession;
|
||||
use crate::web::trace::WebTraceStore;
|
||||
|
||||
// Credential maps, quotas, and token-bucket helpers remain private to the manager.
|
||||
mod state;
|
||||
// Bootstrap credentials and idempotent session creation are isolated from queue accounting.
|
||||
mod credentials;
|
||||
// Stream admission and synthetic tuple ownership are process-scoped.
|
||||
mod admission;
|
||||
// Shutdown and expiry work remain outside request-path coordination.
|
||||
mod lifecycle;
|
||||
use state::{
|
||||
Bootstrap, ManagerState, allow_rate, control_item_reserve, decrement_map,
|
||||
evict_oldest_unused_bootstrap, matching_profile, new_unique_token, profile_key,
|
||||
remove_expired_locked,
|
||||
};
|
||||
// Queue and WebSocket allocations share one process-owned data-plane budget.
|
||||
mod budget;
|
||||
// WebSocket admission, replacement, and liveness are process-scoped.
|
||||
mod websocket;
|
||||
pub(crate) use budget::WebSocketBudgetLease;
|
||||
use budget::{WebDataBudget, WebSocketBudgetClass};
|
||||
use state::ManagerState;
|
||||
pub(crate) use websocket::{WebSocketConnection, WebSocketKind};
|
||||
|
||||
const TOKEN_BYTES: usize = 32;
|
||||
const CLEANUP_INTERVAL: Duration = Duration::from_secs(1);
|
||||
@@ -63,9 +64,18 @@ pub(crate) struct CreateResult {
|
||||
pub(crate) carrier: WebCarrier,
|
||||
}
|
||||
|
||||
/// Successful bridge bootstrap issuance result.
|
||||
pub(crate) struct BootstrapResult {
|
||||
/// Opaque one-use bootstrap credential.
|
||||
pub(crate) token: String,
|
||||
/// Process-unique non-secret trace identifier.
|
||||
pub(crate) trace_session_id: u64,
|
||||
}
|
||||
|
||||
/// Process-owned bounded WEB credential, session, and memory coordinator.
|
||||
pub(crate) struct WebProcessRuntime {
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
trace: Arc<WebTraceStore>,
|
||||
limits: WebLimitsConfig,
|
||||
state: Mutex<ManagerState>,
|
||||
http_connections: Arc<Semaphore>,
|
||||
@@ -74,8 +84,12 @@ pub(crate) struct WebProcessRuntime {
|
||||
body_readers: Arc<Semaphore>,
|
||||
body_bytes: Arc<Semaphore>,
|
||||
stream_handshakes: Arc<Semaphore>,
|
||||
budget_notify: Arc<Notify>,
|
||||
budget_saturated: AtomicBool,
|
||||
websocket_connections: Arc<Semaphore>,
|
||||
websockets: Mutex<websocket::WebSocketRegistry>,
|
||||
websocket_next_id: AtomicU64,
|
||||
websocket_clock: std::time::Instant,
|
||||
websocket_notify: Arc<Notify>,
|
||||
data_budget: Arc<WebDataBudget>,
|
||||
shutdown: CancellationToken,
|
||||
tasks: TaskTracker,
|
||||
sessions_created: AtomicU64,
|
||||
@@ -89,20 +103,39 @@ pub(crate) struct WebProcessRuntime {
|
||||
|
||||
impl WebProcessRuntime {
|
||||
/// Starts one process-scoped manager using immutable allocation ceilings.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn start(active_runtime: Arc<ArcSwap<RuntimeGeneration>>) -> Arc<Self> {
|
||||
let config = active_runtime.load().config();
|
||||
let trace = WebTraceStore::new(config.web.debug.clone(), &config.web.limits);
|
||||
Self::start_with_trace(active_runtime, trace)
|
||||
}
|
||||
|
||||
/// Starts one process-scoped manager with a shared API-visible trace store.
|
||||
pub(crate) fn start_with_trace(
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
trace: Arc<WebTraceStore>,
|
||||
) -> Arc<Self> {
|
||||
let limits = active_runtime.load().config().web.limits.clone();
|
||||
let websocket_connections = limits
|
||||
.max_http_connections
|
||||
.saturating_sub(limits.websocket_http_connection_reserve);
|
||||
let runtime = Arc::new(Self {
|
||||
active_runtime,
|
||||
trace,
|
||||
http_connections: Arc::new(Semaphore::new(limits.max_http_connections)),
|
||||
http_handlers: Arc::new(Semaphore::new(limits.max_http_handlers)),
|
||||
lane_polls: Arc::new(Semaphore::new((limits.max_http_handlers / 2).max(1))),
|
||||
body_readers: Arc::new(Semaphore::new(limits.max_body_readers)),
|
||||
body_bytes: Arc::new(Semaphore::new(limits.max_body_bytes_global)),
|
||||
stream_handshakes: Arc::new(Semaphore::new(limits.max_stream_handshakes)),
|
||||
websocket_connections: Arc::new(Semaphore::new(websocket_connections)),
|
||||
websockets: Mutex::new(websocket::WebSocketRegistry::default()),
|
||||
websocket_next_id: AtomicU64::new(1),
|
||||
websocket_clock: std::time::Instant::now(),
|
||||
websocket_notify: Arc::new(Notify::new()),
|
||||
data_budget: WebDataBudget::new(limits.clone()),
|
||||
limits,
|
||||
state: Mutex::new(ManagerState::default()),
|
||||
budget_notify: Arc::new(Notify::new()),
|
||||
budget_saturated: AtomicBool::new(false),
|
||||
shutdown: CancellationToken::new(),
|
||||
tasks: TaskTracker::new(),
|
||||
sessions_created: AtomicU64::new(0),
|
||||
@@ -125,6 +158,8 @@ impl WebProcessRuntime {
|
||||
let Some(runtime) = weak.upgrade() else {
|
||||
break;
|
||||
};
|
||||
let policy = runtime.active_generation().config().web.debug.clone();
|
||||
runtime.trace.apply_policy(&policy);
|
||||
runtime.cleanup();
|
||||
}
|
||||
}
|
||||
@@ -138,6 +173,11 @@ impl WebProcessRuntime {
|
||||
self.active_runtime.load_full()
|
||||
}
|
||||
|
||||
/// Returns the process-owned WEB debug trace store.
|
||||
pub(crate) fn trace(&self) -> &Arc<WebTraceStore> {
|
||||
&self.trace
|
||||
}
|
||||
|
||||
/// Reserves one accepted HTTP connection.
|
||||
pub(crate) fn try_http_connection(&self) -> Option<OwnedSemaphorePermit> {
|
||||
let permit = Arc::clone(&self.http_connections).try_acquire_owned().ok();
|
||||
@@ -211,302 +251,83 @@ impl WebProcessRuntime {
|
||||
Some((reader, body))
|
||||
}
|
||||
|
||||
/// Issues a one-use bootstrap credential for an active compatible profile.
|
||||
pub(crate) fn issue_bootstrap(
|
||||
&self,
|
||||
profile: Arc<WebRuntimeProfile>,
|
||||
client_ip: IpAddr,
|
||||
) -> std::result::Result<String, ManagerError> {
|
||||
let generation = self.active_generation();
|
||||
let config = generation.config();
|
||||
let profile = config
|
||||
.web
|
||||
.runtime
|
||||
.as_ref()
|
||||
.and_then(|runtime| matching_profile(runtime, &profile))
|
||||
.ok_or(ManagerError::Authentication)?;
|
||||
if !config.web.enabled || !generation.proxy_shared.is_user_enabled(&profile.user) {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let now = Instant::now();
|
||||
let mut state = self.state.lock();
|
||||
remove_expired_locked(&mut state, now);
|
||||
if state.closed
|
||||
|| state
|
||||
.bootstraps_per_ip
|
||||
.get(&client_ip)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
>= self.limits.max_bootstraps_per_ip
|
||||
|| !allow_rate(
|
||||
&mut state.bootstrap_rate,
|
||||
now,
|
||||
self.limits.new_bootstraps_per_minute,
|
||||
self.limits.new_bootstraps_burst,
|
||||
)
|
||||
{
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
if state.bootstraps.len() >= self.limits.max_bootstraps_global
|
||||
&& !evict_oldest_unused_bootstrap(&mut state)
|
||||
{
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
let Some((token, hash)) = new_unique_token(&generation, &state) else {
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
state.bootstraps.insert(
|
||||
hash,
|
||||
Bootstrap {
|
||||
expires_at: now + Duration::from_secs(config.web.timeouts.bootstrap_lifetime_secs),
|
||||
issued_at: now,
|
||||
issuance_ip: client_ip,
|
||||
profile,
|
||||
body_digest: [0; TOKEN_BYTES],
|
||||
session_token: Zeroizing::new(String::new()),
|
||||
session: None,
|
||||
used: false,
|
||||
},
|
||||
);
|
||||
*state.bootstraps_per_ip.entry(client_ip).or_insert(0) += 1;
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
/// Checks whether a bootstrap token is live before reading a request body.
|
||||
pub(crate) fn has_bootstrap(&self, hash: TokenHash, host: &str) -> bool {
|
||||
let now = Instant::now();
|
||||
let state = self.state.lock();
|
||||
state
|
||||
.bootstraps
|
||||
.get(&hash)
|
||||
.is_some_and(|entry| entry.profile.host == host && now <= entry.expires_at)
|
||||
}
|
||||
|
||||
/// Creates a session exactly once or replays the original successful result.
|
||||
pub(crate) fn create_session(
|
||||
self: &Arc<Self>,
|
||||
bootstrap_hash: TokenHash,
|
||||
host: &str,
|
||||
client_ip: IpAddr,
|
||||
body: &[u8],
|
||||
) -> std::result::Result<CreateResult, ManagerError> {
|
||||
if !frame::validate_hello(body, &self.limits) {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let body_digest: TokenHash = Sha256::digest(body).into();
|
||||
let generation = self.active_generation();
|
||||
let config = generation.config();
|
||||
let now = Instant::now();
|
||||
let mut state = self.state.lock();
|
||||
remove_expired_locked(&mut state, now);
|
||||
let Some(entry) = state.bootstraps.get(&bootstrap_hash) else {
|
||||
return Err(ManagerError::Authentication);
|
||||
};
|
||||
if entry.profile.host != host || now > entry.expires_at {
|
||||
return Err(ManagerError::Authentication);
|
||||
}
|
||||
if entry.used {
|
||||
let digest_matches = bool::from(entry.body_digest.ct_eq(&body_digest));
|
||||
if !digest_matches {
|
||||
return Err(ManagerError::Authentication);
|
||||
}
|
||||
let session = entry.session.as_ref().ok_or(ManagerError::Authentication)?;
|
||||
return Ok(CreateResult {
|
||||
token: entry.session_token.as_str().to_owned(),
|
||||
carrier: session.carrier(),
|
||||
});
|
||||
}
|
||||
if state.closed || !config.web.enabled {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let profile = config
|
||||
.web
|
||||
.runtime
|
||||
.as_ref()
|
||||
.and_then(|runtime| matching_profile(runtime, &entry.profile))
|
||||
.filter(|profile| generation.proxy_shared.is_user_enabled(&profile.user))
|
||||
.ok_or(ManagerError::Authentication)?;
|
||||
let profile_key = profile_key(&profile);
|
||||
if state.sessions.len() >= self.limits.max_sessions_global
|
||||
|| state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
|
||||
>= self.limits.max_sessions_per_ip
|
||||
|| state
|
||||
.sessions_per_profile
|
||||
.get(&profile_key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
>= profile.max_sessions
|
||||
|| !allow_rate(
|
||||
&mut state.session_rate,
|
||||
now,
|
||||
self.limits.new_sessions_per_minute,
|
||||
self.limits.new_sessions_burst,
|
||||
)
|
||||
{
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
let session = WebSession::new(
|
||||
Arc::downgrade(self),
|
||||
session_hash,
|
||||
client_ip,
|
||||
profile,
|
||||
profile_key,
|
||||
self.limits.clone(),
|
||||
config.web.timeouts.clone(),
|
||||
);
|
||||
state.sessions.insert(session_hash, Arc::clone(&session));
|
||||
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
|
||||
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
|
||||
let entry = state
|
||||
.bootstraps
|
||||
.get_mut(&bootstrap_hash)
|
||||
.ok_or(ManagerError::Authentication)?;
|
||||
entry.used = true;
|
||||
entry.body_digest = body_digest;
|
||||
entry.session_token = Zeroizing::new(session_token.clone());
|
||||
entry.session = Some(Arc::clone(&session));
|
||||
let issuance_ip = entry.issuance_ip;
|
||||
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
|
||||
self.sessions_created.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(CreateResult {
|
||||
token: session_token,
|
||||
carrier: session.carrier(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolves an authenticated session token.
|
||||
pub(crate) fn get_session(
|
||||
&self,
|
||||
hash: TokenHash,
|
||||
host: &str,
|
||||
) -> std::result::Result<Arc<WebSession>, ManagerError> {
|
||||
self.state
|
||||
.lock()
|
||||
.sessions
|
||||
.get(&hash)
|
||||
.cloned()
|
||||
.filter(|session| session.matches_host(host))
|
||||
.ok_or(ManagerError::Authentication)
|
||||
}
|
||||
|
||||
/// Closes a live token and accepts bounded tombstone retries.
|
||||
pub(crate) fn close_token(
|
||||
&self,
|
||||
hash: TokenHash,
|
||||
host: &str,
|
||||
) -> std::result::Result<(), ManagerError> {
|
||||
let state = self.state.lock();
|
||||
let session = state
|
||||
.sessions
|
||||
.get(&hash)
|
||||
.filter(|session| session.matches_host(host))
|
||||
.cloned();
|
||||
let closed = state
|
||||
.closed_tokens
|
||||
.get(&hash)
|
||||
.is_some_and(|closed| closed.host == host);
|
||||
drop(state);
|
||||
if let Some(session) = session {
|
||||
session.close();
|
||||
return Ok(());
|
||||
}
|
||||
closed.then_some(()).ok_or(ManagerError::Authentication)
|
||||
}
|
||||
|
||||
/// Reserves bounded process-wide queue capacity for data or control traffic.
|
||||
pub(crate) fn try_reserve_pending(
|
||||
&self,
|
||||
owner: ProfileKey,
|
||||
bytes: usize,
|
||||
items: usize,
|
||||
control: bool,
|
||||
downlink: bool,
|
||||
) -> bool {
|
||||
let mut state = self.state.lock();
|
||||
let data_byte_limit = self
|
||||
.limits
|
||||
.pending_bytes_global
|
||||
.saturating_sub(self.limits.control_bytes_global);
|
||||
let control_item_reserve = control_item_reserve(&self.limits);
|
||||
let data_item_limit = self
|
||||
.limits
|
||||
.pending_items_global
|
||||
.saturating_sub(control_item_reserve);
|
||||
if state.closed {
|
||||
return false;
|
||||
}
|
||||
let fits = if control {
|
||||
bytes <= self.limits.control_bytes_global
|
||||
&& items <= control_item_reserve
|
||||
&& state.pending_bytes <= self.limits.pending_bytes_global.saturating_sub(bytes)
|
||||
&& state.pending_items <= self.limits.pending_items_global.saturating_sub(items)
|
||||
&& state.pending_control_bytes
|
||||
<= self.limits.control_bytes_global.saturating_sub(bytes)
|
||||
&& state.pending_control_items <= control_item_reserve.saturating_sub(items)
|
||||
} else {
|
||||
let data_bytes = state
|
||||
.pending_bytes
|
||||
.saturating_sub(state.pending_control_bytes);
|
||||
let data_items = state
|
||||
.pending_items
|
||||
.saturating_sub(state.pending_control_items);
|
||||
let (byte_limit, item_limit) = if downlink {
|
||||
let uplink_bytes = self.limits.max_body_bytes.saturating_add(
|
||||
self.limits
|
||||
.max_frames_per_body
|
||||
.saturating_mul(crate::web::session::QUEUE_ITEM_COST),
|
||||
);
|
||||
(
|
||||
data_byte_limit.saturating_sub(uplink_bytes),
|
||||
data_item_limit.saturating_sub(self.limits.max_frames_per_body),
|
||||
)
|
||||
} else {
|
||||
(data_byte_limit, data_item_limit)
|
||||
};
|
||||
bytes <= byte_limit
|
||||
&& items <= item_limit
|
||||
&& data_bytes <= byte_limit - bytes
|
||||
&& data_items <= item_limit - items
|
||||
};
|
||||
if !fits {
|
||||
self.budget_saturated.store(true, Ordering::Release);
|
||||
if !self
|
||||
.data_budget
|
||||
.try_reserve_queue(owner, bytes, items, control, downlink)
|
||||
{
|
||||
self.record_limit_hit();
|
||||
return false;
|
||||
}
|
||||
state.pending_bytes += bytes;
|
||||
state.pending_items += items;
|
||||
if control {
|
||||
state.pending_control_bytes += bytes;
|
||||
state.pending_control_items += items;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Releases process-wide queue capacity and wakes blocked relay writers.
|
||||
pub(crate) fn release_pending(&self, bytes: usize, items: usize, control: bool) {
|
||||
let mut state = self.state.lock();
|
||||
state.pending_bytes = state.pending_bytes.saturating_sub(bytes);
|
||||
state.pending_items = state.pending_items.saturating_sub(items);
|
||||
if control {
|
||||
state.pending_control_bytes = state.pending_control_bytes.saturating_sub(bytes);
|
||||
state.pending_control_items = state.pending_control_items.saturating_sub(items);
|
||||
}
|
||||
drop(state);
|
||||
if self.budget_saturated.swap(false, Ordering::AcqRel) {
|
||||
self.budget_notify.notify_waiters();
|
||||
}
|
||||
pub(crate) fn release_pending(
|
||||
&self,
|
||||
owner: ProfileKey,
|
||||
bytes: usize,
|
||||
items: usize,
|
||||
control: bool,
|
||||
) {
|
||||
self.data_budget.release_queue(owner, bytes, items, control);
|
||||
}
|
||||
|
||||
/// Returns the shared notification source for global queue capacity changes.
|
||||
pub(crate) fn budget_notify(&self) -> Arc<Notify> {
|
||||
Arc::clone(&self.budget_notify)
|
||||
self.data_budget.notify()
|
||||
}
|
||||
|
||||
/// Reserves fixed WebSocket driver memory below the admission watermark.
|
||||
pub(crate) fn try_websocket_base_budget(
|
||||
&self,
|
||||
owner: ProfileKey,
|
||||
bytes: usize,
|
||||
) -> Option<WebSocketBudgetLease> {
|
||||
self.data_budget
|
||||
.try_reserve_websocket(owner, bytes, WebSocketBudgetClass::Base)
|
||||
}
|
||||
|
||||
/// Reserves one transient WebSocket message below the eviction watermark.
|
||||
pub(crate) fn try_websocket_data_budget(
|
||||
&self,
|
||||
owner: ProfileKey,
|
||||
bytes: usize,
|
||||
) -> Option<WebSocketBudgetLease> {
|
||||
self.data_budget
|
||||
.try_reserve_websocket(owner, bytes, WebSocketBudgetClass::Data)
|
||||
}
|
||||
|
||||
/// Admits one WebSocket with owner-first bounded replacement.
|
||||
pub(crate) async fn admit_websocket(
|
||||
self: &Arc<Self>,
|
||||
owner: ProfileKey,
|
||||
session_id: u64,
|
||||
client_ip: IpAddr,
|
||||
kind: WebSocketKind,
|
||||
base_bytes: usize,
|
||||
liveness_interval: Duration,
|
||||
eviction_timeout: Duration,
|
||||
) -> Result<WebSocketConnection, ManagerError> {
|
||||
websocket::admit(
|
||||
self,
|
||||
owner,
|
||||
session_id,
|
||||
client_ip,
|
||||
kind,
|
||||
base_bytes,
|
||||
liveness_interval,
|
||||
eviction_timeout,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Accounts one successfully committed carrier uplink body.
|
||||
|
||||
@@ -84,7 +84,7 @@ mod tests {
|
||||
async fn global_downlink_budget_preserves_one_maximum_uplink_batch() {
|
||||
let generation = test_runtime_generation(1, ProxyConfig::default());
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(generation)));
|
||||
let control_items = super::super::state::control_item_reserve(&runtime.limits);
|
||||
let control_items = super::super::budget::control_item_reserve(&runtime.limits);
|
||||
let data_bytes = runtime
|
||||
.limits
|
||||
.pending_bytes_global
|
||||
@@ -97,20 +97,30 @@ mod tests {
|
||||
.limits
|
||||
.max_body_bytes
|
||||
.saturating_add(runtime.limits.max_frames_per_body * QUEUE_ITEM_COST);
|
||||
let downlink_bytes = data_bytes - uplink_bytes;
|
||||
let websocket_bytes = runtime.limits.carrier_batch_bytes;
|
||||
let downlink_bytes = data_bytes - uplink_bytes - websocket_bytes;
|
||||
let downlink_items = data_items - runtime.limits.max_frames_per_body;
|
||||
|
||||
assert!(runtime.try_reserve_pending(downlink_bytes, downlink_items, false, true,));
|
||||
assert!(runtime.try_reserve_pending([0; 32], downlink_bytes, downlink_items, false, true,));
|
||||
assert!(runtime.try_reserve_pending(
|
||||
[0; 32],
|
||||
uplink_bytes,
|
||||
runtime.limits.max_frames_per_body,
|
||||
false,
|
||||
false,
|
||||
));
|
||||
assert!(!runtime.try_reserve_pending(1, 1, false, true));
|
||||
let websocket = runtime.try_websocket_data_budget([0; 32], websocket_bytes);
|
||||
assert!(websocket.is_some());
|
||||
assert!(!runtime.try_reserve_pending([0; 32], 1, 1, false, true));
|
||||
|
||||
runtime.release_pending(downlink_bytes, downlink_items, false);
|
||||
runtime.release_pending(uplink_bytes, runtime.limits.max_frames_per_body, false);
|
||||
drop(websocket);
|
||||
runtime.release_pending([0; 32], downlink_bytes, downlink_items, false);
|
||||
runtime.release_pending(
|
||||
[0; 32],
|
||||
uplink_bytes,
|
||||
runtime.limits.max_frames_per_body,
|
||||
false,
|
||||
);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,366 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
use super::ProfileKey;
|
||||
use crate::config::WebLimitsConfig;
|
||||
use crate::web::session::QUEUE_ITEM_COST;
|
||||
|
||||
/// WebSocket allocation class with a distinct pressure watermark.
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) enum WebSocketBudgetClass {
|
||||
/// Long-lived codec and driver memory acquired before an upgrade commits.
|
||||
Base,
|
||||
/// One bounded inbound message or outbound write staging allocation.
|
||||
Data,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct BudgetState {
|
||||
queue_bytes: usize,
|
||||
queue_items: usize,
|
||||
queue_control_bytes: usize,
|
||||
queue_control_items: usize,
|
||||
websocket_bytes: usize,
|
||||
owner_bytes: HashMap<ProfileKey, usize>,
|
||||
high_water_bytes: usize,
|
||||
closed: bool,
|
||||
}
|
||||
|
||||
/// Process-owned byte and item governor shared by queues and WebSocket I/O.
|
||||
pub(super) struct WebDataBudget {
|
||||
limits: WebLimitsConfig,
|
||||
state: Mutex<BudgetState>,
|
||||
notify: Arc<Notify>,
|
||||
pressured: AtomicBool,
|
||||
}
|
||||
|
||||
/// One exact WebSocket allocation released on every cancellation path.
|
||||
pub(crate) struct WebSocketBudgetLease {
|
||||
budget: Arc<WebDataBudget>,
|
||||
owner: ProfileKey,
|
||||
bytes: usize,
|
||||
}
|
||||
|
||||
/// Lock-free diagnostic snapshot of one short locked budget state.
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub(crate) struct WebDataBudgetSnapshot {
|
||||
/// Total queue bytes currently retained.
|
||||
pub(crate) queue_bytes: usize,
|
||||
/// Total queue items currently retained.
|
||||
pub(crate) queue_items: usize,
|
||||
/// Total WebSocket bytes currently retained.
|
||||
pub(crate) websocket_bytes: usize,
|
||||
/// Largest combined byte usage observed since process start.
|
||||
pub(crate) high_water_bytes: usize,
|
||||
}
|
||||
|
||||
impl WebDataBudget {
|
||||
pub(super) fn new(limits: WebLimitsConfig) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
limits,
|
||||
state: Mutex::new(BudgetState::default()),
|
||||
notify: Arc::new(Notify::new()),
|
||||
pressured: AtomicBool::new(false),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn try_reserve_queue(
|
||||
&self,
|
||||
owner: ProfileKey,
|
||||
bytes: usize,
|
||||
items: usize,
|
||||
control: bool,
|
||||
downlink: bool,
|
||||
) -> bool {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed {
|
||||
return false;
|
||||
}
|
||||
let control_item_reserve = control_item_reserve(&self.limits);
|
||||
let data_byte_limit = self
|
||||
.limits
|
||||
.pending_bytes_global
|
||||
.saturating_sub(self.limits.control_bytes_global);
|
||||
let data_item_limit = self
|
||||
.limits
|
||||
.pending_items_global
|
||||
.saturating_sub(control_item_reserve);
|
||||
let (fits, websocket_byte_pressure) = if control {
|
||||
let byte_pressure = state.websocket_bytes != 0
|
||||
&& state.queue_bytes.saturating_add(state.websocket_bytes)
|
||||
> self.limits.pending_bytes_global.saturating_sub(bytes);
|
||||
let fits = bytes <= self.limits.control_bytes_global
|
||||
&& items <= control_item_reserve
|
||||
&& state.queue_bytes.saturating_add(state.websocket_bytes)
|
||||
<= self.limits.pending_bytes_global.saturating_sub(bytes)
|
||||
&& state.queue_items <= self.limits.pending_items_global.saturating_sub(items)
|
||||
&& state.queue_control_bytes
|
||||
<= self.limits.control_bytes_global.saturating_sub(bytes)
|
||||
&& state.queue_control_items <= control_item_reserve.saturating_sub(items);
|
||||
(fits, byte_pressure)
|
||||
} else {
|
||||
let queue_data_bytes = state.queue_bytes.saturating_sub(state.queue_control_bytes);
|
||||
let queue_data_items = state.queue_items.saturating_sub(state.queue_control_items);
|
||||
let (byte_limit, item_limit) = if downlink {
|
||||
let uplink_bytes = self.limits.max_body_bytes.saturating_add(
|
||||
self.limits
|
||||
.max_frames_per_body
|
||||
.saturating_mul(QUEUE_ITEM_COST),
|
||||
);
|
||||
(
|
||||
data_byte_limit
|
||||
.saturating_sub(uplink_bytes)
|
||||
.saturating_sub(self.limits.carrier_batch_bytes),
|
||||
data_item_limit.saturating_sub(self.limits.max_frames_per_body),
|
||||
)
|
||||
} else {
|
||||
(data_byte_limit, data_item_limit)
|
||||
};
|
||||
let byte_pressure = state.websocket_bytes != 0
|
||||
&& queue_data_bytes.saturating_add(state.websocket_bytes)
|
||||
> byte_limit.saturating_sub(bytes);
|
||||
let fits = bytes <= byte_limit
|
||||
&& items <= item_limit
|
||||
&& queue_data_bytes.saturating_add(state.websocket_bytes)
|
||||
<= byte_limit.saturating_sub(bytes)
|
||||
&& queue_data_items <= item_limit.saturating_sub(items);
|
||||
(fits, byte_pressure)
|
||||
};
|
||||
if !fits {
|
||||
if websocket_byte_pressure {
|
||||
self.pressured.store(true, Ordering::Release);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
state.queue_bytes += bytes;
|
||||
state.queue_items += items;
|
||||
if control {
|
||||
state.queue_control_bytes += bytes;
|
||||
state.queue_control_items += items;
|
||||
}
|
||||
add_owner(&mut state.owner_bytes, owner, bytes);
|
||||
update_high_water(&mut state);
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) fn release_queue(
|
||||
&self,
|
||||
owner: ProfileKey,
|
||||
bytes: usize,
|
||||
items: usize,
|
||||
control: bool,
|
||||
) {
|
||||
let mut state = self.state.lock();
|
||||
state.queue_bytes = state.queue_bytes.saturating_sub(bytes);
|
||||
state.queue_items = state.queue_items.saturating_sub(items);
|
||||
if control {
|
||||
state.queue_control_bytes = state.queue_control_bytes.saturating_sub(bytes);
|
||||
state.queue_control_items = state.queue_control_items.saturating_sub(items);
|
||||
}
|
||||
remove_owner(&mut state.owner_bytes, owner, bytes);
|
||||
drop(state);
|
||||
self.pressured.store(false, Ordering::Release);
|
||||
self.notify.notify_waiters();
|
||||
}
|
||||
|
||||
pub(super) fn try_reserve_websocket(
|
||||
self: &Arc<Self>,
|
||||
owner: ProfileKey,
|
||||
bytes: usize,
|
||||
class: WebSocketBudgetClass,
|
||||
) -> Option<WebSocketBudgetLease> {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed || bytes == 0 {
|
||||
return None;
|
||||
}
|
||||
let websocket_limit = match class {
|
||||
WebSocketBudgetClass::Base => watermark(
|
||||
self.limits.websocket_bytes_global,
|
||||
self.limits.websocket_admission_watermark_pct,
|
||||
),
|
||||
WebSocketBudgetClass::Data => watermark(
|
||||
self.limits.websocket_bytes_global,
|
||||
self.limits.websocket_eviction_watermark_pct,
|
||||
),
|
||||
};
|
||||
let data_byte_limit = self
|
||||
.limits
|
||||
.pending_bytes_global
|
||||
.saturating_sub(self.limits.control_bytes_global);
|
||||
let queue_data_bytes = state.queue_bytes.saturating_sub(state.queue_control_bytes);
|
||||
if state.websocket_bytes > websocket_limit.saturating_sub(bytes)
|
||||
|| queue_data_bytes.saturating_add(state.websocket_bytes)
|
||||
> data_byte_limit.saturating_sub(bytes)
|
||||
{
|
||||
self.pressured.store(true, Ordering::Release);
|
||||
return None;
|
||||
}
|
||||
state.websocket_bytes += bytes;
|
||||
add_owner(&mut state.owner_bytes, owner, bytes);
|
||||
update_high_water(&mut state);
|
||||
Some(WebSocketBudgetLease {
|
||||
budget: Arc::clone(self),
|
||||
owner,
|
||||
bytes,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn notify(&self) -> Arc<Notify> {
|
||||
Arc::clone(&self.notify)
|
||||
}
|
||||
|
||||
pub(super) fn take_pressure(&self) -> bool {
|
||||
self.pressured.swap(false, Ordering::AcqRel)
|
||||
}
|
||||
|
||||
pub(super) fn owner_usage(&self, owner: ProfileKey) -> usize {
|
||||
self.state
|
||||
.lock()
|
||||
.owner_bytes
|
||||
.get(&owner)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
pub(super) fn fair_share(&self, additional_owner: Option<ProfileKey>) -> usize {
|
||||
let state = self.state.lock();
|
||||
let mut owners = state.owner_bytes.len();
|
||||
if additional_owner.is_some_and(|owner| !state.owner_bytes.contains_key(&owner)) {
|
||||
owners += 1;
|
||||
}
|
||||
let admission = watermark(
|
||||
self.limits.websocket_bytes_global,
|
||||
self.limits.websocket_admission_watermark_pct,
|
||||
);
|
||||
admission / owners.max(1)
|
||||
}
|
||||
|
||||
pub(super) fn snapshot(&self) -> WebDataBudgetSnapshot {
|
||||
let state = self.state.lock();
|
||||
WebDataBudgetSnapshot {
|
||||
queue_bytes: state.queue_bytes,
|
||||
queue_items: state.queue_items,
|
||||
websocket_bytes: state.websocket_bytes,
|
||||
high_water_bytes: state.high_water_bytes,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn close(&self) {
|
||||
self.state.lock().closed = true;
|
||||
self.notify.notify_waiters();
|
||||
}
|
||||
|
||||
fn release_websocket(&self, owner: ProfileKey, bytes: usize) {
|
||||
let mut state = self.state.lock();
|
||||
state.websocket_bytes = state.websocket_bytes.saturating_sub(bytes);
|
||||
remove_owner(&mut state.owner_bytes, owner, bytes);
|
||||
drop(state);
|
||||
self.pressured.store(false, Ordering::Release);
|
||||
self.notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
impl WebSocketBudgetLease {
|
||||
/// Releases unused worst-case capacity after one message is assembled.
|
||||
pub(crate) fn shrink_to(&mut self, bytes: usize) {
|
||||
let bytes = bytes.min(self.bytes);
|
||||
let released = self.bytes - bytes;
|
||||
if released == 0 {
|
||||
return;
|
||||
}
|
||||
self.bytes = bytes;
|
||||
self.budget.release_websocket(self.owner, released);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WebSocketBudgetLease {
|
||||
fn drop(&mut self) {
|
||||
self.budget.release_websocket(self.owner, self.bytes);
|
||||
}
|
||||
}
|
||||
|
||||
fn watermark(limit: usize, percentage: u8) -> usize {
|
||||
limit.saturating_mul(usize::from(percentage)) / 100
|
||||
}
|
||||
|
||||
pub(super) fn control_item_reserve(limits: &WebLimitsConfig) -> usize {
|
||||
limits
|
||||
.max_sessions_global
|
||||
.saturating_mul(16usize.saturating_add(limits.max_streams_per_session.saturating_mul(3)))
|
||||
}
|
||||
|
||||
fn add_owner(values: &mut HashMap<ProfileKey, usize>, owner: ProfileKey, bytes: usize) {
|
||||
*values.entry(owner).or_insert(0) += bytes;
|
||||
}
|
||||
|
||||
fn remove_owner(values: &mut HashMap<ProfileKey, usize>, owner: ProfileKey, bytes: usize) {
|
||||
let remove = if let Some(value) = values.get_mut(&owner) {
|
||||
*value = value.saturating_sub(bytes);
|
||||
*value == 0
|
||||
} else {
|
||||
false
|
||||
};
|
||||
if remove {
|
||||
values.remove(&owner);
|
||||
}
|
||||
}
|
||||
|
||||
fn update_high_water(state: &mut BudgetState) {
|
||||
state.high_water_bytes = state
|
||||
.high_water_bytes
|
||||
.max(state.queue_bytes.saturating_add(state.websocket_bytes));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn downlink_reservation_preserves_one_uplink_and_websocket_batch() {
|
||||
let limits = WebLimitsConfig::default();
|
||||
let uplink_bytes = limits
|
||||
.max_body_bytes
|
||||
.saturating_add(limits.max_frames_per_body.saturating_mul(QUEUE_ITEM_COST));
|
||||
let downlink_bytes = limits
|
||||
.pending_bytes_global
|
||||
.saturating_sub(limits.control_bytes_global)
|
||||
.saturating_sub(uplink_bytes)
|
||||
.saturating_sub(limits.carrier_batch_bytes);
|
||||
let budget = WebDataBudget::new(limits);
|
||||
|
||||
assert!(budget.try_reserve_queue([1; 32], downlink_bytes, 1, false, true));
|
||||
assert!(!budget.try_reserve_queue([1; 32], 1, 1, false, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn item_limit_rejection_does_not_request_websocket_eviction() {
|
||||
let limits = WebLimitsConfig::default();
|
||||
let rejected_items = limits.pending_items_global.saturating_add(1);
|
||||
let budget = WebDataBudget::new(limits);
|
||||
let _websocket = budget
|
||||
.try_reserve_websocket([1; 32], 1, WebSocketBudgetClass::Data)
|
||||
.unwrap();
|
||||
|
||||
assert!(!budget.try_reserve_queue([2; 32], 1, rejected_items, false, false));
|
||||
assert!(!budget.take_pressure());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn websocket_byte_conflict_requests_pressure_eviction() {
|
||||
let limits = WebLimitsConfig::default();
|
||||
let data_bytes = limits
|
||||
.pending_bytes_global
|
||||
.saturating_sub(limits.control_bytes_global);
|
||||
let budget = WebDataBudget::new(limits);
|
||||
let _websocket = budget
|
||||
.try_reserve_websocket([1; 32], 1, WebSocketBudgetClass::Data)
|
||||
.unwrap();
|
||||
|
||||
assert!(!budget.try_reserve_queue([2; 32], data_bytes, 1, false, false));
|
||||
assert!(budget.take_pressure());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
use subtle::ConstantTimeEq;
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::state::{
|
||||
Bootstrap, allow_rate, decrement_map, evict_oldest_unused_bootstrap, matching_profile,
|
||||
new_unique_token, profile_key, remove_expired_locked,
|
||||
};
|
||||
use super::{
|
||||
BootstrapResult, CreateResult, ManagerError, TOKEN_BYTES, TokenHash, WebProcessRuntime,
|
||||
};
|
||||
use crate::config::WebRuntimeProfile;
|
||||
use crate::web::frame;
|
||||
use crate::web::session::WebSession;
|
||||
|
||||
impl WebProcessRuntime {
|
||||
/// Issues a one-use bootstrap credential for an active compatible profile.
|
||||
pub(crate) fn issue_bootstrap(
|
||||
&self,
|
||||
profile: Arc<WebRuntimeProfile>,
|
||||
client_ip: IpAddr,
|
||||
) -> std::result::Result<BootstrapResult, ManagerError> {
|
||||
let generation = self.active_generation();
|
||||
let config = generation.config();
|
||||
let profile = config
|
||||
.web
|
||||
.runtime
|
||||
.as_ref()
|
||||
.and_then(|runtime| matching_profile(runtime, &profile))
|
||||
.ok_or(ManagerError::Authentication)?;
|
||||
if !config.web.enabled || !generation.proxy_shared.is_user_enabled(&profile.user) {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let now = Instant::now();
|
||||
let mut state = self.state.lock();
|
||||
remove_expired_locked(&mut state, now);
|
||||
if state.closed
|
||||
|| state
|
||||
.bootstraps_per_ip
|
||||
.get(&client_ip)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
>= self.limits.max_bootstraps_per_ip
|
||||
|| !allow_rate(
|
||||
&mut state.bootstrap_rate,
|
||||
now,
|
||||
self.limits.new_bootstraps_per_minute,
|
||||
self.limits.new_bootstraps_burst,
|
||||
)
|
||||
{
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
if state.bootstraps.len() >= self.limits.max_bootstraps_global
|
||||
&& !evict_oldest_unused_bootstrap(&mut state)
|
||||
{
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
let Some((token, hash)) = new_unique_token(&generation, &state) else {
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
let trace_session_id = self.trace.next_session_id();
|
||||
state.bootstraps.insert(
|
||||
hash,
|
||||
Bootstrap {
|
||||
expires_at: now + Duration::from_secs(config.web.timeouts.bootstrap_lifetime_secs),
|
||||
issued_at: now,
|
||||
issuance_ip: client_ip,
|
||||
profile,
|
||||
trace_session_id,
|
||||
body_digest: [0; TOKEN_BYTES],
|
||||
session_token: Zeroizing::new(String::new()),
|
||||
session: None,
|
||||
used: false,
|
||||
},
|
||||
);
|
||||
*state.bootstraps_per_ip.entry(client_ip).or_insert(0) += 1;
|
||||
let profile = state
|
||||
.bootstraps
|
||||
.get(&hash)
|
||||
.map(|entry| Arc::clone(&entry.profile))
|
||||
.ok_or(ManagerError::Closed)?;
|
||||
drop(state);
|
||||
self.trace.record_profile_lifecycle(
|
||||
client_ip,
|
||||
Some(trace_session_id),
|
||||
&profile,
|
||||
crate::web::trace::TraceLifecycleEvent::BridgeIssued,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
Ok(BootstrapResult {
|
||||
token,
|
||||
trace_session_id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolves non-secret bootstrap trace identity without exposing its credential.
|
||||
pub(crate) fn bootstrap_trace_identity(
|
||||
&self,
|
||||
hash: TokenHash,
|
||||
host: &str,
|
||||
) -> Option<(u64, Arc<WebRuntimeProfile>)> {
|
||||
let now = Instant::now();
|
||||
self.state
|
||||
.lock()
|
||||
.bootstraps
|
||||
.get(&hash)
|
||||
.filter(|entry| entry.profile.host == host && now <= entry.expires_at)
|
||||
.map(|entry| (entry.trace_session_id, Arc::clone(&entry.profile)))
|
||||
}
|
||||
|
||||
/// Creates a session exactly once or replays the original successful result.
|
||||
pub(crate) fn create_session(
|
||||
self: &Arc<Self>,
|
||||
bootstrap_hash: TokenHash,
|
||||
host: &str,
|
||||
client_ip: IpAddr,
|
||||
body: &[u8],
|
||||
) -> std::result::Result<CreateResult, ManagerError> {
|
||||
if !frame::validate_hello(body, &self.limits) {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let body_digest: TokenHash = Sha256::digest(body).into();
|
||||
let generation = self.active_generation();
|
||||
let config = generation.config();
|
||||
let now = Instant::now();
|
||||
let mut state = self.state.lock();
|
||||
remove_expired_locked(&mut state, now);
|
||||
let Some(entry) = state.bootstraps.get(&bootstrap_hash) else {
|
||||
return Err(ManagerError::Authentication);
|
||||
};
|
||||
if entry.profile.host != host || now > entry.expires_at {
|
||||
return Err(ManagerError::Authentication);
|
||||
}
|
||||
if entry.used {
|
||||
let digest_matches = bool::from(entry.body_digest.ct_eq(&body_digest));
|
||||
if !digest_matches {
|
||||
return Err(ManagerError::Authentication);
|
||||
}
|
||||
let session = entry.session.as_ref().ok_or(ManagerError::Authentication)?;
|
||||
let result = CreateResult {
|
||||
token: entry.session_token.as_str().to_owned(),
|
||||
carrier: session.carrier(),
|
||||
};
|
||||
let identity = session.trace_identity();
|
||||
drop(state);
|
||||
self.trace.record_lifecycle(
|
||||
None,
|
||||
Some(client_ip),
|
||||
identity,
|
||||
crate::web::trace::TraceLifecycleEvent::SessionReplayed,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
return Ok(result);
|
||||
}
|
||||
let trace_session_id = entry.trace_session_id;
|
||||
let issued_profile = Arc::clone(&entry.profile);
|
||||
if state.closed || !config.web.enabled {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let profile = config
|
||||
.web
|
||||
.runtime
|
||||
.as_ref()
|
||||
.and_then(|runtime| matching_profile(runtime, &issued_profile))
|
||||
.filter(|profile| generation.proxy_shared.is_user_enabled(&profile.user))
|
||||
.ok_or(ManagerError::Authentication)?;
|
||||
let profile_key = profile_key(&profile);
|
||||
if state.sessions.len() >= self.limits.max_sessions_global
|
||||
|| state.sessions_per_ip.get(&client_ip).copied().unwrap_or(0)
|
||||
>= self.limits.max_sessions_per_ip
|
||||
|| state
|
||||
.sessions_per_profile
|
||||
.get(&profile_key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
>= profile.max_sessions
|
||||
|| !allow_rate(
|
||||
&mut state.session_rate,
|
||||
now,
|
||||
self.limits.new_sessions_per_minute,
|
||||
self.limits.new_sessions_burst,
|
||||
)
|
||||
{
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
let session = WebSession::new(
|
||||
Arc::downgrade(self),
|
||||
session_hash,
|
||||
client_ip,
|
||||
trace_session_id,
|
||||
profile,
|
||||
profile_key,
|
||||
self.limits.clone(),
|
||||
config.web.timeouts.clone(),
|
||||
);
|
||||
state.sessions.insert(session_hash, Arc::clone(&session));
|
||||
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
|
||||
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
|
||||
let entry = state
|
||||
.bootstraps
|
||||
.get_mut(&bootstrap_hash)
|
||||
.ok_or(ManagerError::Authentication)?;
|
||||
entry.used = true;
|
||||
entry.body_digest = body_digest;
|
||||
entry.session_token = Zeroizing::new(session_token.clone());
|
||||
entry.session = Some(Arc::clone(&session));
|
||||
let issuance_ip = entry.issuance_ip;
|
||||
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
|
||||
self.sessions_created.fetch_add(1, Ordering::Relaxed);
|
||||
let identity = session.trace_identity();
|
||||
let result = CreateResult {
|
||||
token: session_token,
|
||||
carrier: session.carrier(),
|
||||
};
|
||||
drop(state);
|
||||
self.trace.record_lifecycle(
|
||||
None,
|
||||
Some(client_ip),
|
||||
identity,
|
||||
crate::web::trace::TraceLifecycleEvent::SessionCreated,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Resolves an authenticated session token.
|
||||
pub(crate) fn get_session(
|
||||
&self,
|
||||
hash: TokenHash,
|
||||
host: &str,
|
||||
) -> std::result::Result<Arc<WebSession>, ManagerError> {
|
||||
self.state
|
||||
.lock()
|
||||
.sessions
|
||||
.get(&hash)
|
||||
.cloned()
|
||||
.filter(|session| session.matches_host(host))
|
||||
.ok_or(ManagerError::Authentication)
|
||||
}
|
||||
|
||||
/// Closes a live token and accepts bounded tombstone retries.
|
||||
pub(crate) fn close_token(
|
||||
&self,
|
||||
hash: TokenHash,
|
||||
host: &str,
|
||||
) -> std::result::Result<(), ManagerError> {
|
||||
let state = self.state.lock();
|
||||
let session = state
|
||||
.sessions
|
||||
.get(&hash)
|
||||
.filter(|session| session.matches_host(host))
|
||||
.cloned();
|
||||
let closed = state
|
||||
.closed_tokens
|
||||
.get(&hash)
|
||||
.is_some_and(|closed| closed.host == host);
|
||||
drop(state);
|
||||
if let Some(session) = session {
|
||||
session.close();
|
||||
return Ok(());
|
||||
}
|
||||
closed.then_some(()).ok_or(ManagerError::Authentication)
|
||||
}
|
||||
}
|
||||
@@ -69,6 +69,8 @@ impl WebProcessRuntime {
|
||||
/// Stops issuance, closes all sessions, and joins bounded child work.
|
||||
pub(crate) async fn shutdown(&self) {
|
||||
self.shutdown.cancel();
|
||||
self.close_websockets();
|
||||
self.data_budget.close();
|
||||
let sessions = {
|
||||
let mut state = self.state.lock();
|
||||
state.closed = true;
|
||||
@@ -94,15 +96,11 @@ impl WebProcessRuntime {
|
||||
let _ = tokio::time::timeout(Duration::from_secs(timeout_secs), waits).await;
|
||||
self.tasks.close();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(timeout_secs), self.tasks.wait()).await;
|
||||
let (sessions_live, streams_live, pending_bytes, pending_items) = {
|
||||
let (sessions_live, streams_live) = {
|
||||
let state = self.state.lock();
|
||||
(
|
||||
state.sessions.len(),
|
||||
state.streams_live,
|
||||
state.pending_bytes,
|
||||
state.pending_items,
|
||||
)
|
||||
(state.sessions.len(), state.streams_live)
|
||||
};
|
||||
let budget = self.data_budget.snapshot();
|
||||
info!(
|
||||
target: "telemt::web",
|
||||
sessions_created = self.sessions_created.load(Ordering::Relaxed),
|
||||
@@ -111,8 +109,10 @@ impl WebProcessRuntime {
|
||||
streams_opened = self.streams_opened.load(Ordering::Relaxed),
|
||||
streams_rejected = self.streams_rejected.load(Ordering::Relaxed),
|
||||
streams_live,
|
||||
pending_bytes,
|
||||
pending_items,
|
||||
pending_bytes = budget.queue_bytes,
|
||||
pending_items = budget.queue_items,
|
||||
websocket_bytes = budget.websocket_bytes,
|
||||
data_high_water_bytes = budget.high_water_bytes,
|
||||
bytes_up = self.bytes_up.load(Ordering::Relaxed),
|
||||
bytes_down = self.bytes_down.load(Ordering::Relaxed),
|
||||
limit_hits = self.limit_hits.load(Ordering::Relaxed),
|
||||
@@ -122,6 +122,7 @@ impl WebProcessRuntime {
|
||||
|
||||
/// Expires credentials and closes idle sessions without holding locks across callbacks.
|
||||
pub(super) fn cleanup(&self) {
|
||||
self.cleanup_websockets();
|
||||
let now = Instant::now();
|
||||
let sessions = {
|
||||
let mut state = self.state.lock();
|
||||
|
||||
+18
-18
@@ -8,10 +8,12 @@ use sha2::{Digest, Sha256};
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::{ProfileKey, TOKEN_BYTES, TokenHash};
|
||||
use crate::config::{WebLimitsConfig, WebRuntimeConfig, WebRuntimeProfile};
|
||||
use crate::config::{WebRuntimeConfig, WebRuntimeProfile};
|
||||
use crate::maestro::generation::RuntimeGeneration;
|
||||
use crate::web::session::WebSession;
|
||||
|
||||
const WEB_PROFILE_OWNER_CONTEXT: &[u8] = b"telemt-web-profile-owner-v1\0";
|
||||
|
||||
/// One issued bootstrap and optional idempotent session-creation replay state.
|
||||
pub(super) struct Bootstrap {
|
||||
/// Credential and replay-state expiry deadline.
|
||||
@@ -22,6 +24,8 @@ pub(super) struct Bootstrap {
|
||||
pub(super) issuance_ip: IpAddr,
|
||||
/// Immutable profile selected during capability validation.
|
||||
pub(super) profile: Arc<WebRuntimeProfile>,
|
||||
/// Process-unique non-secret identifier shared by bootstrap and session traces.
|
||||
pub(super) trace_session_id: u64,
|
||||
/// Digest of the accepted HELLO body for idempotent retry matching.
|
||||
pub(super) body_digest: TokenHash,
|
||||
/// Zeroizing copy returned only for an exact session-creation retry.
|
||||
@@ -72,14 +76,6 @@ pub(super) struct ManagerState {
|
||||
/// Process-wide live relay-task count.
|
||||
pub(super) streams_live: usize,
|
||||
stream_ports: HashMap<(IpAddr, SocketAddr), StreamPortState>,
|
||||
/// Total process-wide queued byte reservation.
|
||||
pub(super) pending_bytes: usize,
|
||||
/// Total process-wide queued item reservation.
|
||||
pub(super) pending_items: usize,
|
||||
/// Portion of queued bytes charged to the control reserve.
|
||||
pub(super) pending_control_bytes: usize,
|
||||
/// Portion of queued items charged to the control reserve.
|
||||
pub(super) pending_control_items: usize,
|
||||
/// Bootstrap issuance rate limiter.
|
||||
pub(super) bootstrap_rate: RateState,
|
||||
/// Session creation rate limiter.
|
||||
@@ -110,9 +106,19 @@ pub(super) fn new_unique_token(
|
||||
None
|
||||
}
|
||||
|
||||
/// Returns the precomputed capability as the stable process profile key.
|
||||
/// Derives a secret-independent quota owner stable across capability rotation.
|
||||
pub(super) fn profile_key(profile: &WebRuntimeProfile) -> ProfileKey {
|
||||
profile.capability
|
||||
let mut digest = Sha256::new();
|
||||
digest.update(WEB_PROFILE_OWNER_CONTEXT);
|
||||
digest.update((profile.host.len() as u64).to_be_bytes());
|
||||
digest.update(profile.host.as_bytes());
|
||||
digest.update((profile.user.len() as u64).to_be_bytes());
|
||||
digest.update(profile.user.as_bytes());
|
||||
digest.update([match profile.secret_mode {
|
||||
crate::config::WebSecretMode::Plain => 0,
|
||||
crate::config::WebSecretMode::Dd => 1,
|
||||
}]);
|
||||
digest.finalize().into()
|
||||
}
|
||||
|
||||
/// Re-resolves an issued profile against the active generation without weakening identity.
|
||||
@@ -130,6 +136,7 @@ pub(super) fn matching_profile(
|
||||
&& profile.secret_mode == expected.secret_mode
|
||||
&& profile.carrier == expected.carrier
|
||||
&& profile.capability == expected.capability
|
||||
&& profile.key_fingerprint == expected.key_fingerprint
|
||||
})
|
||||
.cloned()
|
||||
}
|
||||
@@ -208,13 +215,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes the process-wide item reserve required for session control progress.
|
||||
pub(super) fn control_item_reserve(limits: &WebLimitsConfig) -> usize {
|
||||
limits
|
||||
.max_sessions_global
|
||||
.saturating_mul(16usize.saturating_add(limits.max_streams_per_session.saturating_mul(3)))
|
||||
}
|
||||
|
||||
/// Allocates a non-zero source port unique among live streams for one KDF route.
|
||||
pub(super) fn allocate_stream_port(
|
||||
state: &mut ManagerState,
|
||||
|
||||
@@ -0,0 +1,313 @@
|
||||
use std::collections::HashMap;
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::OwnedSemaphorePermit;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::{ManagerError, ProfileKey, WebProcessRuntime, WebSocketBudgetLease};
|
||||
|
||||
/// One process-owned WebSocket carrier class used for eviction priority.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum WebSocketKind {
|
||||
/// One connection multiplexes every logical stream in a session.
|
||||
Multiplex,
|
||||
/// One connection owns exactly one logical stream lane.
|
||||
Lane(u32),
|
||||
}
|
||||
|
||||
pub(super) struct WebSocketEntry {
|
||||
id: u64,
|
||||
owner: ProfileKey,
|
||||
session_id: u64,
|
||||
client_ip: IpAddr,
|
||||
kind: WebSocketKind,
|
||||
liveness_interval_ms: u64,
|
||||
created_tick: u64,
|
||||
last_peer_tick: AtomicU64,
|
||||
last_progress_tick: AtomicU64,
|
||||
opened: AtomicBool,
|
||||
cancel: CancellationToken,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct WebSocketRegistry {
|
||||
entries: HashMap<u64, Arc<WebSocketEntry>>,
|
||||
closed: bool,
|
||||
}
|
||||
|
||||
/// Exact process-owned admission retained through the upgraded socket lifetime.
|
||||
pub(crate) struct WebSocketConnection {
|
||||
runtime: std::sync::Weak<WebProcessRuntime>,
|
||||
entry: Arc<WebSocketEntry>,
|
||||
slot: Option<OwnedSemaphorePermit>,
|
||||
base_budget: Option<WebSocketBudgetLease>,
|
||||
}
|
||||
|
||||
impl WebSocketConnection {
|
||||
/// Returns the cancellation signal used by shutdown and pressure eviction.
|
||||
pub(crate) fn cancellation(&self) -> CancellationToken {
|
||||
self.entry.cancel.clone()
|
||||
}
|
||||
|
||||
/// Returns the process-unique connection identifier used only for debugging.
|
||||
pub(crate) fn id(&self) -> u64 {
|
||||
self.entry.id
|
||||
}
|
||||
|
||||
/// Returns the creation-time transport liveness interval.
|
||||
pub(crate) fn liveness_interval(&self) -> Duration {
|
||||
Duration::from_millis(self.entry.liveness_interval_ms)
|
||||
}
|
||||
|
||||
/// Marks successful ownership transfer from HTTP to the WebSocket codec.
|
||||
pub(crate) fn mark_opened(&self) {
|
||||
self.entry.opened.store(true, Ordering::Release);
|
||||
self.mark_progress();
|
||||
}
|
||||
|
||||
/// Refreshes the peer-liveness deadline after any received WebSocket message.
|
||||
pub(crate) fn mark_peer_activity(&self) {
|
||||
if let Some(runtime) = self.runtime.upgrade() {
|
||||
let now = runtime.websocket_tick();
|
||||
self.entry.last_peer_tick.store(now, Ordering::Release);
|
||||
self.entry.last_progress_tick.store(now, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
/// Refreshes least-recently-progressed ordering after a committed write.
|
||||
pub(crate) fn mark_progress(&self) {
|
||||
if let Some(runtime) = self.runtime.upgrade() {
|
||||
self.entry
|
||||
.last_progress_tick
|
||||
.store(runtime.websocket_tick(), Ordering::Release);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WebSocketConnection {
|
||||
fn drop(&mut self) {
|
||||
if let Some(runtime) = self.runtime.upgrade() {
|
||||
runtime.websockets.lock().entries.remove(&self.entry.id);
|
||||
drop(self.base_budget.take());
|
||||
drop(self.slot.take());
|
||||
runtime.websocket_notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn admit(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
owner: ProfileKey,
|
||||
session_id: u64,
|
||||
client_ip: IpAddr,
|
||||
kind: WebSocketKind,
|
||||
base_bytes: usize,
|
||||
liveness_interval: Duration,
|
||||
eviction_timeout: Duration,
|
||||
) -> Result<WebSocketConnection, ManagerError> {
|
||||
let liveness_interval_ms = liveness_interval.as_millis().min(u128::from(u64::MAX)) as u64;
|
||||
if let Some(connection) = try_admit(
|
||||
runtime,
|
||||
owner,
|
||||
session_id,
|
||||
client_ip,
|
||||
kind,
|
||||
base_bytes,
|
||||
liveness_interval_ms,
|
||||
) {
|
||||
return Ok(connection);
|
||||
}
|
||||
let Some(victim) = select_victim(runtime, owner, session_id, client_ip, None) else {
|
||||
runtime.record_limit_hit();
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
let released = runtime.websocket_notify.notified();
|
||||
victim.cancel.cancel();
|
||||
let _ = tokio::time::timeout(eviction_timeout, released).await;
|
||||
try_admit(
|
||||
runtime,
|
||||
owner,
|
||||
session_id,
|
||||
client_ip,
|
||||
kind,
|
||||
base_bytes,
|
||||
liveness_interval_ms,
|
||||
)
|
||||
.ok_or_else(|| {
|
||||
runtime.record_limit_hit();
|
||||
ManagerError::Limit
|
||||
})
|
||||
}
|
||||
|
||||
fn try_admit(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
owner: ProfileKey,
|
||||
session_id: u64,
|
||||
client_ip: IpAddr,
|
||||
kind: WebSocketKind,
|
||||
base_bytes: usize,
|
||||
liveness_interval_ms: u64,
|
||||
) -> Option<WebSocketConnection> {
|
||||
let slot = Arc::clone(&runtime.websocket_connections)
|
||||
.try_acquire_owned()
|
||||
.ok()?;
|
||||
let base_budget = runtime.try_websocket_base_budget(owner, base_bytes)?;
|
||||
let id = runtime.websocket_next_id.fetch_add(1, Ordering::Relaxed);
|
||||
let now = runtime.websocket_tick();
|
||||
let entry = Arc::new(WebSocketEntry {
|
||||
id,
|
||||
owner,
|
||||
session_id,
|
||||
client_ip,
|
||||
kind,
|
||||
liveness_interval_ms,
|
||||
created_tick: now,
|
||||
last_peer_tick: AtomicU64::new(now),
|
||||
last_progress_tick: AtomicU64::new(now),
|
||||
opened: AtomicBool::new(false),
|
||||
cancel: CancellationToken::new(),
|
||||
});
|
||||
let mut registry = runtime.websockets.lock();
|
||||
if registry.closed {
|
||||
return None;
|
||||
}
|
||||
registry.entries.insert(id, Arc::clone(&entry));
|
||||
drop(registry);
|
||||
Some(WebSocketConnection {
|
||||
runtime: Arc::downgrade(runtime),
|
||||
entry,
|
||||
slot: Some(slot),
|
||||
base_budget: Some(base_budget),
|
||||
})
|
||||
}
|
||||
|
||||
impl WebProcessRuntime {
|
||||
pub(super) fn websocket_tick(&self) -> u64 {
|
||||
self.websocket_clock.elapsed().as_millis() as u64
|
||||
}
|
||||
|
||||
pub(super) fn cleanup_websockets(&self) {
|
||||
let now = self.websocket_tick();
|
||||
let mut victims = self
|
||||
.websockets
|
||||
.lock()
|
||||
.entries
|
||||
.values()
|
||||
.filter(|entry| {
|
||||
now.saturating_sub(entry.last_peer_tick.load(Ordering::Acquire))
|
||||
>= dead_after(entry)
|
||||
})
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
if victims.is_empty()
|
||||
&& self.data_budget.take_pressure()
|
||||
&& let Some(victim) = select_pressure_victim(self, now)
|
||||
{
|
||||
victims.push(victim);
|
||||
}
|
||||
for victim in victims {
|
||||
victim.cancel.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn close_websockets(&self) {
|
||||
let victims = {
|
||||
let mut registry = self.websockets.lock();
|
||||
registry.closed = true;
|
||||
registry.entries.values().cloned().collect::<Vec<_>>()
|
||||
};
|
||||
for victim in victims {
|
||||
victim.cancel.cancel();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn select_victim(
|
||||
runtime: &WebProcessRuntime,
|
||||
owner: ProfileKey,
|
||||
session_id: u64,
|
||||
client_ip: IpAddr,
|
||||
excluded_id: Option<u64>,
|
||||
) -> Option<Arc<WebSocketEntry>> {
|
||||
let fair_share = runtime.data_budget.fair_share(Some(owner));
|
||||
let requester_usage = runtime.data_budget.owner_usage(owner);
|
||||
let now = runtime.websocket_tick();
|
||||
runtime
|
||||
.websockets
|
||||
.lock()
|
||||
.entries
|
||||
.values()
|
||||
.filter(|entry| Some(entry.id) != excluded_id)
|
||||
.filter_map(|entry| {
|
||||
let owner_rank = if entry.session_id == session_id {
|
||||
0
|
||||
} else if entry.owner == owner {
|
||||
1
|
||||
} else if entry.client_ip == client_ip {
|
||||
2
|
||||
} else {
|
||||
if requester_usage >= fair_share
|
||||
|| runtime.data_budget.owner_usage(entry.owner) <= fair_share
|
||||
{
|
||||
return None;
|
||||
}
|
||||
3
|
||||
};
|
||||
let priority = entry_priority(entry, now);
|
||||
Some((
|
||||
(
|
||||
owner_rank,
|
||||
priority,
|
||||
entry.last_progress_tick.load(Ordering::Acquire),
|
||||
entry.created_tick,
|
||||
entry.id,
|
||||
),
|
||||
Arc::clone(entry),
|
||||
))
|
||||
})
|
||||
.min_by_key(|(key, _)| *key)
|
||||
.map(|(_, entry)| entry)
|
||||
}
|
||||
|
||||
fn select_pressure_victim(runtime: &WebProcessRuntime, now: u64) -> Option<Arc<WebSocketEntry>> {
|
||||
runtime
|
||||
.websockets
|
||||
.lock()
|
||||
.entries
|
||||
.values()
|
||||
.map(|entry| {
|
||||
(
|
||||
(
|
||||
entry_priority(entry, now),
|
||||
entry.last_progress_tick.load(Ordering::Acquire),
|
||||
entry.created_tick,
|
||||
entry.id,
|
||||
),
|
||||
Arc::clone(entry),
|
||||
)
|
||||
})
|
||||
.min_by_key(|(key, _)| *key)
|
||||
.map(|(_, entry)| entry)
|
||||
}
|
||||
|
||||
fn entry_priority(entry: &WebSocketEntry, now: u64) -> u8 {
|
||||
if !entry.opened.load(Ordering::Acquire)
|
||||
|| now.saturating_sub(entry.last_peer_tick.load(Ordering::Acquire)) >= dead_after(entry)
|
||||
{
|
||||
0
|
||||
} else if matches!(entry.kind, WebSocketKind::Lane(_)) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
}
|
||||
}
|
||||
|
||||
fn dead_after(entry: &WebSocketEntry) -> u64 {
|
||||
entry.liveness_interval_ms.saturating_mul(2)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,40 @@
|
||||
use super::*;
|
||||
|
||||
fn entry(kind: WebSocketKind, opened: bool, peer_tick: u64) -> WebSocketEntry {
|
||||
WebSocketEntry {
|
||||
id: 1,
|
||||
owner: [0; 32],
|
||||
session_id: 1,
|
||||
client_ip: "192.0.2.10".parse().unwrap(),
|
||||
kind,
|
||||
liveness_interval_ms: 10,
|
||||
created_tick: 1,
|
||||
last_peer_tick: AtomicU64::new(peer_tick),
|
||||
last_progress_tick: AtomicU64::new(peer_tick),
|
||||
opened: AtomicBool::new(opened),
|
||||
cancel: CancellationToken::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preopen_and_dead_entries_precede_live_lane_and_multiplex_victims() {
|
||||
let preopen = entry(WebSocketKind::Multiplex, false, 90);
|
||||
let dead = entry(WebSocketKind::Multiplex, true, 1);
|
||||
let lane = entry(WebSocketKind::Lane(7), true, 90);
|
||||
let multiplex = entry(WebSocketKind::Multiplex, true, 90);
|
||||
|
||||
assert_eq!(entry_priority(&preopen, 100), 0);
|
||||
assert_eq!(entry_priority(&dead, 100), 0);
|
||||
assert_eq!(entry_priority(&lane, 100), 1);
|
||||
assert_eq!(entry_priority(&multiplex, 100), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dead_classification_keeps_each_connections_creation_time_interval() {
|
||||
let short_interval = entry(WebSocketKind::Multiplex, true, 80);
|
||||
let mut long_interval = entry(WebSocketKind::Multiplex, true, 80);
|
||||
long_interval.liveness_interval_ms = 100;
|
||||
|
||||
assert_eq!(entry_priority(&short_interval, 100), 0);
|
||||
assert_eq!(entry_priority(&long_interval, 100), 2);
|
||||
}
|
||||
@@ -12,3 +12,5 @@ pub(crate) mod manager;
|
||||
pub(crate) mod session;
|
||||
/// AsyncRead and AsyncWrite adapter for one logical MTProxy stream.
|
||||
pub(crate) mod stream;
|
||||
/// Process-owned bounded WEB debugging records and capture lifecycle.
|
||||
pub(crate) mod trace;
|
||||
|
||||
+51
-4
@@ -22,6 +22,9 @@ mod backend;
|
||||
mod downlink;
|
||||
// Lane carrier state isolates request sequencing and downlink replay per logical stream.
|
||||
mod lanes;
|
||||
// WebSocket carrier state owns pre-OPEN lane reservations and failure isolation.
|
||||
mod websocket;
|
||||
pub(crate) use websocket::WebSocketLaneReservation;
|
||||
// Uplink batches own exactly-once sequencing and client-frame validation.
|
||||
mod uplink;
|
||||
|
||||
@@ -107,6 +110,7 @@ struct SessionState {
|
||||
last_up_sequence: u64,
|
||||
last_up_digest: TokenHash,
|
||||
carrier_lanes: HashMap<u32, CarrierLane>,
|
||||
websocket_lane_reservations: HashMap<u32, u16>,
|
||||
pending_bytes: usize,
|
||||
pending_items: usize,
|
||||
pending_control_bytes: usize,
|
||||
@@ -120,6 +124,7 @@ pub(crate) struct WebSession {
|
||||
manager: std::sync::Weak<WebProcessRuntime>,
|
||||
token_hash: TokenHash,
|
||||
client_ip: IpAddr,
|
||||
trace_session_id: u64,
|
||||
profile: Arc<WebRuntimeProfile>,
|
||||
profile_key: ProfileKey,
|
||||
limits: WebLimitsConfig,
|
||||
@@ -150,6 +155,7 @@ impl WebSession {
|
||||
manager: std::sync::Weak<WebProcessRuntime>,
|
||||
token_hash: TokenHash,
|
||||
client_ip: IpAddr,
|
||||
trace_session_id: u64,
|
||||
profile: Arc<WebRuntimeProfile>,
|
||||
profile_key: ProfileKey,
|
||||
limits: WebLimitsConfig,
|
||||
@@ -163,6 +169,7 @@ impl WebSession {
|
||||
manager,
|
||||
token_hash,
|
||||
client_ip,
|
||||
trace_session_id,
|
||||
profile,
|
||||
profile_key,
|
||||
limits,
|
||||
@@ -180,6 +187,7 @@ impl WebSession {
|
||||
last_up_sequence: 0,
|
||||
last_up_digest: [0; 32],
|
||||
carrier_lanes,
|
||||
websocket_lane_reservations: HashMap::new(),
|
||||
pending_bytes: 0,
|
||||
pending_items: 0,
|
||||
pending_control_bytes: 0,
|
||||
@@ -211,6 +219,40 @@ impl WebSession {
|
||||
self.profile.carrier
|
||||
}
|
||||
|
||||
/// Returns the stable quota owner without exposing profile credentials.
|
||||
pub(crate) fn profile_key(&self) -> ProfileKey {
|
||||
self.profile_key
|
||||
}
|
||||
|
||||
/// Returns the process-unique non-secret trace identifier.
|
||||
pub(crate) fn trace_session_id(&self) -> u64 {
|
||||
self.trace_session_id
|
||||
}
|
||||
|
||||
/// Returns a cloned non-secret identity only for enabled debug capture.
|
||||
pub(crate) fn trace_identity(&self) -> crate::web::trace::TraceIdentity {
|
||||
crate::web::trace::TraceIdentity::from_profile(self.trace_session_id, &self.profile)
|
||||
}
|
||||
|
||||
/// Records one typed lifecycle event without exposing session credentials.
|
||||
pub(super) fn trace_lifecycle(
|
||||
&self,
|
||||
event: crate::web::trace::TraceLifecycleEvent,
|
||||
stream_id: Option<u32>,
|
||||
reason: Option<&'static str>,
|
||||
) {
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
manager.trace().record_profile_lifecycle(
|
||||
self.client_ip,
|
||||
Some(self.trace_session_id),
|
||||
&self.profile,
|
||||
event,
|
||||
stream_id,
|
||||
reason,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Closes carrier state while relay tasks retain their admission until exit.
|
||||
pub(crate) fn close(&self) {
|
||||
let (data_bytes, data_items, control_bytes, control_items) = {
|
||||
@@ -246,13 +288,18 @@ impl WebSession {
|
||||
(data_bytes, data_items, control_bytes, control_items)
|
||||
};
|
||||
self.cancel.cancel();
|
||||
if self.carrier() == WebCarrier::Https {
|
||||
if self.carrier().is_multiplexed() {
|
||||
self.down_notify.notify_waiters();
|
||||
}
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
manager.release_pending(data_bytes, data_items, false);
|
||||
manager.release_pending(control_bytes, control_items, true);
|
||||
manager.release_pending(self.profile_key, data_bytes, data_items, false);
|
||||
manager.release_pending(self.profile_key, control_bytes, control_items, true);
|
||||
if !self.finished.swap(true, Ordering::AcqRel) {
|
||||
self.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::SessionClosed,
|
||||
None,
|
||||
Some("closed"),
|
||||
);
|
||||
manager.session_finished(
|
||||
self.token_hash,
|
||||
self.client_ip,
|
||||
@@ -362,7 +409,7 @@ impl WebSession {
|
||||
stream.send_credit -= count as u64;
|
||||
state.last_activity = Instant::now();
|
||||
drop(state);
|
||||
if self.carrier() == WebCarrier::Https {
|
||||
if self.carrier().is_multiplexed() {
|
||||
self.down_notify.notify_waiters();
|
||||
}
|
||||
Poll::Ready(Ok(count))
|
||||
|
||||
+179
-32
@@ -1,6 +1,6 @@
|
||||
use std::io;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::proxy::shared_state::ConntrackClosePolicy;
|
||||
@@ -9,52 +9,109 @@ use crate::web::stream::WebLogicalStream;
|
||||
|
||||
use super::{WebSession, inbound_queue_cost};
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "backend_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
impl WebSession {
|
||||
/// Starts one owned inner handshake and relay task for an admitted stream.
|
||||
pub(super) fn spawn_stream(self: &Arc<Self>, stream_id: u32, peer_port: u16) {
|
||||
pub(super) fn spawn_stream(
|
||||
self: &Arc<Self>,
|
||||
stream_id: u32,
|
||||
peer_port: u16,
|
||||
retain_reservation_on_reject: bool,
|
||||
) -> bool {
|
||||
let Some(manager) = self.manager.upgrade() else {
|
||||
self.stream_finished(stream_id, peer_port);
|
||||
return;
|
||||
self.stream_rejected_before_spawn(stream_id, peer_port, retain_reservation_on_reject);
|
||||
return false;
|
||||
};
|
||||
let generation = manager.active_generation();
|
||||
if !*generation.admission_rx.borrow() {
|
||||
self.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamRejected,
|
||||
Some(stream_id),
|
||||
Some("admission_closed"),
|
||||
);
|
||||
self.stream_rejected_before_spawn(stream_id, peer_port, retain_reservation_on_reject);
|
||||
return false;
|
||||
}
|
||||
let Ok(connection_permit) = generation.max_connections.clone().try_acquire_owned() else {
|
||||
manager.record_stream_rejected();
|
||||
self.stream_finished(stream_id, peer_port);
|
||||
return;
|
||||
};
|
||||
let Some(handshake_permit) = manager.try_stream_handshake() else {
|
||||
self.stream_finished(stream_id, peer_port);
|
||||
return;
|
||||
self.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamRejected,
|
||||
Some(stream_id),
|
||||
Some("connection_limit"),
|
||||
);
|
||||
self.stream_rejected_before_spawn(stream_id, peer_port, retain_reservation_on_reject);
|
||||
return false;
|
||||
};
|
||||
let deps = generation.client_runtime_deps();
|
||||
let replay_checker = Arc::clone(&generation.replay_checker);
|
||||
let session = Arc::clone(self);
|
||||
let cancel = self.cancel.clone();
|
||||
let retain_rejected = Arc::new(AtomicBool::new(false));
|
||||
self.tasks_live.fetch_add(1, Ordering::AcqRel);
|
||||
let spawned = generation.spawn_session(async move {
|
||||
let completion = StreamCompletion {
|
||||
session: Arc::clone(&session),
|
||||
stream_id,
|
||||
peer_port,
|
||||
retain_rejected: Arc::clone(&retain_rejected),
|
||||
};
|
||||
let future = async move {
|
||||
let _connection_permit = connection_permit;
|
||||
let _completion = StreamCompletion {
|
||||
session: Arc::clone(&session),
|
||||
stream_id,
|
||||
peer_port,
|
||||
};
|
||||
let _completion = completion;
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamAdmitted,
|
||||
Some(stream_id),
|
||||
None,
|
||||
);
|
||||
let stream = WebLogicalStream::new(Arc::clone(&session), stream_id);
|
||||
tokio::select! {
|
||||
_ = cancel.cancelled() => {}
|
||||
_ = run_stream(
|
||||
Arc::clone(&session),
|
||||
stream_id,
|
||||
stream,
|
||||
deps,
|
||||
replay_checker,
|
||||
handshake_permit,
|
||||
peer_port,
|
||||
) => {}
|
||||
}
|
||||
});
|
||||
if !spawned {
|
||||
self.tasks_live.fetch_sub(1, Ordering::AcqRel);
|
||||
};
|
||||
if let Err(future) = generation.try_spawn_session(future) {
|
||||
retain_rejected.store(retain_reservation_on_reject, Ordering::Release);
|
||||
self.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamRejected,
|
||||
Some(stream_id),
|
||||
Some("generation_closed"),
|
||||
);
|
||||
drop(future);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn stream_rejected_before_spawn(
|
||||
&self,
|
||||
stream_id: u32,
|
||||
peer_port: u16,
|
||||
retain_reservation: bool,
|
||||
) {
|
||||
if !retain_reservation {
|
||||
self.stream_finished(stream_id, peer_port);
|
||||
self.tasks_done.notify_waiters();
|
||||
return;
|
||||
}
|
||||
let queued = {
|
||||
let mut state = self.state.lock();
|
||||
state.streams.remove(&stream_id).map(|stream| {
|
||||
let (bytes, items) = inbound_queue_cost(&stream.inbound);
|
||||
self.release_locked(&mut state, bytes, items, false);
|
||||
self.remember_closed_locked(&mut state, stream_id);
|
||||
self.queue_control_locked(&mut state, FrameType::Close, stream_id, &[])
|
||||
})
|
||||
};
|
||||
if queued.is_some_and(|queued| !queued) {
|
||||
self.close();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,7 +139,7 @@ impl WebSession {
|
||||
if !queued {
|
||||
self.close();
|
||||
}
|
||||
if self.carrier() == crate::config::WebCarrier::Https {
|
||||
if self.carrier().is_multiplexed() {
|
||||
self.down_notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
@@ -93,11 +150,22 @@ struct StreamCompletion {
|
||||
session: Arc<WebSession>,
|
||||
stream_id: u32,
|
||||
peer_port: u16,
|
||||
retain_rejected: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl Drop for StreamCompletion {
|
||||
fn drop(&mut self) {
|
||||
self.session.stream_finished(self.stream_id, self.peer_port);
|
||||
self.session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamClosed,
|
||||
Some(self.stream_id),
|
||||
None,
|
||||
);
|
||||
if self.retain_rejected.load(Ordering::Acquire) {
|
||||
self.session
|
||||
.stream_rejected_before_spawn(self.stream_id, self.peer_port, true);
|
||||
} else {
|
||||
self.session.stream_finished(self.stream_id, self.peer_port);
|
||||
}
|
||||
if self.session.tasks_live.fetch_sub(1, Ordering::AcqRel) == 1 {
|
||||
self.session.tasks_done.notify_waiters();
|
||||
}
|
||||
@@ -106,10 +174,10 @@ impl Drop for StreamCompletion {
|
||||
|
||||
async fn run_stream(
|
||||
session: Arc<WebSession>,
|
||||
stream_id: u32,
|
||||
stream: WebLogicalStream,
|
||||
deps: crate::proxy::authenticated::ClientRuntimeDeps,
|
||||
replay_checker: Arc<crate::stats::ReplayChecker>,
|
||||
handshake_permit: tokio::sync::OwnedSemaphorePermit,
|
||||
peer_port: u16,
|
||||
) {
|
||||
use tokio::io::AsyncReadExt;
|
||||
@@ -122,10 +190,40 @@ async fn run_stream(
|
||||
let mut handshake = [0u8; HANDSHAKE_LEN];
|
||||
let peer = std::net::SocketAddr::new(session.client_ip, peer_port);
|
||||
deps.stats.increment_connects_all();
|
||||
|
||||
// A carrier may publish OPEN before the local MTProto socket writes its
|
||||
// first byte. Session and stream quotas bound this idle phase without
|
||||
// consuming the process-wide active-handshake budget.
|
||||
if reader.read_exact(&mut handshake[..1]).await.is_err() {
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::HandshakeIo,
|
||||
Some(stream_id),
|
||||
Some("first_byte_io"),
|
||||
);
|
||||
deps.stats
|
||||
.increment_connects_bad_with_class("web_mtproto_handshake_io");
|
||||
return;
|
||||
}
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamFirstByte,
|
||||
Some(stream_id),
|
||||
None,
|
||||
);
|
||||
let Some(manager) = session.manager.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let Some(handshake_permit) = manager.try_stream_handshake() else {
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamRejected,
|
||||
Some(stream_id),
|
||||
Some("handshake_limit"),
|
||||
);
|
||||
return;
|
||||
};
|
||||
let handshake_result = tokio::time::timeout(
|
||||
Duration::from_secs(session.timeouts.stream_handshake_secs),
|
||||
async {
|
||||
reader.read_exact(&mut handshake).await?;
|
||||
reader.read_exact(&mut handshake[1..]).await?;
|
||||
Ok::<_, io::Error>(
|
||||
handle_mtproto_handshake_for_web_user(
|
||||
&handshake,
|
||||
@@ -144,14 +242,54 @@ async fn run_stream(
|
||||
)
|
||||
.await;
|
||||
drop(handshake_permit);
|
||||
let Ok(Ok(crate::error::HandshakeResult::Success((reader, writer, success)))) =
|
||||
handshake_result
|
||||
else {
|
||||
deps.stats
|
||||
.increment_connects_bad_with_class("web_mtproto_bad_client");
|
||||
return;
|
||||
let (reader, writer, success) = match handshake_result {
|
||||
Err(_) => {
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::HandshakeTimeout,
|
||||
Some(stream_id),
|
||||
Some("timeout"),
|
||||
);
|
||||
deps.stats
|
||||
.increment_connects_bad_with_class("web_mtproto_handshake_timeout");
|
||||
deps.stats.increment_handshake_timeouts();
|
||||
deps.stats.increment_handshake_failure_class("timeout");
|
||||
return;
|
||||
}
|
||||
Ok(Err(_)) => {
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::HandshakeIo,
|
||||
Some(stream_id),
|
||||
Some("io"),
|
||||
);
|
||||
deps.stats
|
||||
.increment_connects_bad_with_class("web_mtproto_handshake_io");
|
||||
return;
|
||||
}
|
||||
Ok(Ok(crate::error::HandshakeResult::Success((reader, writer, success)))) => {
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::HandshakeSucceeded,
|
||||
Some(stream_id),
|
||||
None,
|
||||
);
|
||||
(reader, writer, success)
|
||||
}
|
||||
Ok(Ok(_)) => {
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::HandshakeRejected,
|
||||
Some(stream_id),
|
||||
Some("bad_client"),
|
||||
);
|
||||
deps.stats
|
||||
.increment_connects_bad_with_class("web_mtproto_bad_client");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let _ = run_authenticated(
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::RelayStarted,
|
||||
Some(stream_id),
|
||||
None,
|
||||
);
|
||||
let relay_result = run_authenticated(
|
||||
reader,
|
||||
writer,
|
||||
success,
|
||||
@@ -161,4 +299,13 @@ async fn run_stream(
|
||||
ConntrackClosePolicy::Suppress,
|
||||
)
|
||||
.await;
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::RelayEnded,
|
||||
Some(stream_id),
|
||||
Some(if relay_result.is_ok() {
|
||||
"completed"
|
||||
} else {
|
||||
"error"
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,363 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::watch;
|
||||
|
||||
use super::*;
|
||||
use crate::config::{
|
||||
ProxyConfig, UpstreamConfig, UpstreamType, WebCarrier, WebRuntimeConfig, WebRuntimeProfile,
|
||||
WebSecretMode,
|
||||
};
|
||||
use crate::crypto::{AesCtr, sha256};
|
||||
use crate::maestro::generation::{RuntimeGeneration, test_runtime_generation_with_admission};
|
||||
use crate::protocol::constants::{
|
||||
DC_IDX_POS, HANDSHAKE_LEN, IV_LEN, PREKEY_LEN, PROTO_TAG_POS, ProtoTag, SKIP_LEN,
|
||||
};
|
||||
use crate::web::frame;
|
||||
use crate::web::manager::{ManagerError, WebProcessRuntime};
|
||||
|
||||
struct TestRuntime {
|
||||
session: Arc<WebSession>,
|
||||
manager: Arc<WebProcessRuntime>,
|
||||
generation: Arc<RuntimeGeneration>,
|
||||
admission_tx: watch::Sender<bool>,
|
||||
}
|
||||
|
||||
impl TestRuntime {
|
||||
fn process_frame(
|
||||
&self,
|
||||
stream_id: u32,
|
||||
sequence: u64,
|
||||
frame_type: FrameType,
|
||||
payload: &[u8],
|
||||
) -> Result<u64, ManagerError> {
|
||||
let encoded = frame::encode(frame_type, stream_id, payload);
|
||||
match self.session.carrier() {
|
||||
WebCarrier::Https | WebCarrier::Websocket => {
|
||||
self.session.process_up(sequence, &encoded)
|
||||
}
|
||||
WebCarrier::HttpsLanes | WebCarrier::WebsocketLanes => {
|
||||
self.session.process_up_lane(stream_id, sequence, &encoded)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn shutdown(self) {
|
||||
self.session.close();
|
||||
self.session.wait().await;
|
||||
self.manager.shutdown().await;
|
||||
self.generation.stop_sessions().await;
|
||||
self.generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
|
||||
fn test_runtime(carrier: WebCarrier, max_stream_handshakes: usize) -> TestRuntime {
|
||||
test_runtime_with_dc(carrier, max_stream_handshakes, None)
|
||||
}
|
||||
|
||||
fn test_runtime_with_dc(
|
||||
carrier: WebCarrier,
|
||||
max_stream_handshakes: usize,
|
||||
dc_addr: Option<SocketAddr>,
|
||||
) -> TestRuntime {
|
||||
let profile = Arc::new(WebRuntimeProfile {
|
||||
host: "proxy.example.com".to_string(),
|
||||
public_addr: "203.0.113.10:443".parse().unwrap(),
|
||||
user: "default".to_string(),
|
||||
secret_mode: WebSecretMode::Plain,
|
||||
carrier,
|
||||
capability: [7; 32],
|
||||
key_fingerprint: "0000000000000000".to_string(),
|
||||
max_sessions: 4,
|
||||
max_streams: 16,
|
||||
max_streams_per_session: 4,
|
||||
});
|
||||
let mut config = ProxyConfig::default();
|
||||
config.web.enabled = true;
|
||||
config.web.carrier = carrier;
|
||||
config.web.limits.max_stream_handshakes = max_stream_handshakes;
|
||||
config.web.timeouts.stream_handshake_secs = 1;
|
||||
config.web.timeouts.shutdown_secs = 1;
|
||||
config.censorship.server_hello_delay_min_ms = 0;
|
||||
config.censorship.server_hello_delay_max_ms = 0;
|
||||
if let Some(dc_addr) = dc_addr {
|
||||
config
|
||||
.dc_overrides
|
||||
.insert("2".to_string(), vec![dc_addr.to_string()]);
|
||||
config.upstreams.push(UpstreamConfig {
|
||||
upstream_type: UpstreamType::Direct {
|
||||
interface: None,
|
||||
bind_addresses: None,
|
||||
bindtodevice: None,
|
||||
},
|
||||
weight: 1,
|
||||
enabled: true,
|
||||
scopes: String::new(),
|
||||
selected_scope: String::new(),
|
||||
ipv4: Some(true),
|
||||
ipv6: Some(false),
|
||||
prefer: Some(4),
|
||||
});
|
||||
}
|
||||
config.web.runtime = Some(Arc::new(WebRuntimeConfig {
|
||||
vhosts: BTreeMap::new(),
|
||||
profiles: vec![Arc::clone(&profile)],
|
||||
}));
|
||||
config.rebuild_runtime_user_auth().unwrap();
|
||||
let limits = config.web.limits.clone();
|
||||
let timeouts = config.web.timeouts.clone();
|
||||
let (admission_tx, admission_rx) = watch::channel(true);
|
||||
let generation = test_runtime_generation_with_admission(1, config, admission_rx);
|
||||
let manager = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
let session = WebSession::new(
|
||||
Arc::downgrade(&manager),
|
||||
[8; 32],
|
||||
"192.0.2.10".parse().unwrap(),
|
||||
1,
|
||||
profile,
|
||||
[7; 32],
|
||||
limits,
|
||||
timeouts,
|
||||
);
|
||||
TestRuntime {
|
||||
session,
|
||||
manager,
|
||||
generation,
|
||||
admission_tx,
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_plain_handshake() -> [u8; HANDSHAKE_LEN] {
|
||||
let secret = [0u8; 16];
|
||||
let mut handshake = [0x5a; HANDSHAKE_LEN];
|
||||
for (index, byte) in handshake[SKIP_LEN..SKIP_LEN + PREKEY_LEN + IV_LEN]
|
||||
.iter_mut()
|
||||
.enumerate()
|
||||
{
|
||||
*byte = (index as u8).wrapping_add(1);
|
||||
}
|
||||
let dec_prekey = &handshake[SKIP_LEN..SKIP_LEN + PREKEY_LEN];
|
||||
let dec_iv_bytes = &handshake[SKIP_LEN + PREKEY_LEN..SKIP_LEN + PREKEY_LEN + IV_LEN];
|
||||
let mut dec_key_input = Vec::with_capacity(PREKEY_LEN + secret.len());
|
||||
dec_key_input.extend_from_slice(dec_prekey);
|
||||
dec_key_input.extend_from_slice(&secret);
|
||||
let dec_key = sha256(&dec_key_input);
|
||||
let mut dec_iv = [0u8; IV_LEN];
|
||||
dec_iv.copy_from_slice(dec_iv_bytes);
|
||||
let mut cipher = AesCtr::new(&dec_key, u128::from_be_bytes(dec_iv));
|
||||
let keystream = cipher.encrypt(&[0u8; HANDSHAKE_LEN]);
|
||||
let mut plaintext = [0u8; HANDSHAKE_LEN];
|
||||
plaintext[PROTO_TAG_POS..PROTO_TAG_POS + 4].copy_from_slice(&ProtoTag::Intermediate.to_bytes());
|
||||
plaintext[DC_IDX_POS..DC_IDX_POS + 2].copy_from_slice(&2i16.to_le_bytes());
|
||||
for index in PROTO_TAG_POS..HANDSHAKE_LEN {
|
||||
handshake[index] = plaintext[index] ^ keystream[index];
|
||||
}
|
||||
handshake
|
||||
}
|
||||
|
||||
async fn settle_tasks() {
|
||||
for _ in 0..4 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
|
||||
fn bad_class(runtime: &TestRuntime, class: &str) -> u64 {
|
||||
runtime
|
||||
.generation
|
||||
.stats
|
||||
.get_connects_bad_class_counts()
|
||||
.into_iter()
|
||||
.find_map(|(name, total)| (name == class).then_some(total))
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn open_without_data_does_not_start_the_inner_handshake_timeout() {
|
||||
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||
let runtime = test_runtime(carrier, 1);
|
||||
|
||||
assert_eq!(runtime.process_frame(1, 1, FrameType::Open, &[]), Ok(1));
|
||||
settle_tasks().await;
|
||||
assert!(runtime.session.state.lock().streams.contains_key(&1));
|
||||
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
settle_tasks().await;
|
||||
|
||||
assert!(
|
||||
runtime.session.state.lock().streams.contains_key(&1),
|
||||
"OPEN without DATA must remain live until session cancellation or idle expiry"
|
||||
);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn the_first_inner_byte_starts_the_handshake_timeout() {
|
||||
let runtime = test_runtime(WebCarrier::Https, 1);
|
||||
|
||||
assert_eq!(runtime.process_frame(1, 1, FrameType::Open, &[]), Ok(1));
|
||||
settle_tasks().await;
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
assert_eq!(runtime.process_frame(1, 2, FrameType::Data, &[0x5a]), Ok(2));
|
||||
settle_tasks().await;
|
||||
assert!(runtime.session.state.lock().streams.contains_key(&1));
|
||||
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
settle_tasks().await;
|
||||
|
||||
assert!(!runtime.session.state.lock().streams.contains_key(&1));
|
||||
assert_eq!(bad_class(&runtime, "web_mtproto_handshake_timeout"), 1);
|
||||
assert_eq!(runtime.generation.stats.get_handshake_timeouts(), 1);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.generation
|
||||
.stats
|
||||
.get_handshake_failure_class_counts(),
|
||||
vec![("timeout".to_string(), 1)]
|
||||
);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn delayed_complete_handshake_is_classified_after_data_arrives() {
|
||||
let runtime = test_runtime(WebCarrier::Https, 1);
|
||||
|
||||
assert_eq!(runtime.process_frame(1, 1, FrameType::Open, &[]), Ok(1));
|
||||
settle_tasks().await;
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
assert_eq!(
|
||||
runtime.process_frame(1, 2, FrameType::Data, &[0; 64]),
|
||||
Ok(2)
|
||||
);
|
||||
settle_tasks().await;
|
||||
|
||||
assert!(!runtime.session.state.lock().streams.contains_key(&1));
|
||||
assert_eq!(bad_class(&runtime, "web_mtproto_bad_client"), 1);
|
||||
assert_eq!(bad_class(&runtime, "web_mtproto_handshake_timeout"), 0);
|
||||
assert_eq!(runtime.generation.stats.get_handshake_timeouts(), 0);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn delayed_valid_handshake_reaches_the_authenticated_relay() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let runtime = test_runtime_with_dc(WebCarrier::Https, 1, Some(listener.local_addr().unwrap()));
|
||||
|
||||
assert_eq!(runtime.process_frame(1, 1, FrameType::Open, &[]), Ok(1));
|
||||
settle_tasks().await;
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
assert_eq!(
|
||||
runtime.process_frame(1, 2, FrameType::Data, &valid_plain_handshake()),
|
||||
Ok(2)
|
||||
);
|
||||
|
||||
let accepted = tokio::time::timeout(Duration::from_secs(1), listener.accept()).await;
|
||||
assert!(
|
||||
accepted.is_ok(),
|
||||
"valid handshake did not reach the configured upstream; bad classes: {:?}",
|
||||
runtime.generation.stats.get_connects_bad_class_counts()
|
||||
);
|
||||
let (upstream, _) = accepted.unwrap().unwrap();
|
||||
assert!(runtime.session.state.lock().streams.contains_key(&1));
|
||||
assert_eq!(runtime.generation.stats.get_connects_bad(), 0);
|
||||
assert_eq!(runtime.generation.stats.get_handshake_timeouts(), 0);
|
||||
|
||||
runtime.shutdown().await;
|
||||
drop(upstream);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn closed_generation_admission_rejects_web_stream_before_backend() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let runtime = test_runtime_with_dc(WebCarrier::Https, 1, Some(listener.local_addr().unwrap()));
|
||||
runtime.admission_tx.send_replace(false);
|
||||
|
||||
assert_eq!(runtime.process_frame(1, 1, FrameType::Open, &[]), Ok(1));
|
||||
settle_tasks().await;
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), listener.accept())
|
||||
.await
|
||||
.is_err(),
|
||||
"WEB stream bypassed the closed generation admission gate"
|
||||
);
|
||||
assert!(!runtime.session.state.lock().streams.contains_key(&1));
|
||||
assert_eq!(runtime.generation.max_connections.available_permits(), 64);
|
||||
|
||||
runtime.admission_tx.send_replace(true);
|
||||
assert_eq!(runtime.process_frame(2, 2, FrameType::Open, &[]), Ok(2));
|
||||
assert_eq!(
|
||||
runtime.process_frame(2, 3, FrameType::Data, &valid_plain_handshake()),
|
||||
Ok(3)
|
||||
);
|
||||
let (upstream, _) = tokio::time::timeout(Duration::from_secs(1), listener.accept())
|
||||
.await
|
||||
.expect("a new WEB stream did not start after admission reopened")
|
||||
.unwrap();
|
||||
assert_eq!(runtime.generation.max_connections.available_permits(), 63);
|
||||
|
||||
runtime.shutdown().await;
|
||||
drop(upstream);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn silent_streams_do_not_consume_active_handshake_capacity() {
|
||||
let runtime = test_runtime(WebCarrier::Https, 1);
|
||||
|
||||
assert_eq!(runtime.process_frame(1, 1, FrameType::Open, &[]), Ok(1));
|
||||
assert_eq!(runtime.process_frame(2, 2, FrameType::Open, &[]), Ok(2));
|
||||
settle_tasks().await;
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
settle_tasks().await;
|
||||
{
|
||||
let state = runtime.session.state.lock();
|
||||
assert!(state.streams.contains_key(&1));
|
||||
assert!(state.streams.contains_key(&2));
|
||||
}
|
||||
|
||||
assert_eq!(runtime.process_frame(1, 3, FrameType::Data, &[1]), Ok(3));
|
||||
settle_tasks().await;
|
||||
assert_eq!(runtime.process_frame(2, 4, FrameType::Data, &[2]), Ok(4));
|
||||
settle_tasks().await;
|
||||
|
||||
let state = runtime.session.state.lock();
|
||||
assert!(state.streams.contains_key(&1));
|
||||
assert!(!state.streams.contains_key(&2));
|
||||
drop(state);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cancellation_while_waiting_for_data_releases_stream_ownership() {
|
||||
let runtime = test_runtime(WebCarrier::Https, 1);
|
||||
|
||||
assert_eq!(runtime.process_frame(1, 1, FrameType::Open, &[]), Ok(1));
|
||||
settle_tasks().await;
|
||||
assert_eq!(runtime.session.tasks_live.load(Ordering::Acquire), 1);
|
||||
|
||||
runtime.session.close();
|
||||
runtime.session.wait().await;
|
||||
|
||||
assert_eq!(runtime.session.tasks_live.load(Ordering::Acquire), 0);
|
||||
assert!(runtime.session.state.lock().active_peer_ports.is_empty());
|
||||
let peer_port = runtime
|
||||
.manager
|
||||
.try_acquire_stream(
|
||||
runtime.session.profile_key,
|
||||
runtime.session.profile.max_streams,
|
||||
runtime.session.client_ip,
|
||||
runtime.session.profile.public_addr,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(peer_port, 1);
|
||||
runtime.manager.release_stream(
|
||||
runtime.session.profile_key,
|
||||
runtime.session.client_ip,
|
||||
runtime.session.profile.public_addr,
|
||||
peer_port,
|
||||
);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
@@ -5,14 +5,13 @@ use bytes::{BufMut, Bytes, BytesMut};
|
||||
use super::{
|
||||
DownBatch, PendingClass, PollResult, QUEUE_ITEM_COST, QueuedFrame, SessionState, WebSession,
|
||||
};
|
||||
use crate::config::WebCarrier;
|
||||
use crate::web::frame::{self, FrameType};
|
||||
use crate::web::manager::ManagerError;
|
||||
|
||||
impl WebSession {
|
||||
/// Polls pending downlink frames with cursor replay and newest-poll-wins semantics.
|
||||
pub(crate) async fn poll_down(&self, cursor: u64) -> Result<PollResult, ManagerError> {
|
||||
if self.carrier() != WebCarrier::Https {
|
||||
if !self.carrier().is_multiplexed() {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let epoch = {
|
||||
@@ -167,7 +166,13 @@ impl WebSession {
|
||||
let Some(manager) = self.manager.upgrade() else {
|
||||
return false;
|
||||
};
|
||||
if !manager.try_reserve_pending(bytes, items, control, class == PendingClass::Downlink) {
|
||||
if !manager.try_reserve_pending(
|
||||
self.profile_key,
|
||||
bytes,
|
||||
items,
|
||||
control,
|
||||
class == PendingClass::Downlink,
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
state.pending_bytes += bytes;
|
||||
@@ -194,7 +199,7 @@ impl WebSession {
|
||||
state.pending_control_items = state.pending_control_items.saturating_sub(items);
|
||||
}
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
manager.release_pending(bytes, items, control);
|
||||
manager.release_pending(self.profile_key, bytes, items, control);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,7 +213,7 @@ impl WebSession {
|
||||
if amount == 0 {
|
||||
return true;
|
||||
}
|
||||
if self.carrier() == WebCarrier::HttpsLanes {
|
||||
if self.carrier().uses_lanes() {
|
||||
return self.queue_control_locked(
|
||||
state,
|
||||
FrameType::Window,
|
||||
@@ -257,7 +262,7 @@ impl WebSession {
|
||||
stream_id: u32,
|
||||
payload: &[u8],
|
||||
) -> bool {
|
||||
if self.carrier() == WebCarrier::HttpsLanes {
|
||||
if self.carrier().uses_lanes() {
|
||||
return self.queue_frame_locked(state, FrameType::Data, stream_id, payload, false);
|
||||
}
|
||||
let can_coalesce = state.pending_frames.back().is_some_and(|last| {
|
||||
@@ -290,7 +295,7 @@ impl WebSession {
|
||||
payload: &[u8],
|
||||
control: bool,
|
||||
) -> bool {
|
||||
if self.carrier() == WebCarrier::HttpsLanes {
|
||||
if self.carrier().uses_lanes() {
|
||||
return self.queue_lane_frame_locked(state, frame_type, stream_id, payload, control);
|
||||
}
|
||||
let cost = frame::HEADER_BYTES + payload.len() + QUEUE_ITEM_COST;
|
||||
@@ -409,7 +414,9 @@ mod tests {
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::config::{WebLimitsConfig, WebRuntimeProfile, WebSecretMode, WebTimeoutsConfig};
|
||||
use crate::config::{
|
||||
WebCarrier, WebLimitsConfig, WebRuntimeProfile, WebSecretMode, WebTimeoutsConfig,
|
||||
};
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
|
||||
fn session() -> Arc<WebSession> {
|
||||
@@ -420,6 +427,7 @@ mod tests {
|
||||
secret_mode: WebSecretMode::Plain,
|
||||
carrier: WebCarrier::Https,
|
||||
capability: [0; 32],
|
||||
key_fingerprint: "0000000000000000".to_string(),
|
||||
max_sessions: 1,
|
||||
max_streams: 1,
|
||||
max_streams_per_session: 1,
|
||||
@@ -428,6 +436,7 @@ mod tests {
|
||||
std::sync::Weak::<WebProcessRuntime>::new(),
|
||||
[1; 32],
|
||||
"192.0.2.10".parse().unwrap(),
|
||||
1,
|
||||
profile,
|
||||
[2; 32],
|
||||
WebLimitsConfig::default(),
|
||||
|
||||
@@ -113,6 +113,7 @@ impl WebSession {
|
||||
&mut state,
|
||||
&frames,
|
||||
&mut opened,
|
||||
&mut None,
|
||||
&mut unused_bytes,
|
||||
&mut unused_items,
|
||||
);
|
||||
@@ -137,7 +138,7 @@ impl WebSession {
|
||||
return result;
|
||||
}
|
||||
for (stream_id, peer_port) in opened {
|
||||
self.spawn_stream(stream_id, peer_port);
|
||||
self.spawn_stream(stream_id, peer_port, false);
|
||||
}
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
manager.record_up(body.len());
|
||||
@@ -151,7 +152,7 @@ impl WebSession {
|
||||
lane_id: u32,
|
||||
cursor: u64,
|
||||
) -> Result<PollResult, ManagerError> {
|
||||
if self.carrier() != WebCarrier::HttpsLanes || lane_id > frame::MAX_STREAM_ID {
|
||||
if !self.carrier().uses_lanes() || lane_id > frame::MAX_STREAM_ID {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let (epoch, notify) = {
|
||||
@@ -404,7 +405,7 @@ impl WebSession {
|
||||
|
||||
pub(super) fn remember_closed_locked(&self, state: &mut SessionState, stream_id: u32) {
|
||||
let evicted = remember_closed(state, stream_id, self.limits.max_tombstones_per_session);
|
||||
if self.carrier() != WebCarrier::HttpsLanes {
|
||||
if !self.carrier().uses_lanes() {
|
||||
return;
|
||||
}
|
||||
if let Some(evicted) = evicted {
|
||||
@@ -415,7 +416,7 @@ impl WebSession {
|
||||
}
|
||||
}
|
||||
|
||||
fn release_lane_locked(&self, state: &mut SessionState, lane_id: u32) {
|
||||
pub(super) fn release_lane_locked(&self, state: &mut SessionState, lane_id: u32) {
|
||||
let Some(mut lane) = state.carrier_lanes.remove(&lane_id) else {
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -15,6 +15,7 @@ fn session_with_limits(limits: WebLimitsConfig) -> Arc<WebSession> {
|
||||
secret_mode: WebSecretMode::Plain,
|
||||
carrier: WebCarrier::HttpsLanes,
|
||||
capability: [0; 32],
|
||||
key_fingerprint: "0000000000000000".to_string(),
|
||||
max_sessions: 1,
|
||||
max_streams: 2,
|
||||
max_streams_per_session: 2,
|
||||
@@ -23,6 +24,7 @@ fn session_with_limits(limits: WebLimitsConfig) -> Arc<WebSession> {
|
||||
std::sync::Weak::<WebProcessRuntime>::new(),
|
||||
[1; 32],
|
||||
"192.0.2.10".parse().unwrap(),
|
||||
1,
|
||||
profile,
|
||||
[2; 32],
|
||||
limits,
|
||||
|
||||
@@ -11,7 +11,6 @@ use super::{
|
||||
InboundChunk, PendingClass, QUEUE_ITEM_COST, SessionState, StreamState, WebSession,
|
||||
inbound_queue_cost,
|
||||
};
|
||||
use crate::config::WebCarrier;
|
||||
use crate::web::frame::{self, Frame, FrameType};
|
||||
use crate::web::manager::{ManagerError, TokenHash};
|
||||
|
||||
@@ -22,7 +21,7 @@ impl WebSession {
|
||||
sequence: u64,
|
||||
body: &[u8],
|
||||
) -> Result<u64, ManagerError> {
|
||||
if self.carrier() != WebCarrier::Https {
|
||||
if !self.carrier().is_multiplexed() {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
if self
|
||||
@@ -90,6 +89,7 @@ impl WebSession {
|
||||
&mut state,
|
||||
&frames,
|
||||
&mut opened,
|
||||
&mut None,
|
||||
&mut unused_bytes,
|
||||
&mut unused_items,
|
||||
);
|
||||
@@ -113,7 +113,7 @@ impl WebSession {
|
||||
return result;
|
||||
}
|
||||
for (stream_id, peer_port) in opened {
|
||||
self.spawn_stream(stream_id, peer_port);
|
||||
self.spawn_stream(stream_id, peer_port, false);
|
||||
}
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
manager.record_up(body.len());
|
||||
@@ -126,6 +126,7 @@ impl WebSession {
|
||||
state: &mut SessionState,
|
||||
frames: &[Frame<'_>],
|
||||
opened: &mut Vec<(u32, u16)>,
|
||||
reserved_open: &mut Option<(u32, u16)>,
|
||||
unused_bytes: &mut usize,
|
||||
unused_items: &mut usize,
|
||||
) -> bool {
|
||||
@@ -136,13 +137,31 @@ impl WebSession {
|
||||
let was_closed = state.closed_streams.contains(&value.stream_id);
|
||||
match value.frame_type {
|
||||
FrameType::Open => {
|
||||
let Some(peer_port) = self.reserve_stream_locked(state) else {
|
||||
self.remember_closed_locked(state, value.stream_id);
|
||||
if !self.queue_control_locked(state, FrameType::Close, value.stream_id, &[])
|
||||
let peer_port = match reserved_open.take() {
|
||||
Some((reserved_stream_id, peer_port))
|
||||
if reserved_stream_id == value.stream_id =>
|
||||
{
|
||||
peer_port
|
||||
}
|
||||
Some(reserved) => {
|
||||
*reserved_open = Some(reserved);
|
||||
return false;
|
||||
}
|
||||
continue;
|
||||
None => {
|
||||
let Some(peer_port) = self.reserve_stream_locked(state) else {
|
||||
self.remember_closed_locked(state, value.stream_id);
|
||||
if !self.queue_control_locked(
|
||||
state,
|
||||
FrameType::Close,
|
||||
value.stream_id,
|
||||
&[],
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
continue;
|
||||
};
|
||||
peer_port
|
||||
}
|
||||
};
|
||||
state.streams.insert(
|
||||
value.stream_id,
|
||||
@@ -331,7 +350,9 @@ mod tests {
|
||||
use super::*;
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use crate::config::{WebLimitsConfig, WebRuntimeProfile, WebSecretMode, WebTimeoutsConfig};
|
||||
use crate::config::{
|
||||
WebCarrier, WebLimitsConfig, WebRuntimeProfile, WebSecretMode, WebTimeoutsConfig,
|
||||
};
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
|
||||
fn session() -> Arc<WebSession> {
|
||||
@@ -342,6 +363,7 @@ mod tests {
|
||||
secret_mode: WebSecretMode::Plain,
|
||||
carrier: WebCarrier::Https,
|
||||
capability: [0; 32],
|
||||
key_fingerprint: "0000000000000000".to_string(),
|
||||
max_sessions: 1,
|
||||
max_streams: 1,
|
||||
max_streams_per_session: 1,
|
||||
@@ -350,6 +372,7 @@ mod tests {
|
||||
std::sync::Weak::<WebProcessRuntime>::new(),
|
||||
[1; 32],
|
||||
"192.0.2.10".parse().unwrap(),
|
||||
1,
|
||||
profile,
|
||||
[2; 32],
|
||||
WebLimitsConfig::default(),
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::uplink::{inbound_reservation, validate_batch};
|
||||
use super::{CarrierLane, PendingClass, WebSession, inbound_queue_cost};
|
||||
use crate::config::WebCarrier;
|
||||
use crate::web::frame;
|
||||
use crate::web::manager::ManagerError;
|
||||
|
||||
/// Pre-OPEN stream quota and synthetic tuple ownership for one WebSocket lane.
|
||||
pub(crate) struct WebSocketLaneReservation {
|
||||
session: Arc<WebSession>,
|
||||
lane_id: u32,
|
||||
peer_port: u16,
|
||||
transferred: bool,
|
||||
}
|
||||
|
||||
impl WebSocketLaneReservation {
|
||||
/// Returns the logical stream owned by this connection.
|
||||
pub(crate) fn lane_id(&self) -> u32 {
|
||||
self.lane_id
|
||||
}
|
||||
|
||||
fn transfer_to_stream(&mut self) {
|
||||
let removed = self
|
||||
.session
|
||||
.state
|
||||
.lock()
|
||||
.websocket_lane_reservations
|
||||
.remove(&self.lane_id);
|
||||
if removed == Some(self.peer_port) {
|
||||
self.transferred = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WebSocketLaneReservation {
|
||||
fn drop(&mut self) {
|
||||
if !self.transferred {
|
||||
self.session
|
||||
.release_websocket_lane_reservation(self.lane_id, self.peer_port);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl WebSession {
|
||||
/// Acquires stream quota and tuple ownership before a lane returns HTTP 101.
|
||||
pub(crate) fn reserve_websocket_lane(
|
||||
self: &Arc<Self>,
|
||||
lane_id: u32,
|
||||
) -> Result<WebSocketLaneReservation, ManagerError> {
|
||||
if self.carrier() != WebCarrier::WebsocketLanes
|
||||
|| lane_id == 0
|
||||
|| lane_id > frame::MAX_STREAM_ID
|
||||
{
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let mut state = self.state.lock();
|
||||
if state.closed {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
if state.active_peer_ports.len() >= self.profile.max_streams_per_session
|
||||
|| state.streams.contains_key(&lane_id)
|
||||
|| state.closed_streams.contains(&lane_id)
|
||||
|| state.websocket_lane_reservations.contains_key(&lane_id)
|
||||
{
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
let Some(manager) = self.manager.upgrade() else {
|
||||
return Err(ManagerError::Closed);
|
||||
};
|
||||
let Some(peer_port) = manager.try_acquire_stream(
|
||||
self.profile_key,
|
||||
self.profile.max_streams,
|
||||
self.client_ip,
|
||||
self.profile.public_addr,
|
||||
) else {
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
if !state.active_peer_ports.insert(peer_port) {
|
||||
manager.release_stream(
|
||||
self.profile_key,
|
||||
self.client_ip,
|
||||
self.profile.public_addr,
|
||||
peer_port,
|
||||
);
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
state.websocket_lane_reservations.insert(lane_id, peer_port);
|
||||
state.carrier_lanes.insert(lane_id, CarrierLane::new());
|
||||
Ok(WebSocketLaneReservation {
|
||||
session: Arc::clone(self),
|
||||
lane_id,
|
||||
peer_port,
|
||||
transferred: false,
|
||||
})
|
||||
}
|
||||
|
||||
/// Applies one ordered WebSocket lane message without closing sibling lanes.
|
||||
pub(crate) fn process_websocket_lane(
|
||||
self: &Arc<Self>,
|
||||
reservation: &mut WebSocketLaneReservation,
|
||||
sequence: u64,
|
||||
body: &[u8],
|
||||
) -> Result<(), ManagerError> {
|
||||
if !Arc::ptr_eq(self, &reservation.session)
|
||||
|| reservation.lane_id == 0
|
||||
|| reservation.lane_id > frame::MAX_STREAM_ID
|
||||
{
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let lane_id = reservation.lane_id;
|
||||
let frames = frame::parse_all(body, &self.limits).map_err(|_| ManagerError::Protocol)?;
|
||||
if frames
|
||||
.iter()
|
||||
.copied()
|
||||
.any(|value| value.stream_id != lane_id || frame::validate_client_shape(value).is_err())
|
||||
{
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let digest = Sha256::digest(body).into();
|
||||
let mut opened = Vec::new();
|
||||
let result = {
|
||||
let mut state = self.state.lock();
|
||||
if state.closed {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
if !reservation.transferred
|
||||
&& state.websocket_lane_reservations.get(&lane_id) != Some(&reservation.peer_port)
|
||||
{
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let Some(lane) = state.carrier_lanes.get_mut(&lane_id) else {
|
||||
return Err(ManagerError::Closed);
|
||||
};
|
||||
if sequence == 0 || sequence != lane.last_up_sequence.saturating_add(1) {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
if lane.up_active {
|
||||
return Err(ManagerError::Concurrent);
|
||||
}
|
||||
lane.up_active = true;
|
||||
if !validate_batch(&state, &frames) {
|
||||
if let Some(lane) = state.carrier_lanes.get_mut(&lane_id) {
|
||||
lane.up_active = false;
|
||||
}
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let (reserve_bytes, reserve_items) = inbound_reservation(&state, &frames);
|
||||
if !self.reserve_locked(
|
||||
&mut state,
|
||||
reserve_bytes,
|
||||
reserve_items,
|
||||
PendingClass::Uplink,
|
||||
) {
|
||||
if let Some(lane) = state.carrier_lanes.get_mut(&lane_id) {
|
||||
lane.up_active = false;
|
||||
}
|
||||
return Err(ManagerError::Backpressure);
|
||||
}
|
||||
let mut unused_bytes = reserve_bytes;
|
||||
let mut unused_items = reserve_items;
|
||||
let mut reserved_open =
|
||||
(!reservation.transferred).then_some((lane_id, reservation.peer_port));
|
||||
let applied = self.apply_batch_locked(
|
||||
&mut state,
|
||||
&frames,
|
||||
&mut opened,
|
||||
&mut reserved_open,
|
||||
&mut unused_bytes,
|
||||
&mut unused_items,
|
||||
);
|
||||
self.release_locked(&mut state, unused_bytes, unused_items, false);
|
||||
if let Some(lane) = state.carrier_lanes.get_mut(&lane_id) {
|
||||
lane.up_active = false;
|
||||
if applied {
|
||||
lane.last_up_sequence = sequence;
|
||||
lane.last_up_digest = digest;
|
||||
}
|
||||
}
|
||||
state.last_activity = Instant::now();
|
||||
applied.then_some(()).ok_or(ManagerError::Protocol)
|
||||
};
|
||||
result?;
|
||||
for (stream_id, peer_port) in opened {
|
||||
if stream_id != lane_id || peer_port != reservation.peer_port {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
if !self.spawn_stream(stream_id, peer_port, true) {
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
reservation.transfer_to_stream();
|
||||
}
|
||||
if !reservation.transferred {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
manager.record_up(body.len());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Ends one failed or disconnected lane without closing its parent session.
|
||||
pub(crate) fn close_websocket_lane(&self, lane_id: u32) {
|
||||
let reserved = {
|
||||
let mut state = self.state.lock();
|
||||
let reserved = state.websocket_lane_reservations.remove(&lane_id);
|
||||
if let Some(stream) = state.streams.remove(&lane_id) {
|
||||
let (bytes, items) = inbound_queue_cost(&stream.inbound);
|
||||
self.release_locked(&mut state, bytes, items, false);
|
||||
if let Some(waker) = stream.read_waker {
|
||||
waker.wake();
|
||||
}
|
||||
if let Some(waker) = stream.write_waker {
|
||||
waker.wake();
|
||||
}
|
||||
}
|
||||
self.remember_closed_locked(&mut state, lane_id);
|
||||
self.release_lane_locked(&mut state, lane_id);
|
||||
reserved
|
||||
};
|
||||
if let Some(peer_port) = reserved {
|
||||
self.release_websocket_lane_reservation(lane_id, peer_port);
|
||||
}
|
||||
}
|
||||
|
||||
fn release_websocket_lane_reservation(&self, lane_id: u32, peer_port: u16) {
|
||||
let removed = {
|
||||
let mut state = self.state.lock();
|
||||
if state.websocket_lane_reservations.get(&lane_id) == Some(&peer_port) {
|
||||
state.websocket_lane_reservations.remove(&lane_id);
|
||||
}
|
||||
self.release_lane_locked(&mut state, lane_id);
|
||||
state.active_peer_ports.remove(&peer_port)
|
||||
};
|
||||
if removed && let Some(manager) = self.manager.upgrade() {
|
||||
manager.release_stream(
|
||||
self.profile_key,
|
||||
self.client_ip,
|
||||
self.profile.public_addr,
|
||||
peer_port,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,135 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use tokio::sync::watch;
|
||||
|
||||
use super::*;
|
||||
use crate::config::{ProxyConfig, WebRuntimeConfig, WebRuntimeProfile, WebSecretMode};
|
||||
use crate::maestro::generation::{RuntimeGeneration, test_runtime_generation_with_admission};
|
||||
use crate::web::frame::FrameType;
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
|
||||
struct TestRuntime {
|
||||
session: Arc<WebSession>,
|
||||
manager: Arc<WebProcessRuntime>,
|
||||
generation: Arc<RuntimeGeneration>,
|
||||
}
|
||||
|
||||
impl TestRuntime {
|
||||
async fn shutdown(self) {
|
||||
self.session.close();
|
||||
self.session.wait().await;
|
||||
self.manager.shutdown().await;
|
||||
self.generation.stop_sessions().await;
|
||||
self.generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
|
||||
fn runtime(admission: bool) -> TestRuntime {
|
||||
let profile = Arc::new(WebRuntimeProfile {
|
||||
host: "proxy.example.com".to_string(),
|
||||
public_addr: "203.0.113.10:443".parse().unwrap(),
|
||||
user: "default".to_string(),
|
||||
secret_mode: WebSecretMode::Plain,
|
||||
carrier: WebCarrier::WebsocketLanes,
|
||||
capability: [7; 32],
|
||||
key_fingerprint: "0000000000000000".to_string(),
|
||||
max_sessions: 2,
|
||||
max_streams: 1,
|
||||
max_streams_per_session: 1,
|
||||
});
|
||||
let mut config = ProxyConfig::default();
|
||||
config.web.enabled = true;
|
||||
config.web.carrier = WebCarrier::WebsocketLanes;
|
||||
config.web.timeouts.shutdown_secs = 1;
|
||||
config.web.runtime = Some(Arc::new(WebRuntimeConfig {
|
||||
vhosts: BTreeMap::new(),
|
||||
profiles: vec![Arc::clone(&profile)],
|
||||
}));
|
||||
config.rebuild_runtime_user_auth().unwrap();
|
||||
let limits = config.web.limits.clone();
|
||||
let timeouts = config.web.timeouts.clone();
|
||||
let (_admission_tx, admission_rx) = watch::channel(admission);
|
||||
let generation = test_runtime_generation_with_admission(1, config, admission_rx);
|
||||
let manager = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
let session = WebSession::new(
|
||||
Arc::downgrade(&manager),
|
||||
[8; 32],
|
||||
"192.0.2.10".parse().unwrap(),
|
||||
1,
|
||||
profile,
|
||||
[7; 32],
|
||||
limits,
|
||||
timeouts,
|
||||
);
|
||||
TestRuntime {
|
||||
session,
|
||||
manager,
|
||||
generation,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_open_retains_stream_quota_until_lane_socket_teardown() {
|
||||
let runtime = runtime(false);
|
||||
let mut reservation = runtime.session.reserve_websocket_lane(7).unwrap();
|
||||
let open = frame::encode(FrameType::Open, 7, &[]);
|
||||
|
||||
assert_eq!(
|
||||
runtime
|
||||
.session
|
||||
.process_websocket_lane(&mut reservation, 1, &open),
|
||||
Err(ManagerError::Limit),
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.manager
|
||||
.try_acquire_stream(
|
||||
runtime.session.profile_key,
|
||||
runtime.session.profile.max_streams,
|
||||
runtime.session.client_ip,
|
||||
runtime.session.profile.public_addr,
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
|
||||
runtime.session.close_websocket_lane(7);
|
||||
drop(reservation);
|
||||
let peer_port = runtime
|
||||
.manager
|
||||
.try_acquire_stream(
|
||||
runtime.session.profile_key,
|
||||
runtime.session.profile.max_streams,
|
||||
runtime.session.client_ip,
|
||||
runtime.session.profile.public_addr,
|
||||
)
|
||||
.unwrap();
|
||||
runtime.manager.release_stream(
|
||||
runtime.session.profile_key,
|
||||
runtime.session.client_ip,
|
||||
runtime.session.profile.public_addr,
|
||||
peer_port,
|
||||
);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_lane_message_does_not_close_sibling_session_state() {
|
||||
let runtime = runtime(true);
|
||||
let mut reservation = runtime.session.reserve_websocket_lane(7).unwrap();
|
||||
let data = frame::encode(FrameType::Data, 7, &[1]);
|
||||
|
||||
assert_eq!(
|
||||
runtime
|
||||
.session
|
||||
.process_websocket_lane(&mut reservation, 1, &data),
|
||||
Err(ManagerError::Protocol),
|
||||
);
|
||||
assert!(!runtime.session.state.lock().closed);
|
||||
|
||||
runtime.session.close_websocket_lane(7);
|
||||
drop(reservation);
|
||||
assert!(runtime.session.reserve_websocket_lane(8).is_ok());
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
@@ -0,0 +1,487 @@
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::sanitize::{
|
||||
bounded_text, capture_limit, frame_error_name, frame_type_name, request_dynamic_bytes,
|
||||
response_dynamic_bytes, sanitized_headers, scrub_body, sensitive_values,
|
||||
};
|
||||
use super::store::{WebTraceStore, epoch_millis};
|
||||
use super::types::{
|
||||
TraceBodySnapshot, TraceBodyState, TraceDirection, TraceFrame, TraceHeader, TraceHttpRecord,
|
||||
TraceIdentity, TraceRecord, TraceRecordKind, TraceRoute, TraceTimings,
|
||||
};
|
||||
use crate::config::{WebDebugBodyCapture, WebDebugConfig, WebLimitsConfig, WebRuntimeProfile};
|
||||
use crate::web::frame::{self, FrameType};
|
||||
|
||||
const USER_AGENT_MAX_BYTES: usize = 512;
|
||||
|
||||
#[derive(Default)]
|
||||
struct BodyCapture {
|
||||
observed_bytes: u64,
|
||||
captured: Vec<u8>,
|
||||
truncated: bool,
|
||||
state: Option<TraceBodyState>,
|
||||
}
|
||||
|
||||
struct ExchangeState {
|
||||
method: String,
|
||||
path: String,
|
||||
route: TraceRoute,
|
||||
peer_ip: IpAddr,
|
||||
effective_ip: Option<IpAddr>,
|
||||
user_agent: Option<String>,
|
||||
identity: TraceIdentity,
|
||||
request_headers: Vec<TraceHeader>,
|
||||
response_headers: Vec<TraceHeader>,
|
||||
request_body: BodyCapture,
|
||||
response_body: BodyCapture,
|
||||
status: Option<u16>,
|
||||
frames: Vec<TraceFrame>,
|
||||
timings: TraceTimings,
|
||||
redactions: Vec<Zeroizing<Vec<u8>>>,
|
||||
body_capture_blocked: bool,
|
||||
}
|
||||
|
||||
/// One in-flight request-to-response capture with process-wide byte leases.
|
||||
pub(crate) struct HttpTraceExchange {
|
||||
store: Arc<WebTraceStore>,
|
||||
epoch: u64,
|
||||
policy: Arc<WebDebugConfig>,
|
||||
started: Instant,
|
||||
started_epoch_millis: u64,
|
||||
state: Mutex<ExchangeState>,
|
||||
reserved: AtomicUsize,
|
||||
committed: AtomicBool,
|
||||
}
|
||||
|
||||
impl HttpTraceExchange {
|
||||
/// Creates one enabled exchange after the store reserved its base record lease.
|
||||
pub(super) fn new<B>(
|
||||
store: Arc<WebTraceStore>,
|
||||
epoch: u64,
|
||||
policy: Arc<WebDebugConfig>,
|
||||
request: &hyper::Request<B>,
|
||||
peer_ip: IpAddr,
|
||||
base_reservation: usize,
|
||||
) -> Arc<Self> {
|
||||
let started = Instant::now();
|
||||
let started_epoch_millis = epoch_millis();
|
||||
let dynamic = request_dynamic_bytes(request, &policy);
|
||||
let dynamic_reserved = store.try_reserve(dynamic);
|
||||
let (method, path, user_agent, request_headers, redactions) = if dynamic_reserved {
|
||||
(
|
||||
request.method().as_str().to_string(),
|
||||
request.uri().path().to_string(),
|
||||
request
|
||||
.headers()
|
||||
.get(hyper::header::USER_AGENT)
|
||||
.map(|value| bounded_text(value.as_bytes(), USER_AGENT_MAX_BYTES)),
|
||||
policy
|
||||
.capture_headers
|
||||
.then(|| sanitized_headers(request.headers()))
|
||||
.unwrap_or_default(),
|
||||
sensitive_values(request.headers(), request.uri().query()),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
bounded_text(request.method().as_str().as_bytes(), 32),
|
||||
bounded_text(request.uri().path().as_bytes(), 512),
|
||||
None,
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
)
|
||||
};
|
||||
Arc::new(Self {
|
||||
store,
|
||||
epoch,
|
||||
policy,
|
||||
started,
|
||||
started_epoch_millis,
|
||||
state: Mutex::new(ExchangeState {
|
||||
method,
|
||||
path,
|
||||
route: TraceRoute::Unknown,
|
||||
peer_ip,
|
||||
effective_ip: None,
|
||||
user_agent,
|
||||
identity: TraceIdentity::default(),
|
||||
request_headers,
|
||||
response_headers: Vec::new(),
|
||||
request_body: BodyCapture::default(),
|
||||
response_body: BodyCapture::default(),
|
||||
status: None,
|
||||
frames: Vec::new(),
|
||||
timings: TraceTimings::default(),
|
||||
redactions,
|
||||
body_capture_blocked: !dynamic_reserved,
|
||||
}),
|
||||
reserved: AtomicUsize::new(
|
||||
base_reservation + if dynamic_reserved { dynamic } else { 0 },
|
||||
),
|
||||
committed: AtomicBool::new(false),
|
||||
})
|
||||
}
|
||||
|
||||
/// Sets the final request route before body polling or decoy forwarding.
|
||||
pub(crate) fn set_route(&self, route: TraceRoute) {
|
||||
self.state.lock().route = route;
|
||||
}
|
||||
|
||||
/// Sets the trusted effective client address after proxy-header validation.
|
||||
pub(crate) fn set_effective_ip(&self, client_ip: IpAddr) {
|
||||
self.state.lock().effective_ip = Some(client_ip);
|
||||
}
|
||||
|
||||
/// Binds non-secret profile and process session identity.
|
||||
pub(crate) fn bind_profile(&self, profile: &WebRuntimeProfile, session_id: u64) {
|
||||
let dynamic = profile
|
||||
.user
|
||||
.len()
|
||||
.saturating_add(profile.key_fingerprint.len());
|
||||
let mut state = self.state.lock();
|
||||
state.identity.session_id = Some(session_id);
|
||||
if self.reserve(dynamic) {
|
||||
state.identity.user = Some(profile.user.clone());
|
||||
state.identity.key_fingerprint = Some(profile.key_fingerprint.clone());
|
||||
}
|
||||
}
|
||||
|
||||
/// Binds an already resolved non-secret session identity.
|
||||
pub(crate) fn bind_identity(&self, identity: TraceIdentity) {
|
||||
let dynamic = identity
|
||||
.user
|
||||
.as_ref()
|
||||
.map_or(0, String::len)
|
||||
.saturating_add(identity.key_fingerprint.as_ref().map_or(0, String::len));
|
||||
let mut state = self.state.lock();
|
||||
state.identity.session_id = identity.session_id;
|
||||
if self.reserve(dynamic) {
|
||||
state.identity.user = identity.user;
|
||||
state.identity.key_fingerprint = identity.key_fingerprint;
|
||||
}
|
||||
}
|
||||
|
||||
/// Registers an ephemeral credential for body scrubbing before commit.
|
||||
pub(crate) fn register_redaction(&self, value: &[u8]) {
|
||||
if value.is_empty() {
|
||||
return;
|
||||
}
|
||||
if !self.reserve(value.len()) {
|
||||
self.block_body_capture();
|
||||
return;
|
||||
}
|
||||
self.state
|
||||
.lock()
|
||||
.redactions
|
||||
.push(Zeroizing::new(value.to_vec()));
|
||||
}
|
||||
|
||||
/// Captures response status and sanitized headers at handler completion.
|
||||
pub(crate) fn response_ready<B>(&self, response: &hyper::Response<B>) {
|
||||
let dynamic = response_dynamic_bytes(response, &self.policy);
|
||||
let reserved = self.reserve(dynamic);
|
||||
let mut state = self.state.lock();
|
||||
state.status = Some(response.status().as_u16());
|
||||
if self.policy.capture_headers && reserved {
|
||||
state.response_headers = sanitized_headers(response.headers());
|
||||
}
|
||||
if reserved {
|
||||
state
|
||||
.redactions
|
||||
.extend(sensitive_values(response.headers(), None));
|
||||
} else if dynamic != 0 {
|
||||
state.body_capture_blocked = true;
|
||||
state.request_body.captured.clear();
|
||||
state.response_body.captured.clear();
|
||||
state.request_body.truncated = true;
|
||||
state.response_body.truncated = true;
|
||||
}
|
||||
if self.policy.capture_timings {
|
||||
state.timings.response_ready_us = Some(self.elapsed_us());
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends one body data frame without changing the proxied bytes.
|
||||
pub(crate) fn body_data(&self, direction: TraceDirection, data: &[u8]) {
|
||||
let mut state = self.state.lock();
|
||||
let route = state.route;
|
||||
let body_capture_blocked = state.body_capture_blocked;
|
||||
let body = match direction {
|
||||
TraceDirection::Request => &mut state.request_body,
|
||||
TraceDirection::Response => &mut state.response_body,
|
||||
};
|
||||
body.observed_bytes = body
|
||||
.observed_bytes
|
||||
.saturating_add(u64::try_from(data.len()).unwrap_or(u64::MAX));
|
||||
if data.is_empty() {
|
||||
return;
|
||||
}
|
||||
if body_capture_blocked {
|
||||
body.truncated |= !data.is_empty();
|
||||
return;
|
||||
}
|
||||
let Some(limit) = capture_limit(&self.policy, route, self.store.max_carrier_body_bytes())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
if body.captured.len() >= limit {
|
||||
if !data.is_empty() && !body.truncated {
|
||||
self.store.record_truncation();
|
||||
}
|
||||
body.truncated |= !data.is_empty();
|
||||
return;
|
||||
}
|
||||
if body.captured.capacity() == 0 {
|
||||
if !self.reserve(limit) {
|
||||
body.truncated = true;
|
||||
return;
|
||||
}
|
||||
body.captured = Vec::with_capacity(limit);
|
||||
}
|
||||
let take = data.len().min(limit - body.captured.len());
|
||||
body.captured.extend_from_slice(&data[..take]);
|
||||
if take < data.len() {
|
||||
if !body.truncated {
|
||||
self.store.record_truncation();
|
||||
}
|
||||
body.truncated = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Marks one request or response body terminal state.
|
||||
pub(crate) fn body_finished(&self, direction: TraceDirection, terminal: TraceBodyState) {
|
||||
let mut state = self.state.lock();
|
||||
let body = match direction {
|
||||
TraceDirection::Request => &mut state.request_body,
|
||||
TraceDirection::Response => &mut state.response_body,
|
||||
};
|
||||
if body.state.is_none() {
|
||||
body.state = Some(terminal);
|
||||
}
|
||||
if self.policy.capture_timings {
|
||||
match direction {
|
||||
TraceDirection::Request => state.timings.request_body_us = Some(self.elapsed_us()),
|
||||
TraceDirection::Response => {
|
||||
state.timings.response_body_us = Some(self.elapsed_us())
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(state);
|
||||
if direction == TraceDirection::Response {
|
||||
self.commit();
|
||||
}
|
||||
}
|
||||
|
||||
/// Attaches parsed carrier frame metadata from one complete bounded body.
|
||||
pub(crate) fn record_frames(
|
||||
&self,
|
||||
direction: TraceDirection,
|
||||
body: &[u8],
|
||||
limits: &WebLimitsConfig,
|
||||
) {
|
||||
if !self.policy.capture_frames {
|
||||
return;
|
||||
}
|
||||
let estimated_frames = body
|
||||
.len()
|
||||
.div_ceil(frame::HEADER_BYTES)
|
||||
.clamp(1, limits.max_frames_per_body);
|
||||
let reservation = estimated_frames.saturating_mul(std::mem::size_of::<TraceFrame>());
|
||||
if !self.reserve(reservation) {
|
||||
return;
|
||||
}
|
||||
let frames = match frame::parse_all(body, limits) {
|
||||
Ok(frames) => frames
|
||||
.into_iter()
|
||||
.map(|frame| TraceFrame {
|
||||
direction,
|
||||
frame_type: Some(frame_type_name(frame.frame_type)),
|
||||
stream_id: Some(frame.stream_id),
|
||||
payload_len: Some(frame.payload.len()),
|
||||
window_delta: (frame.frame_type == FrameType::Window)
|
||||
.then(|| frame::window_amount(frame.payload).ok())
|
||||
.flatten(),
|
||||
parse_error: None,
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
Err(error) => vec![TraceFrame {
|
||||
direction,
|
||||
frame_type: None,
|
||||
stream_id: None,
|
||||
payload_len: None,
|
||||
window_delta: None,
|
||||
parse_error: Some(frame_error_name(error)),
|
||||
}],
|
||||
};
|
||||
self.state.lock().frames.extend(frames);
|
||||
}
|
||||
|
||||
/// Commits once after response body consumption or drop.
|
||||
pub(crate) fn commit(&self) {
|
||||
if self.committed.swap(true, Ordering::AcqRel) {
|
||||
return;
|
||||
}
|
||||
let reserved = self.reserved.load(Ordering::Acquire);
|
||||
let record = self.build_record();
|
||||
if !self.store.try_commit(record, reserved, self.epoch) {
|
||||
self.store.release(reserved);
|
||||
}
|
||||
}
|
||||
|
||||
fn reserve(&self, bytes: usize) -> bool {
|
||||
if bytes == 0 {
|
||||
return true;
|
||||
}
|
||||
if self.store.try_reserve(bytes) {
|
||||
self.reserved.fetch_add(bytes, Ordering::AcqRel);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
fn block_body_capture(&self) {
|
||||
let mut state = self.state.lock();
|
||||
state.body_capture_blocked = true;
|
||||
state.request_body.captured.clear();
|
||||
state.response_body.captured.clear();
|
||||
state.request_body.truncated = true;
|
||||
state.response_body.truncated = true;
|
||||
}
|
||||
|
||||
fn build_record(&self) -> TraceRecord {
|
||||
let mut state = self.state.lock();
|
||||
let redactions = std::mem::take(&mut state.redactions);
|
||||
scrub_body(&mut state.request_body.captured, &redactions);
|
||||
scrub_body(&mut state.response_body.captured, &redactions);
|
||||
let request_body = body_snapshot(&self.policy, &mut state.request_body);
|
||||
let response_body = body_snapshot(&self.policy, &mut state.response_body);
|
||||
TraceRecord {
|
||||
seq: self.store.next_record_seq(),
|
||||
epoch_millis: self.started_epoch_millis,
|
||||
peer_ip: Some(state.peer_ip),
|
||||
effective_ip: state.effective_ip,
|
||||
user_agent: state.user_agent.take(),
|
||||
identity: std::mem::take(&mut state.identity),
|
||||
kind: TraceRecordKind::Http(TraceHttpRecord {
|
||||
method: std::mem::take(&mut state.method),
|
||||
path: std::mem::take(&mut state.path),
|
||||
route: state.route,
|
||||
request_headers: std::mem::take(&mut state.request_headers),
|
||||
response_headers: std::mem::take(&mut state.response_headers),
|
||||
request_body,
|
||||
status: state.status,
|
||||
response_body,
|
||||
frames: std::mem::take(&mut state.frames),
|
||||
timings: self
|
||||
.policy
|
||||
.capture_timings
|
||||
.then(|| std::mem::take(&mut state.timings)),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn elapsed_us(&self) -> u64 {
|
||||
self.started.elapsed().as_micros().min(u128::from(u64::MAX)) as u64
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for HttpTraceExchange {
|
||||
fn drop(&mut self) {
|
||||
if !self.committed.load(Ordering::Acquire) {
|
||||
self.body_finished(TraceDirection::Request, TraceBodyState::Aborted);
|
||||
self.body_finished(TraceDirection::Response, TraceBodyState::Aborted);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn body_snapshot(policy: &WebDebugConfig, body: &mut BodyCapture) -> Option<TraceBodySnapshot> {
|
||||
(policy.body_capture != WebDebugBodyCapture::Off).then(|| TraceBodySnapshot {
|
||||
observed_bytes: body.observed_bytes,
|
||||
captured: std::mem::take(&mut body.captured),
|
||||
truncated: body.truncated,
|
||||
state: body.state.unwrap_or(TraceBodyState::Aborted),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn request_response_capture_redacts_credentials_and_omits_query() {
|
||||
let request_token = "request-token-0123456789";
|
||||
let capability = "capability-0123456789";
|
||||
let response_token = "response-token-0123456789";
|
||||
let request = hyper::Request::builder()
|
||||
.uri(format!("/?bridge={capability}"))
|
||||
.header("authorization", format!("Bearer {request_token}"))
|
||||
.body(())
|
||||
.unwrap();
|
||||
let mut policy = WebDebugConfig::default();
|
||||
policy.enabled = true;
|
||||
policy.body_capture = WebDebugBodyCapture::Prefix;
|
||||
policy.body_prefix_bytes = 256;
|
||||
let mut limits = WebLimitsConfig::default();
|
||||
limits.debug_records_capacity = 4;
|
||||
limits.debug_bytes_global = 16 * 1024;
|
||||
let store = WebTraceStore::new(policy, &limits);
|
||||
let exchange = store
|
||||
.begin_http(&request, "192.0.2.30".parse().unwrap())
|
||||
.unwrap();
|
||||
exchange.set_route(TraceRoute::Bridge);
|
||||
exchange.body_data(
|
||||
TraceDirection::Request,
|
||||
format!("{request_token}:{capability}").as_bytes(),
|
||||
);
|
||||
exchange.body_finished(TraceDirection::Request, TraceBodyState::Complete);
|
||||
|
||||
let response = hyper::Response::builder()
|
||||
.status(hyper::StatusCode::OK)
|
||||
.header("x-session-token", response_token)
|
||||
.body(())
|
||||
.unwrap();
|
||||
exchange.response_ready(&response);
|
||||
exchange.body_data(TraceDirection::Response, response_token.as_bytes());
|
||||
exchange.body_finished(TraceDirection::Response, TraceBodyState::Complete);
|
||||
|
||||
let records = store.snapshot_matching(|_| true);
|
||||
assert_eq!(records.len(), 1);
|
||||
let TraceRecordKind::Http(http) = &records[0].record.kind else {
|
||||
panic!("expected HTTP debug record");
|
||||
};
|
||||
assert_eq!(http.path, "/");
|
||||
assert_eq!(http.route, TraceRoute::Bridge);
|
||||
assert!(
|
||||
http.request_headers
|
||||
.iter()
|
||||
.any(|header| header.name == "authorization" && header.value.is_none())
|
||||
);
|
||||
assert!(
|
||||
http.response_headers
|
||||
.iter()
|
||||
.any(|header| header.name == "x-session-token" && header.value.is_none())
|
||||
);
|
||||
let request_body = http.request_body.as_ref().unwrap();
|
||||
let response_body = http.response_body.as_ref().unwrap();
|
||||
for secret in [request_token.as_bytes(), capability.as_bytes()] {
|
||||
assert!(
|
||||
!request_body
|
||||
.captured
|
||||
.windows(secret.len())
|
||||
.any(|value| value == secret)
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
!response_body
|
||||
.captured
|
||||
.windows(response_token.len())
|
||||
.any(|value| value == response_token.as_bytes())
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
//! Process-owned bounded WEB request, frame, and lifecycle debug recording.
|
||||
|
||||
// HTTP exchange ownership binds byte leases to one request and response.
|
||||
mod exchange;
|
||||
// Header, credential, body, and frame sanitization stays independent from storage.
|
||||
mod sanitize;
|
||||
// The process-wide store owns policy epochs, ring eviction, and render admission.
|
||||
mod store;
|
||||
// Closed record types define the status-page data contract.
|
||||
mod types;
|
||||
|
||||
pub(crate) use exchange::HttpTraceExchange;
|
||||
pub(crate) use store::{StoredTraceRecord, WebTraceStore, epoch_millis as store_epoch_millis};
|
||||
pub(crate) use types::{
|
||||
TraceBodySnapshot, TraceBodyState, TraceDirection, TraceFrame, TraceHeader, TraceIdentity,
|
||||
TraceLifecycleEvent, TraceRecord, TraceRecordKind, TraceRoute, TraceWebSocketContext,
|
||||
};
|
||||
@@ -0,0 +1,298 @@
|
||||
use hyper::header;
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::types::{TraceHeader, TraceRoute};
|
||||
use crate::config::{WebDebugBodyCapture, WebDebugConfig};
|
||||
use crate::web::frame::{FrameError, FrameType};
|
||||
|
||||
const USER_AGENT_MAX_BYTES: usize = 512;
|
||||
const MIN_REDACTION_BYTES: usize = 8;
|
||||
const MAX_REDACTION_BYTES: usize = 512;
|
||||
const MAX_HEADER_REDACTIONS: usize = 16;
|
||||
|
||||
/// Calculates a conservative request metadata and credential lease.
|
||||
pub(super) fn request_dynamic_bytes<B>(
|
||||
request: &hyper::Request<B>,
|
||||
policy: &WebDebugConfig,
|
||||
) -> usize {
|
||||
request
|
||||
.method()
|
||||
.as_str()
|
||||
.len()
|
||||
.saturating_add(request.uri().path().len())
|
||||
.saturating_add(
|
||||
request
|
||||
.headers()
|
||||
.get(header::USER_AGENT)
|
||||
.map_or(0, |value| {
|
||||
lossy_text_reservation(value.as_bytes(), USER_AGENT_MAX_BYTES)
|
||||
}),
|
||||
)
|
||||
.saturating_add(
|
||||
policy
|
||||
.capture_headers
|
||||
.then(|| sanitized_header_bytes(request.headers()))
|
||||
.unwrap_or(0),
|
||||
)
|
||||
.saturating_add(sensitive_value_bytes(
|
||||
request.headers(),
|
||||
request.uri().query(),
|
||||
))
|
||||
}
|
||||
|
||||
/// Calculates a conservative response metadata and credential lease.
|
||||
pub(super) fn response_dynamic_bytes<B>(
|
||||
response: &hyper::Response<B>,
|
||||
policy: &WebDebugConfig,
|
||||
) -> usize {
|
||||
policy
|
||||
.capture_headers
|
||||
.then(|| sanitized_header_bytes(response.headers()))
|
||||
.unwrap_or(0)
|
||||
.saturating_add(sensitive_value_bytes(response.headers(), None))
|
||||
}
|
||||
|
||||
/// Copies header names and only allowlisted bounded values.
|
||||
pub(super) fn sanitized_headers(headers: &hyper::HeaderMap) -> Vec<TraceHeader> {
|
||||
headers
|
||||
.iter()
|
||||
.map(|(name, value)| TraceHeader {
|
||||
name: name.as_str().to_string(),
|
||||
value: header_value_allowed(name).then(|| bounded_text(value.as_bytes(), 4096)),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Copies a bounded set of ephemeral credentials into zeroizing scrub patterns.
|
||||
pub(super) fn sensitive_values(
|
||||
headers: &hyper::HeaderMap,
|
||||
query: Option<&str>,
|
||||
) -> Vec<Zeroizing<Vec<u8>>> {
|
||||
let mut values = Vec::new();
|
||||
for name in [
|
||||
"authorization",
|
||||
"proxy-authorization",
|
||||
"x-session-token",
|
||||
"sec-websocket-protocol",
|
||||
] {
|
||||
for value in headers.get_all(name) {
|
||||
push_redaction(&mut values, value.as_bytes());
|
||||
if let Some(token) = value.as_bytes().strip_prefix(b"Bearer ") {
|
||||
push_redaction(&mut values, token);
|
||||
}
|
||||
if values.len() >= MAX_HEADER_REDACTIONS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if values.len() >= MAX_HEADER_REDACTIONS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if let Some(capability) = query.and_then(|query| query.strip_prefix("bridge=")) {
|
||||
push_redaction(&mut values, capability.as_bytes());
|
||||
}
|
||||
values
|
||||
}
|
||||
|
||||
/// Converts at most `limit` raw bytes into loss-tolerant display text.
|
||||
pub(super) fn bounded_text(value: &[u8], limit: usize) -> String {
|
||||
String::from_utf8_lossy(&value[..value.len().min(limit)]).into_owned()
|
||||
}
|
||||
|
||||
/// Overwrites complete credentials and a prefix cut by body truncation.
|
||||
pub(super) fn scrub_body(body: &mut [u8], redactions: &[Zeroizing<Vec<u8>>]) {
|
||||
for secret in redactions.iter().filter(|secret| !secret.is_empty()) {
|
||||
if secret.len() <= body.len() {
|
||||
for offset in 0..=body.len() - secret.len() {
|
||||
if body[offset..].starts_with(secret) {
|
||||
body[offset..offset + secret.len()].fill(b'*');
|
||||
}
|
||||
}
|
||||
}
|
||||
let overlap = secret.len().min(body.len());
|
||||
for length in (1..=overlap).rev() {
|
||||
if body.ends_with(&secret[..length]) {
|
||||
let start = body.len() - length;
|
||||
body[start..].fill(b'*');
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves the route-sensitive retained body allocation ceiling.
|
||||
pub(super) fn capture_limit(
|
||||
policy: &WebDebugConfig,
|
||||
route: TraceRoute,
|
||||
max_carrier_body_bytes: usize,
|
||||
) -> Option<usize> {
|
||||
match policy.body_capture {
|
||||
WebDebugBodyCapture::Off | WebDebugBodyCapture::Metadata => None,
|
||||
WebDebugBodyCapture::Prefix => Some(if decoy_route(route) {
|
||||
policy.decoy_body_prefix_bytes
|
||||
} else {
|
||||
policy.body_prefix_bytes
|
||||
}),
|
||||
WebDebugBodyCapture::Full => Some(if decoy_route(route) {
|
||||
policy.decoy_body_prefix_bytes
|
||||
} else {
|
||||
max_carrier_body_bytes
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a closed display label for one parsed frame type.
|
||||
pub(super) fn frame_type_name(frame_type: FrameType) -> &'static str {
|
||||
match frame_type {
|
||||
FrameType::Open => "OPEN",
|
||||
FrameType::Data => "DATA",
|
||||
FrameType::Close => "CLOSE",
|
||||
FrameType::Window => "WINDOW",
|
||||
FrameType::Ping => "PING",
|
||||
FrameType::Pong => "PONG",
|
||||
FrameType::Hello => "HELLO",
|
||||
FrameType::Welcome => "WELCOME",
|
||||
FrameType::Bye => "BYE",
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a closed display label for one frame parse failure.
|
||||
pub(super) fn frame_error_name(error: FrameError) -> &'static str {
|
||||
match error {
|
||||
FrameError::EmptyBatch => "empty_batch",
|
||||
FrameError::TooManyFrames => "too_many_frames",
|
||||
FrameError::Incomplete => "incomplete",
|
||||
FrameError::PayloadLimit => "payload_limit",
|
||||
FrameError::UnknownType => "unknown_type",
|
||||
FrameError::InvalidShape => "invalid_shape",
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitized_header_bytes(headers: &hyper::HeaderMap) -> usize {
|
||||
headers.iter().fold(0usize, |total, (name, value)| {
|
||||
total
|
||||
.saturating_add(std::mem::size_of::<TraceHeader>())
|
||||
.saturating_add(name.as_str().len())
|
||||
.saturating_add(
|
||||
header_value_allowed(name)
|
||||
.then(|| lossy_text_reservation(value.as_bytes(), 4096))
|
||||
.unwrap_or(0),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn header_value_allowed(name: &hyper::header::HeaderName) -> bool {
|
||||
matches!(
|
||||
name.as_str(),
|
||||
"host"
|
||||
| "user-agent"
|
||||
| "content-type"
|
||||
| "content-length"
|
||||
| "origin"
|
||||
| "x-forwarded-for"
|
||||
| "x-up-seq"
|
||||
| "x-up-ack"
|
||||
| "x-down-cursor"
|
||||
| "x-lane-id"
|
||||
| "x-lane-closed"
|
||||
| "x-carrier-mode"
|
||||
| "retry-after"
|
||||
| "cache-control"
|
||||
| "etag"
|
||||
| "if-none-match"
|
||||
| "accept"
|
||||
| "accept-encoding"
|
||||
)
|
||||
}
|
||||
|
||||
fn sensitive_value_bytes(headers: &hyper::HeaderMap, query: Option<&str>) -> usize {
|
||||
let mut total = 0usize;
|
||||
let mut retained = 0usize;
|
||||
for name in [
|
||||
"authorization",
|
||||
"proxy-authorization",
|
||||
"x-session-token",
|
||||
"sec-websocket-protocol",
|
||||
] {
|
||||
for value in headers.get_all(name) {
|
||||
for candidate in [
|
||||
Some(value.as_bytes()),
|
||||
value.as_bytes().strip_prefix(b"Bearer "),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if (MIN_REDACTION_BYTES..=MAX_REDACTION_BYTES).contains(&candidate.len()) {
|
||||
total = total
|
||||
.saturating_add(std::mem::size_of::<Vec<u8>>())
|
||||
.saturating_add(candidate.len());
|
||||
retained += 1;
|
||||
if retained >= MAX_HEADER_REDACTIONS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if retained >= MAX_HEADER_REDACTIONS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if retained >= MAX_HEADER_REDACTIONS {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if let Some(capability) = query.and_then(|query| query.strip_prefix("bridge="))
|
||||
&& (MIN_REDACTION_BYTES..=MAX_REDACTION_BYTES).contains(&capability.len())
|
||||
{
|
||||
total = total
|
||||
.saturating_add(std::mem::size_of::<Vec<u8>>())
|
||||
.saturating_add(capability.len());
|
||||
}
|
||||
total
|
||||
}
|
||||
|
||||
fn lossy_text_reservation(value: &[u8], limit: usize) -> usize {
|
||||
value.len().min(limit).saturating_mul(3)
|
||||
}
|
||||
|
||||
fn decoy_route(route: TraceRoute) -> bool {
|
||||
matches!(route, TraceRoute::Unknown | TraceRoute::Decoy)
|
||||
}
|
||||
|
||||
fn push_redaction(values: &mut Vec<Zeroizing<Vec<u8>>>, value: &[u8]) {
|
||||
if value.len() < MIN_REDACTION_BYTES || value.len() > MAX_REDACTION_BYTES {
|
||||
return;
|
||||
}
|
||||
if !values.iter().any(|existing| existing.as_slice() == value) {
|
||||
values.push(Zeroizing::new(value.to_vec()));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn full_capture_keeps_decoy_bodies_prefix_bounded() {
|
||||
let mut policy = WebDebugConfig::default();
|
||||
policy.body_capture = WebDebugBodyCapture::Full;
|
||||
policy.decoy_body_prefix_bytes = 123;
|
||||
assert_eq!(capture_limit(&policy, TraceRoute::Decoy, 4096), Some(123));
|
||||
assert_eq!(capture_limit(&policy, TraceRoute::Uplink, 4096), Some(4096));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scrub_removes_complete_and_prefix_truncated_credentials() {
|
||||
let secret = Zeroizing::new(b"credential-value".to_vec());
|
||||
let mut complete = b"before credential-value after".to_vec();
|
||||
scrub_body(&mut complete, std::slice::from_ref(&secret));
|
||||
assert!(
|
||||
!complete
|
||||
.windows(secret.len())
|
||||
.any(|value| value == secret.as_slice())
|
||||
);
|
||||
|
||||
let mut truncated = b"before credent".to_vec();
|
||||
scrub_body(&mut truncated, &[secret]);
|
||||
assert!(truncated.ends_with(b"*******"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,439 @@
|
||||
use std::collections::VecDeque;
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use parking_lot::Mutex;
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
||||
|
||||
use super::exchange::HttpTraceExchange;
|
||||
use super::types::{
|
||||
TraceIdentity, TraceLifecycleEvent, TraceLifecycleRecord, TraceRecord, TraceRecordKind,
|
||||
};
|
||||
use crate::config::{WebDebugConfig, WebLimitsConfig};
|
||||
|
||||
// WebSocket message capture is isolated from HTTP exchange storage.
|
||||
mod websocket;
|
||||
|
||||
const BASE_RECORD_RESERVATION: usize = 1024;
|
||||
|
||||
struct RingState {
|
||||
records: VecDeque<Arc<StoredTraceRecord>>,
|
||||
}
|
||||
|
||||
/// One retained record whose lease survives status-page snapshots.
|
||||
pub(crate) struct StoredTraceRecord {
|
||||
/// Immutable trace record.
|
||||
pub(crate) record: TraceRecord,
|
||||
bytes: usize,
|
||||
used_bytes: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Drop for StoredTraceRecord {
|
||||
fn drop(&mut self) {
|
||||
self.used_bytes.fetch_sub(self.bytes, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
/// Point-in-time process-owned trace counters and ring bounds.
|
||||
pub(crate) struct TraceStoreStatus {
|
||||
/// Current debug policy.
|
||||
pub(crate) policy: Arc<WebDebugConfig>,
|
||||
/// Retained record count.
|
||||
pub(crate) records: usize,
|
||||
/// Configured record capacity.
|
||||
pub(crate) records_capacity: usize,
|
||||
/// Retained plus in-flight byte leases.
|
||||
pub(crate) used_bytes: usize,
|
||||
/// Configured byte capacity.
|
||||
pub(crate) bytes_capacity: usize,
|
||||
/// Records dropped on commit-lock contention.
|
||||
pub(crate) contention_drops: u64,
|
||||
/// Oldest records evicted by ring capacity.
|
||||
pub(crate) evictions: u64,
|
||||
/// Body or metadata captures truncated by policy or byte capacity.
|
||||
pub(crate) byte_truncations: u64,
|
||||
/// Earliest retained record sequence.
|
||||
pub(crate) earliest_seq: Option<u64>,
|
||||
/// Latest retained record sequence.
|
||||
pub(crate) latest_seq: Option<u64>,
|
||||
}
|
||||
|
||||
/// Process-owned bounded WEB debug trace store.
|
||||
pub(crate) struct WebTraceStore {
|
||||
policy: ArcSwap<WebDebugConfig>,
|
||||
policy_update: Mutex<()>,
|
||||
enabled: AtomicBool,
|
||||
epoch: AtomicU64,
|
||||
records_capacity: usize,
|
||||
bytes_capacity: usize,
|
||||
max_carrier_body_bytes: usize,
|
||||
frame_limits: WebLimitsConfig,
|
||||
used_bytes: Arc<AtomicUsize>,
|
||||
ring: Mutex<RingState>,
|
||||
next_record_seq: AtomicU64,
|
||||
next_session_id: AtomicU64,
|
||||
contention_drops: AtomicU64,
|
||||
evictions: AtomicU64,
|
||||
byte_truncations: AtomicU64,
|
||||
renders: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
impl WebTraceStore {
|
||||
/// Creates a process-owned store from restart-only capacity and initial policy.
|
||||
pub(crate) fn new(policy: WebDebugConfig, limits: &WebLimitsConfig) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
enabled: AtomicBool::new(policy.enabled),
|
||||
policy: ArcSwap::from_pointee(policy),
|
||||
policy_update: Mutex::new(()),
|
||||
epoch: AtomicU64::new(1),
|
||||
records_capacity: limits.debug_records_capacity,
|
||||
bytes_capacity: limits.debug_bytes_global,
|
||||
max_carrier_body_bytes: limits.max_body_bytes.max(limits.carrier_batch_bytes),
|
||||
frame_limits: limits.clone(),
|
||||
used_bytes: Arc::new(AtomicUsize::new(0)),
|
||||
ring: Mutex::new(RingState {
|
||||
records: VecDeque::with_capacity(limits.debug_records_capacity),
|
||||
}),
|
||||
next_record_seq: AtomicU64::new(1),
|
||||
next_session_id: AtomicU64::new(1),
|
||||
contention_drops: AtomicU64::new(0),
|
||||
evictions: AtomicU64::new(0),
|
||||
byte_truncations: AtomicU64::new(0),
|
||||
renders: Arc::new(Semaphore::new(2)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Applies one hot policy and clears incompatible retained records.
|
||||
pub(crate) fn apply_policy(&self, policy: &WebDebugConfig) {
|
||||
let _policy_update = self.policy_update.lock();
|
||||
let current = self.policy.load_full();
|
||||
if current.as_ref() == policy {
|
||||
return;
|
||||
}
|
||||
let capture_changed = current.enabled != policy.enabled
|
||||
|| current.capture_lifecycle != policy.capture_lifecycle
|
||||
|| current.capture_headers != policy.capture_headers
|
||||
|| current.capture_timings != policy.capture_timings
|
||||
|| current.capture_frames != policy.capture_frames
|
||||
|| current.body_capture != policy.body_capture
|
||||
|| current.body_prefix_bytes != policy.body_prefix_bytes
|
||||
|| current.decoy_body_prefix_bytes != policy.decoy_body_prefix_bytes;
|
||||
self.policy.store(Arc::new(policy.clone()));
|
||||
self.enabled.store(policy.enabled, Ordering::Release);
|
||||
if capture_changed {
|
||||
self.epoch.fetch_add(1, Ordering::AcqRel);
|
||||
self.ring.lock().records.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/// Allocates a process-unique monotonic WEB session trace identifier.
|
||||
pub(crate) fn next_session_id(&self) -> u64 {
|
||||
self.next_session_id.fetch_add(1, Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Starts one HTTP exchange only when debugging is enabled and budgeted.
|
||||
pub(crate) fn begin_http<B>(
|
||||
self: &Arc<Self>,
|
||||
request: &hyper::Request<B>,
|
||||
peer_ip: IpAddr,
|
||||
) -> Option<Arc<HttpTraceExchange>> {
|
||||
if !self.enabled.load(Ordering::Acquire) {
|
||||
return None;
|
||||
}
|
||||
let epoch = self.epoch.load(Ordering::Acquire);
|
||||
let policy = self.policy.load_full();
|
||||
if !policy.enabled || !self.try_reserve_record(BASE_RECORD_RESERVATION) {
|
||||
return None;
|
||||
}
|
||||
Some(HttpTraceExchange::new(
|
||||
Arc::clone(self),
|
||||
epoch,
|
||||
policy,
|
||||
request,
|
||||
peer_ip,
|
||||
BASE_RECORD_RESERVATION,
|
||||
))
|
||||
}
|
||||
|
||||
/// Records one typed lifecycle event without retaining dynamic error strings.
|
||||
pub(crate) fn record_lifecycle(
|
||||
&self,
|
||||
peer_ip: Option<IpAddr>,
|
||||
effective_ip: Option<IpAddr>,
|
||||
identity: TraceIdentity,
|
||||
event: TraceLifecycleEvent,
|
||||
stream_id: Option<u32>,
|
||||
reason: Option<&'static str>,
|
||||
) {
|
||||
if !self.enabled.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
let epoch = self.epoch.load(Ordering::Acquire);
|
||||
let policy = self.policy.load_full();
|
||||
let identity_bytes = identity
|
||||
.user
|
||||
.as_ref()
|
||||
.map_or(0, String::len)
|
||||
.checked_add(identity.key_fingerprint.as_ref().map_or(0, String::len));
|
||||
let Some(reservation) =
|
||||
identity_bytes.and_then(|bytes| BASE_RECORD_RESERVATION.checked_add(bytes))
|
||||
else {
|
||||
self.record_truncation();
|
||||
return;
|
||||
};
|
||||
if !policy.enabled || !policy.capture_lifecycle || !self.try_reserve_record(reservation) {
|
||||
return;
|
||||
}
|
||||
let record = TraceRecord {
|
||||
seq: self.next_record_seq(),
|
||||
epoch_millis: epoch_millis(),
|
||||
peer_ip,
|
||||
effective_ip,
|
||||
user_agent: None,
|
||||
identity,
|
||||
kind: TraceRecordKind::Lifecycle(TraceLifecycleRecord {
|
||||
event,
|
||||
stream_id,
|
||||
reason,
|
||||
}),
|
||||
};
|
||||
if !self.try_commit(record, reservation, epoch) {
|
||||
self.release(reservation);
|
||||
}
|
||||
}
|
||||
|
||||
/// Records lifecycle identity from a profile only after enabled-policy checks.
|
||||
pub(crate) fn record_profile_lifecycle(
|
||||
&self,
|
||||
effective_ip: IpAddr,
|
||||
session_id: Option<u64>,
|
||||
profile: &crate::config::WebRuntimeProfile,
|
||||
event: TraceLifecycleEvent,
|
||||
stream_id: Option<u32>,
|
||||
reason: Option<&'static str>,
|
||||
) {
|
||||
if !self.enabled.load(Ordering::Acquire) || !self.policy.load().capture_lifecycle {
|
||||
return;
|
||||
}
|
||||
self.record_lifecycle(
|
||||
None,
|
||||
Some(effective_ip),
|
||||
TraceIdentity::from_optional_profile(session_id, profile),
|
||||
event,
|
||||
stream_id,
|
||||
reason,
|
||||
);
|
||||
}
|
||||
|
||||
/// Returns records matching the supplied predicate in newest-first order.
|
||||
pub(crate) fn snapshot_matching<F>(&self, mut matches: F) -> Vec<Arc<StoredTraceRecord>>
|
||||
where
|
||||
F: FnMut(&TraceRecord) -> bool,
|
||||
{
|
||||
self.ring
|
||||
.lock()
|
||||
.records
|
||||
.iter()
|
||||
.rev()
|
||||
.filter(|record| matches(&record.record))
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Returns current bounds, counters, and retained sequence range.
|
||||
pub(crate) fn status(&self) -> TraceStoreStatus {
|
||||
let ring = self.ring.lock();
|
||||
TraceStoreStatus {
|
||||
policy: self.policy.load_full(),
|
||||
records: ring.records.len(),
|
||||
records_capacity: self.records_capacity,
|
||||
used_bytes: self.used_bytes.load(Ordering::Acquire),
|
||||
bytes_capacity: self.bytes_capacity,
|
||||
contention_drops: self.contention_drops.load(Ordering::Relaxed),
|
||||
evictions: self.evictions.load(Ordering::Relaxed),
|
||||
byte_truncations: self.byte_truncations.load(Ordering::Relaxed),
|
||||
earliest_seq: ring.records.front().map(|record| record.record.seq),
|
||||
latest_seq: ring.records.back().map(|record| record.record.seq),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reserves one of two bounded concurrent status-page render slots.
|
||||
pub(crate) fn try_render_permit(&self) -> Option<OwnedSemaphorePermit> {
|
||||
Arc::clone(&self.renders).try_acquire_owned().ok()
|
||||
}
|
||||
|
||||
/// Returns the current capture-policy epoch.
|
||||
pub(super) fn policy_epoch(&self) -> u64 {
|
||||
self.epoch.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
/// Returns the restart-frozen recognized carrier body ceiling.
|
||||
pub(super) fn max_carrier_body_bytes(&self) -> usize {
|
||||
self.max_carrier_body_bytes
|
||||
}
|
||||
|
||||
/// Atomically reserves debug bytes or records one truncation.
|
||||
pub(super) fn try_reserve(&self, bytes: usize) -> bool {
|
||||
if self.try_reserve_inner(bytes) {
|
||||
true
|
||||
} else {
|
||||
self.byte_truncations.fetch_add(1, Ordering::Relaxed);
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
fn try_reserve_inner(&self, bytes: usize) -> bool {
|
||||
let mut current = self.used_bytes.load(Ordering::Acquire);
|
||||
loop {
|
||||
let Some(next) = current.checked_add(bytes) else {
|
||||
return false;
|
||||
};
|
||||
if next > self.bytes_capacity {
|
||||
return false;
|
||||
}
|
||||
match self.used_bytes.compare_exchange_weak(
|
||||
current,
|
||||
next,
|
||||
Ordering::AcqRel,
|
||||
Ordering::Acquire,
|
||||
) {
|
||||
Ok(_) => return true,
|
||||
Err(actual) => current = actual,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn try_reserve_record(&self, bytes: usize) -> bool {
|
||||
if self.try_reserve_inner(bytes) {
|
||||
return true;
|
||||
}
|
||||
let Some(mut ring) = self.ring.try_lock() else {
|
||||
self.byte_truncations.fetch_add(1, Ordering::Relaxed);
|
||||
return false;
|
||||
};
|
||||
while let Some(record) = ring.records.pop_front() {
|
||||
self.evictions.fetch_add(1, Ordering::Relaxed);
|
||||
drop(record);
|
||||
if self.try_reserve_inner(bytes) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
self.byte_truncations.fetch_add(1, Ordering::Relaxed);
|
||||
false
|
||||
}
|
||||
|
||||
/// Releases an in-flight lease that was not transferred into a record.
|
||||
pub(super) fn release(&self, bytes: usize) {
|
||||
self.used_bytes.fetch_sub(bytes, Ordering::AcqRel);
|
||||
}
|
||||
|
||||
/// Increments the closed truncation counter.
|
||||
pub(super) fn record_truncation(&self) {
|
||||
self.byte_truncations.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Allocates a process-monotonic record sequence number.
|
||||
pub(super) fn next_record_seq(&self) -> u64 {
|
||||
self.next_record_seq.fetch_add(1, Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Attempts one non-blocking ring commit under the originating policy epoch.
|
||||
pub(super) fn try_commit(&self, record: TraceRecord, bytes: usize, epoch: u64) -> bool {
|
||||
if epoch != self.policy_epoch() {
|
||||
return false;
|
||||
}
|
||||
let Some(mut ring) = self.ring.try_lock() else {
|
||||
self.contention_drops.fetch_add(1, Ordering::Relaxed);
|
||||
return false;
|
||||
};
|
||||
if epoch != self.policy_epoch() {
|
||||
return false;
|
||||
}
|
||||
while ring.records.len() >= self.records_capacity {
|
||||
ring.records.pop_front();
|
||||
self.evictions.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
ring.records.push_back(Arc::new(StoredTraceRecord {
|
||||
record,
|
||||
bytes,
|
||||
used_bytes: Arc::clone(&self.used_bytes),
|
||||
}));
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns Unix epoch milliseconds with saturation for stored display timestamps.
|
||||
pub(crate) fn epoch_millis() -> u64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_millis()
|
||||
.min(u128::from(u64::MAX)) as u64
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn store(records_capacity: usize, bytes_capacity: usize) -> Arc<WebTraceStore> {
|
||||
let mut policy = WebDebugConfig::default();
|
||||
policy.enabled = true;
|
||||
let mut limits = WebLimitsConfig::default();
|
||||
limits.debug_records_capacity = records_capacity;
|
||||
limits.debug_bytes_global = bytes_capacity;
|
||||
WebTraceStore::new(policy, &limits)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ring_evicts_oldest_records_and_snapshot_leases_survive_clear() {
|
||||
let store = store(2, 4 * BASE_RECORD_RESERVATION);
|
||||
for _ in 0..3 {
|
||||
store.record_lifecycle(
|
||||
None,
|
||||
Some("192.0.2.10".parse().unwrap()),
|
||||
TraceIdentity::default(),
|
||||
TraceLifecycleEvent::BridgeIssued,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
}
|
||||
|
||||
let snapshot = store.snapshot_matching(|_| true);
|
||||
assert_eq!(
|
||||
snapshot
|
||||
.iter()
|
||||
.map(|record| record.record.seq)
|
||||
.collect::<Vec<_>>(),
|
||||
vec![3, 2]
|
||||
);
|
||||
assert_eq!(store.status().evictions, 1);
|
||||
assert_eq!(store.status().used_bytes, 2 * BASE_RECORD_RESERVATION);
|
||||
|
||||
let policy = WebDebugConfig::default();
|
||||
store.apply_policy(&policy);
|
||||
assert_eq!(store.status().records, 0);
|
||||
assert_eq!(store.status().used_bytes, 2 * BASE_RECORD_RESERVATION);
|
||||
drop(snapshot);
|
||||
assert_eq!(store.status().used_bytes, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capture_policy_epoch_rejects_an_inflight_old_policy_record() {
|
||||
let store = store(4, 8 * BASE_RECORD_RESERVATION);
|
||||
let request = hyper::Request::builder().uri("/").body(()).unwrap();
|
||||
let exchange = store
|
||||
.begin_http(&request, "192.0.2.20".parse().unwrap())
|
||||
.unwrap();
|
||||
|
||||
let mut changed = WebDebugConfig::default();
|
||||
changed.enabled = true;
|
||||
changed.capture_headers = false;
|
||||
store.apply_policy(&changed);
|
||||
exchange.commit();
|
||||
|
||||
assert_eq!(store.status().records, 0);
|
||||
assert_eq!(store.status().used_bytes, 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
use std::net::IpAddr;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use crate::config::WebDebugBodyCapture;
|
||||
use crate::web::frame::{self, FrameType};
|
||||
|
||||
use super::super::types::{
|
||||
TraceBodySnapshot, TraceBodyState, TraceDirection, TraceFrame, TraceIdentity, TraceRecord,
|
||||
TraceRecordKind, TraceWebSocketContext, TraceWebSocketRecord,
|
||||
};
|
||||
use super::{BASE_RECORD_RESERVATION, WebTraceStore, epoch_millis};
|
||||
|
||||
impl WebTraceStore {
|
||||
/// Builds connection metadata only while WebSocket debugging is enabled.
|
||||
pub(crate) fn websocket_context<B, F>(
|
||||
&self,
|
||||
request: &hyper::Request<B>,
|
||||
peer_ip: IpAddr,
|
||||
effective_ip: IpAddr,
|
||||
connection_id: u64,
|
||||
lane_id: Option<u32>,
|
||||
identity: F,
|
||||
) -> Option<TraceWebSocketContext>
|
||||
where
|
||||
F: FnOnce() -> TraceIdentity,
|
||||
{
|
||||
if !self.enabled.load(Ordering::Acquire) || !self.policy.load().enabled {
|
||||
return None;
|
||||
}
|
||||
let user_agent = request
|
||||
.headers()
|
||||
.get(hyper::header::USER_AGENT)
|
||||
.map(|value| super::super::sanitize::bounded_text(value.as_bytes(), 512));
|
||||
Some(TraceWebSocketContext {
|
||||
connection_id,
|
||||
peer_ip,
|
||||
effective_ip,
|
||||
user_agent,
|
||||
identity: identity(),
|
||||
lane_id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Records one policy-bounded WebSocket message without retaining credentials.
|
||||
pub(crate) fn record_websocket_message(
|
||||
&self,
|
||||
context: &TraceWebSocketContext,
|
||||
direction: TraceDirection,
|
||||
message_type: &'static str,
|
||||
payload: &[u8],
|
||||
duration_us: u64,
|
||||
) {
|
||||
if !self.enabled.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
let epoch = self.epoch.load(Ordering::Acquire);
|
||||
let policy = self.policy.load_full();
|
||||
if !policy.enabled {
|
||||
return;
|
||||
}
|
||||
let capture_limit = super::super::sanitize::capture_limit(
|
||||
&policy,
|
||||
super::super::types::TraceRoute::Websocket,
|
||||
self.max_carrier_body_bytes,
|
||||
)
|
||||
.unwrap_or(0);
|
||||
let capture_bytes = payload.len().min(capture_limit);
|
||||
let frame_reservation = policy
|
||||
.capture_frames
|
||||
.then(|| {
|
||||
payload
|
||||
.len()
|
||||
.div_ceil(frame::HEADER_BYTES)
|
||||
.clamp(1, self.frame_limits.max_frames_per_body)
|
||||
.saturating_mul(std::mem::size_of::<TraceFrame>())
|
||||
})
|
||||
.unwrap_or(0);
|
||||
let identity_bytes = context
|
||||
.identity
|
||||
.user
|
||||
.as_ref()
|
||||
.map_or(0, String::len)
|
||||
.saturating_add(
|
||||
context
|
||||
.identity
|
||||
.key_fingerprint
|
||||
.as_ref()
|
||||
.map_or(0, String::len),
|
||||
);
|
||||
let reservation = BASE_RECORD_RESERVATION
|
||||
.saturating_add(identity_bytes)
|
||||
.saturating_add(context.user_agent.as_ref().map_or(0, String::len))
|
||||
.saturating_add(capture_bytes)
|
||||
.saturating_add(frame_reservation);
|
||||
if !self.try_reserve_record(reservation) {
|
||||
return;
|
||||
}
|
||||
let body = (policy.body_capture != WebDebugBodyCapture::Off).then(|| TraceBodySnapshot {
|
||||
observed_bytes: payload.len() as u64,
|
||||
captured: payload[..capture_bytes].to_vec(),
|
||||
truncated: capture_limit != 0 && capture_bytes < payload.len(),
|
||||
state: TraceBodyState::Complete,
|
||||
});
|
||||
let frames = if policy.capture_frames && message_type == "binary" {
|
||||
websocket_frames(direction, payload, &self.frame_limits)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let record = TraceRecord {
|
||||
seq: self.next_record_seq(),
|
||||
epoch_millis: epoch_millis(),
|
||||
peer_ip: Some(context.peer_ip),
|
||||
effective_ip: Some(context.effective_ip),
|
||||
user_agent: context.user_agent.clone(),
|
||||
identity: context.identity.clone(),
|
||||
kind: TraceRecordKind::Websocket(TraceWebSocketRecord {
|
||||
direction,
|
||||
message_type,
|
||||
payload_bytes: payload.len(),
|
||||
body,
|
||||
frames,
|
||||
duration_us: policy.capture_timings.then_some(duration_us),
|
||||
connection_id: context.connection_id,
|
||||
lane_id: context.lane_id,
|
||||
}),
|
||||
};
|
||||
if !self.try_commit(record, reservation, epoch) {
|
||||
self.release(reservation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn websocket_frames(
|
||||
direction: TraceDirection,
|
||||
payload: &[u8],
|
||||
limits: &crate::config::WebLimitsConfig,
|
||||
) -> Vec<TraceFrame> {
|
||||
match frame::parse_all(payload, limits) {
|
||||
Ok(frames) => frames
|
||||
.into_iter()
|
||||
.map(|value| TraceFrame {
|
||||
direction,
|
||||
frame_type: Some(super::super::sanitize::frame_type_name(value.frame_type)),
|
||||
stream_id: Some(value.stream_id),
|
||||
payload_len: Some(value.payload.len()),
|
||||
window_delta: (value.frame_type == FrameType::Window)
|
||||
.then(|| frame::window_amount(value.payload).ok())
|
||||
.flatten(),
|
||||
parse_error: None,
|
||||
})
|
||||
.collect(),
|
||||
Err(error) => vec![TraceFrame {
|
||||
direction,
|
||||
frame_type: None,
|
||||
stream_id: None,
|
||||
payload_len: None,
|
||||
window_delta: None,
|
||||
parse_error: Some(super::super::sanitize::frame_error_name(error)),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::{WebDebugConfig, WebLimitsConfig};
|
||||
use crate::web::frame::FrameType;
|
||||
|
||||
#[test]
|
||||
fn websocket_message_capture_retains_bounded_identity_body_timing_and_frames() {
|
||||
let mut policy = WebDebugConfig::default();
|
||||
policy.enabled = true;
|
||||
policy.capture_frames = true;
|
||||
policy.capture_timings = true;
|
||||
policy.body_capture = WebDebugBodyCapture::Full;
|
||||
let mut limits = WebLimitsConfig::default();
|
||||
limits.debug_records_capacity = 4;
|
||||
limits.debug_bytes_global = 64 * 1024;
|
||||
let store = WebTraceStore::new(policy, &limits);
|
||||
let request = hyper::Request::builder()
|
||||
.header(hyper::header::USER_AGENT, "trace-client")
|
||||
.body(())
|
||||
.unwrap();
|
||||
let context = store
|
||||
.websocket_context(
|
||||
&request,
|
||||
"127.0.0.1".parse().unwrap(),
|
||||
"192.0.2.10".parse().unwrap(),
|
||||
17,
|
||||
Some(7),
|
||||
|| TraceIdentity {
|
||||
session_id: Some(42),
|
||||
user: Some("alice".to_string()),
|
||||
key_fingerprint: Some("0123456789abcdef".to_string()),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let payload = crate::web::frame::encode(FrameType::Pong, 0, &[]);
|
||||
store.record_websocket_message(&context, TraceDirection::Request, "binary", &payload, 123);
|
||||
|
||||
let records = store.snapshot_matching(|_| true);
|
||||
assert_eq!(records.len(), 1);
|
||||
assert_eq!(
|
||||
records[0].record.user_agent.as_deref(),
|
||||
Some("trace-client")
|
||||
);
|
||||
let TraceRecordKind::Websocket(message) = &records[0].record.kind else {
|
||||
panic!("expected WebSocket trace");
|
||||
};
|
||||
assert_eq!(message.connection_id, 17);
|
||||
assert_eq!(message.lane_id, Some(7));
|
||||
assert_eq!(message.duration_us, Some(123));
|
||||
assert_eq!(
|
||||
message.body.as_ref().unwrap().captured.as_slice(),
|
||||
payload.as_ref()
|
||||
);
|
||||
assert_eq!(message.frames.len(), 1);
|
||||
assert_eq!(message.frames[0].frame_type, Some("PONG"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
use std::net::IpAddr;
|
||||
|
||||
/// WEB HTTP route classification retained by the debug trace.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum TraceRoute {
|
||||
/// Routing has not completed yet.
|
||||
Unknown,
|
||||
/// Ordinary static or upstream fallback traffic.
|
||||
Decoy,
|
||||
/// Authenticated bridge page issuance.
|
||||
Bridge,
|
||||
/// Bootstrap-to-session exchange.
|
||||
Session,
|
||||
/// Carrier uplink exchange.
|
||||
Uplink,
|
||||
/// Carrier downlink exchange.
|
||||
Downlink,
|
||||
/// WebSocket upgrade handshake.
|
||||
Websocket,
|
||||
}
|
||||
|
||||
impl TraceRoute {
|
||||
/// Returns the stable status-page label.
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Unknown => "unknown",
|
||||
Self::Decoy => "decoy",
|
||||
Self::Bridge => "bridge",
|
||||
Self::Session => "session",
|
||||
Self::Uplink => "uplink",
|
||||
Self::Downlink => "downlink",
|
||||
Self::Websocket => "websocket",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Direction of captured HTTP body or carrier frame data.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum TraceDirection {
|
||||
/// Client-to-server data.
|
||||
Request,
|
||||
/// Server-to-client data.
|
||||
Response,
|
||||
}
|
||||
|
||||
impl TraceDirection {
|
||||
/// Returns the stable status-page label.
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Request => "request",
|
||||
Self::Response => "response",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Terminal observation state of one HTTP body.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum TraceBodyState {
|
||||
/// The body completed normally.
|
||||
Complete,
|
||||
/// Body polling returned an error.
|
||||
Error,
|
||||
/// The body was dropped before a terminal poll.
|
||||
Aborted,
|
||||
}
|
||||
|
||||
impl TraceBodyState {
|
||||
/// Returns the stable status-page label.
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Complete => "complete",
|
||||
Self::Error => "error",
|
||||
Self::Aborted => "aborted",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One sanitized HTTP header.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct TraceHeader {
|
||||
/// Lowercase header name.
|
||||
pub(crate) name: String,
|
||||
/// Allowlisted value, or `None` when only the name may be retained.
|
||||
pub(crate) value: Option<String>,
|
||||
}
|
||||
|
||||
/// One bounded HTTP body observation.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct TraceBodySnapshot {
|
||||
/// Total bytes observed on the wire-facing Hyper body.
|
||||
pub(crate) observed_bytes: u64,
|
||||
/// Retained prefix or full bounded body bytes.
|
||||
pub(crate) captured: Vec<u8>,
|
||||
/// Indicates that bytes were omitted by policy or capacity.
|
||||
pub(crate) truncated: bool,
|
||||
/// Terminal body state.
|
||||
pub(crate) state: TraceBodyState,
|
||||
}
|
||||
|
||||
/// One parsed carrier frame without payload retention.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct TraceFrame {
|
||||
/// HTTP body direction containing the frame.
|
||||
pub(crate) direction: TraceDirection,
|
||||
/// Stable protocol frame type, when the header parsed.
|
||||
pub(crate) frame_type: Option<&'static str>,
|
||||
/// Logical stream or lane identifier.
|
||||
pub(crate) stream_id: Option<u32>,
|
||||
/// Frame payload length.
|
||||
pub(crate) payload_len: Option<usize>,
|
||||
/// Decoded non-zero WINDOW delta.
|
||||
pub(crate) window_delta: Option<u32>,
|
||||
/// Closed parse or shape error category.
|
||||
pub(crate) parse_error: Option<&'static str>,
|
||||
}
|
||||
|
||||
/// Request lifecycle timing points measured from service entry.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct TraceTimings {
|
||||
/// Request body terminal poll time.
|
||||
pub(crate) request_body_us: Option<u64>,
|
||||
/// Handler response-ready time.
|
||||
pub(crate) response_ready_us: Option<u64>,
|
||||
/// Response body terminal poll or drop time.
|
||||
pub(crate) response_body_us: Option<u64>,
|
||||
}
|
||||
|
||||
/// Stable non-secret WEB trace identity.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct TraceIdentity {
|
||||
/// Process-unique monotonic WEB session identifier.
|
||||
pub(crate) session_id: Option<u64>,
|
||||
/// Exact access user name.
|
||||
pub(crate) user: Option<String>,
|
||||
/// Domain-separated client-secret fingerprint.
|
||||
pub(crate) key_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
impl TraceIdentity {
|
||||
/// Builds a non-secret identity from one validated runtime profile.
|
||||
pub(crate) fn from_profile(
|
||||
session_id: u64,
|
||||
profile: &crate::config::WebRuntimeProfile,
|
||||
) -> Self {
|
||||
Self::from_optional_profile(Some(session_id), profile)
|
||||
}
|
||||
|
||||
/// Builds a profile identity when admission failed before session allocation.
|
||||
pub(crate) fn from_optional_profile(
|
||||
session_id: Option<u64>,
|
||||
profile: &crate::config::WebRuntimeProfile,
|
||||
) -> Self {
|
||||
Self {
|
||||
session_id,
|
||||
user: Some(profile.user.clone()),
|
||||
key_fingerprint: Some(profile.key_fingerprint.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Complete request-to-response WEB HTTP trace.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct TraceHttpRecord {
|
||||
/// HTTP method.
|
||||
pub(crate) method: String,
|
||||
/// URI path without query material.
|
||||
pub(crate) path: String,
|
||||
/// Final route classification.
|
||||
pub(crate) route: TraceRoute,
|
||||
/// Sanitized request headers.
|
||||
pub(crate) request_headers: Vec<TraceHeader>,
|
||||
/// Sanitized response headers.
|
||||
pub(crate) response_headers: Vec<TraceHeader>,
|
||||
/// Request body observation when enabled by policy.
|
||||
pub(crate) request_body: Option<TraceBodySnapshot>,
|
||||
/// Response status when a response head was produced.
|
||||
pub(crate) status: Option<u16>,
|
||||
/// Response body observation when enabled by policy.
|
||||
pub(crate) response_body: Option<TraceBodySnapshot>,
|
||||
/// Parsed carrier frame metadata.
|
||||
pub(crate) frames: Vec<TraceFrame>,
|
||||
/// Monotonic request timing points.
|
||||
pub(crate) timings: Option<TraceTimings>,
|
||||
}
|
||||
|
||||
/// Stable non-secret metadata retained across one WebSocket connection.
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct TraceWebSocketContext {
|
||||
/// Process-unique connection identifier.
|
||||
pub(crate) connection_id: u64,
|
||||
/// Direct listener peer address.
|
||||
pub(crate) peer_ip: IpAddr,
|
||||
/// Trusted effective client address.
|
||||
pub(crate) effective_ip: IpAddr,
|
||||
/// Bounded user-agent copied only while debugging is enabled.
|
||||
pub(crate) user_agent: Option<String>,
|
||||
/// Session and profile identity without credentials.
|
||||
pub(crate) identity: TraceIdentity,
|
||||
/// Logical lane identifier for websocket-lanes.
|
||||
pub(crate) lane_id: Option<u32>,
|
||||
}
|
||||
|
||||
/// One bounded ordered WebSocket message observation.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct TraceWebSocketRecord {
|
||||
/// Wire direction of this message.
|
||||
pub(crate) direction: TraceDirection,
|
||||
/// Closed RFC 6455 message category.
|
||||
pub(crate) message_type: &'static str,
|
||||
/// Total payload bytes observed.
|
||||
pub(crate) payload_bytes: usize,
|
||||
/// Policy-bounded message body observation.
|
||||
pub(crate) body: Option<TraceBodySnapshot>,
|
||||
/// Parsed carrier frames for binary messages.
|
||||
pub(crate) frames: Vec<TraceFrame>,
|
||||
/// Message processing or write duration when timing capture is enabled.
|
||||
pub(crate) duration_us: Option<u64>,
|
||||
/// Process-unique owning WebSocket connection.
|
||||
pub(crate) connection_id: u64,
|
||||
/// Logical lane identifier for websocket-lanes.
|
||||
pub(crate) lane_id: Option<u32>,
|
||||
}
|
||||
|
||||
/// Closed WEB lifecycle event category.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum TraceLifecycleEvent {
|
||||
/// A bridge bootstrap was issued.
|
||||
BridgeIssued,
|
||||
/// Bootstrap or bridge admission was rejected.
|
||||
BootstrapRejected,
|
||||
/// A new session was created.
|
||||
SessionCreated,
|
||||
/// An idempotent session creation was replayed.
|
||||
SessionReplayed,
|
||||
/// Session creation was rejected.
|
||||
SessionRejected,
|
||||
/// A session closed.
|
||||
SessionClosed,
|
||||
/// A logical stream was admitted.
|
||||
StreamAdmitted,
|
||||
/// A logical stream was rejected.
|
||||
StreamRejected,
|
||||
/// A logical stream delivered its first inner byte.
|
||||
StreamFirstByte,
|
||||
/// An admitted logical stream released its relay and tuple ownership.
|
||||
StreamClosed,
|
||||
/// The inner MTProxy handshake succeeded.
|
||||
HandshakeSucceeded,
|
||||
/// The inner MTProxy handshake timed out.
|
||||
HandshakeTimeout,
|
||||
/// The inner MTProxy handshake failed on I/O.
|
||||
HandshakeIo,
|
||||
/// The inner MTProxy handshake was rejected.
|
||||
HandshakeRejected,
|
||||
/// Authenticated relay started.
|
||||
RelayStarted,
|
||||
/// Authenticated relay ended.
|
||||
RelayEnded,
|
||||
}
|
||||
|
||||
impl TraceLifecycleEvent {
|
||||
/// Returns the stable status-page label.
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::BridgeIssued => "bridge_issued",
|
||||
Self::BootstrapRejected => "bootstrap_rejected",
|
||||
Self::SessionCreated => "session_created",
|
||||
Self::SessionReplayed => "session_replayed",
|
||||
Self::SessionRejected => "session_rejected",
|
||||
Self::SessionClosed => "session_closed",
|
||||
Self::StreamAdmitted => "stream_admitted",
|
||||
Self::StreamRejected => "stream_rejected",
|
||||
Self::StreamFirstByte => "stream_first_byte",
|
||||
Self::StreamClosed => "stream_closed",
|
||||
Self::HandshakeSucceeded => "handshake_succeeded",
|
||||
Self::HandshakeTimeout => "handshake_timeout",
|
||||
Self::HandshakeIo => "handshake_io",
|
||||
Self::HandshakeRejected => "handshake_rejected",
|
||||
Self::RelayStarted => "relay_started",
|
||||
Self::RelayEnded => "relay_ended",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One typed WEB lifecycle observation.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct TraceLifecycleRecord {
|
||||
/// Closed event category.
|
||||
pub(crate) event: TraceLifecycleEvent,
|
||||
/// Logical stream identifier when applicable.
|
||||
pub(crate) stream_id: Option<u32>,
|
||||
/// Closed outcome or rejection reason.
|
||||
pub(crate) reason: Option<&'static str>,
|
||||
}
|
||||
|
||||
/// Trace record payload variant.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum TraceRecordKind {
|
||||
/// HTTP request-to-response exchange.
|
||||
Http(TraceHttpRecord),
|
||||
/// One ordered WebSocket message.
|
||||
Websocket(TraceWebSocketRecord),
|
||||
/// Session or stream lifecycle event.
|
||||
Lifecycle(TraceLifecycleRecord),
|
||||
}
|
||||
|
||||
/// Common stored WEB trace record.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct TraceRecord {
|
||||
/// Process-unique monotonic record sequence.
|
||||
pub(crate) seq: u64,
|
||||
/// Event completion time in Unix epoch milliseconds.
|
||||
pub(crate) epoch_millis: u64,
|
||||
/// Direct TCP peer accepted by the WEB listener.
|
||||
pub(crate) peer_ip: Option<IpAddr>,
|
||||
/// Trusted effective client identity.
|
||||
pub(crate) effective_ip: Option<IpAddr>,
|
||||
/// Bounded user-agent value retained for filtering.
|
||||
pub(crate) user_agent: Option<String>,
|
||||
/// Non-secret WEB identity.
|
||||
pub(crate) identity: TraceIdentity,
|
||||
/// Typed record payload.
|
||||
pub(crate) kind: TraceRecordKind,
|
||||
}
|
||||
Reference in New Issue
Block a user