mirror of
https://github.com/telemt/telemt.git
synced 2026-09-05 18:16:06 +03:00
Alexey
Generated
+1
-1
@@ -2900,7 +2900,7 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
|
||||
|
||||
[[package]]
|
||||
name = "telemt"
|
||||
version = "3.5.4"
|
||||
version = "3.5.5"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"anyhow",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "telemt"
|
||||
version = "3.5.4"
|
||||
version = "3.5.5"
|
||||
edition = "2024"
|
||||
|
||||
[features]
|
||||
|
||||
@@ -104,6 +104,13 @@ Notes:
|
||||
| `GET` | `/v1/runtime/connections/summary` | none | `200` | `RuntimeEdgeConnectionsSummaryData` |
|
||||
| `GET` | `/v1/runtime/events/recent` | none | `200` | `RuntimeEdgeEventsData` |
|
||||
| `GET` | `/v1/runtime/tls-fingerprints` | optional `limit=1..1000` | `200` | `RuntimeEdgeTlsFingerprintsData` |
|
||||
| `GET` | `/v1/runtime/web/status` | none | `200` | `WebStatusData` |
|
||||
| `GET` | `/v1/runtime/web/sessions` | bounded query | `200` | `SessionPage` |
|
||||
| `GET` | `/v1/runtime/web/sessions/{session_ref}` | none | `200` or `410` | `SessionRow` or closed tombstone |
|
||||
| `POST` | `/v1/runtime/web/sessions/close` | `CloseRequest` | `202` | `ControlOperationStatus` |
|
||||
| `GET` | `/v1/runtime/web/operations/{operation_id}` | none | `200` | `ControlOperationStatus` |
|
||||
| `POST` | `/v1/runtime/web/debug/clear` | `RuntimeInstanceRequest` | `200` | `DebugClearData` |
|
||||
| `POST` | `/v1/runtime/web/carrier-learning/reset` | `RuntimeInstanceRequest` | `200` | `LearningResetData` |
|
||||
| `GET` | `/v1/stats/users/active-ips` | none | `200` | `UserActiveIps[]` |
|
||||
| `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` |
|
||||
| `GET` | `/v1/config` | none | `200` | `ConfigData` |
|
||||
@@ -145,6 +152,13 @@ Notes:
|
||||
| `GET /v1/runtime/me-selftest` | Returns ME self-test state for KDF, time skew, IP family, PID, and SOCKS BND observations. |
|
||||
| `GET /v1/runtime/connections/summary` | Returns runtime-edge connection totals and top-N users by connections/throughput. |
|
||||
| `GET /v1/runtime/events/recent` | Returns recent API/runtime event records with optional `limit` query. |
|
||||
| `GET /v1/runtime/web/status` | Returns WEB listener lifecycle and a non-blocking, plane-local snapshot of the process-owned WEB runtime when available. |
|
||||
| `GET /v1/runtime/web/sessions` | Returns a bounded ordered page of live WEB sessions with strict filters and opaque process-fenced references. |
|
||||
| `GET /v1/runtime/web/sessions/{session_ref}` | Returns one exact live-session snapshot or a retained closed-session tombstone. |
|
||||
| `POST /v1/runtime/web/sessions/close` | Accepts one bounded asynchronous point-in-time close operation. |
|
||||
| `GET /v1/runtime/web/operations/{operation_id}` | Returns one of the 32 most recently retained WEB close-operation states. |
|
||||
| `POST /v1/runtime/web/debug/clear` | Clears the bounded WEB debug ring under an epoch fence. |
|
||||
| `POST /v1/runtime/web/carrier-learning/reset` | Clears process-local carrier-learning evidence without changing live attempt chains. |
|
||||
| `GET /v1/stats/users/active-ips` | Returns users that currently have non-empty active source-IP lists. |
|
||||
| `GET /v1/stats/users` | Alias of `GET /v1/users`; returns disk-first user views with runtime lag flag. |
|
||||
| `GET /v1/config` | Returns the current editable config sections as JSON (no `access.*`) plus the revision. |
|
||||
@@ -176,12 +190,19 @@ Notes:
|
||||
| `405` | `method_not_allowed` | Unsupported method for `/v1/users/{username}` route shape. |
|
||||
| `409` | `revision_conflict` | `If-Match` revision mismatch. |
|
||||
| `409` | `reload_in_progress` | Another reload operation is non-terminal. |
|
||||
| `409` | `web_runtime_mismatch` | A runtime instance, session reference, or operation reference belongs to another WEB process instance. |
|
||||
| `409` | `web_issuance_enabled` | A WEB close-all operation was requested while effective issuance remained enabled. |
|
||||
| `409` | `web_operation_in_progress` | Another bounded WEB close operation is active. |
|
||||
| `409` | `user_exists` | User already exists on create. |
|
||||
| `409` | `last_user_forbidden` | Attempt to delete last configured user. |
|
||||
| `413` | `payload_too_large` | Body exceeds `request_body_limit_bytes`. |
|
||||
| `415` | `unsupported_media_type` | A WEB runtime POST does not carry exactly one `Content-Type: application/json` header. |
|
||||
| `410` | success envelope | A valid WEB session reference resolves to a retained closed-session tombstone. |
|
||||
| `500` | `internal_error` | Internal error (I/O, serialization, config load/save). |
|
||||
| `503` | `api_disabled` | API disabled in config. |
|
||||
| `503` | `maestro_unavailable` | Maestro's reload command channel is unavailable. |
|
||||
| `503` | `web_runtime_unavailable` | The WEB runtime is not running or has left the readable lifecycle. |
|
||||
| `503` | `web_snapshot_busy` | An exact non-blocking WEB session snapshot encountered lock contention. |
|
||||
|
||||
## Routing and Method Edge Cases
|
||||
|
||||
@@ -204,8 +225,8 @@ Notes:
|
||||
- Body size limit is enforced during streaming read (`413 payload_too_large`).
|
||||
- Invalid transport body frame returns `400 bad_request` (`Invalid request body`).
|
||||
- Invalid JSON returns `400 bad_request` (`Invalid JSON body`).
|
||||
- `Content-Type` is not required for JSON parsing.
|
||||
- Unknown JSON fields are ignored by deserialization.
|
||||
- `Content-Type` is not required for existing config/user/reload JSON parsing. Every WEB runtime POST requires exactly one header with the exact value `application/json`.
|
||||
- Existing request DTOs retain their documented unknown-field behavior. WEB runtime POST DTOs reject unknown JSON fields.
|
||||
- `PATCH` uses JSON Merge Patch semantics for optional per-user fields: omitted means unchanged, explicit `null` removes the config entry, and a non-null value sets it.
|
||||
- `If-Match` supports both quoted and unquoted values; surrounding whitespace is trimmed.
|
||||
|
||||
@@ -214,6 +235,7 @@ Notes:
|
||||
| Endpoint | Query | Behavior |
|
||||
| --- | --- | --- |
|
||||
| `GET /v1/runtime/events/recent` | `limit=<usize>` | Optional. Invalid/missing value falls back to default `50`. Effective value is clamped to `[1, 1000]` and additionally bounded by ring-buffer capacity. |
|
||||
| `GET /v1/runtime/web/sessions` | `limit`, `cursor`, `session_ref`, `ip`, `host`, `user`, `user_agent_id`, `key_id`, `carrier`, `state` | `limit` defaults to 50 and is bounded to `1..=200`; the ordered candidate scan stops at 1000. Duplicate and unknown fields are rejected. `session_ref` selects an exact logical session and cannot be combined with `cursor` or `limit`. |
|
||||
|
||||
## Request Contracts
|
||||
|
||||
@@ -261,11 +283,11 @@ bob = ["198.51.100.42/32"]
|
||||
|
||||
### `PatchConfigRequest`
|
||||
|
||||
A sparse JSON object containing only the top-level config sections to modify. Each key must be one of the editable sections (`general`, `timeouts`, `censorship`, `upstreams`, `dc_overrides`) or the partially editable `server` object (only `listeners` is allowed under `server`; see below). Tables within a section are deep-merged field-by-field into the existing config; arrays and scalar values replace the existing value wholesale. Untouched sections and file comments are preserved.
|
||||
A sparse JSON object containing only the top-level config sections to modify. Each key must be one of the editable sections (`general`, `timeouts`, `censorship`, `upstreams`, `dc_overrides`, `web`) or the partially editable `server` object (only `listeners` is allowed under `server`; see below). Tables within a section are deep-merged field-by-field into the existing config; arrays and scalar values replace the existing value wholesale. Untouched sections and file comments are preserved.
|
||||
|
||||
**Rejected keys:**
|
||||
- `access` → `400 access_not_editable` (users/secrets are managed via `POST/PATCH /v1/users`).
|
||||
- `network`, `web`, or any unknown top-level key → `400 section_not_editable`.
|
||||
- `network` or any unknown top-level key → `400 section_not_editable`.
|
||||
- `server` with any key other than `listeners` (e.g. `port`, `api`, `admin_api`) → `400 field_not_editable`.
|
||||
- An object with no editable keys → `400 bad_request` (empty patch).
|
||||
|
||||
@@ -299,9 +321,91 @@ Returned by `GET /v1/config` as the envelope `data`. The fields are exactly the
|
||||
| `censorship` | `object?` | `[censorship]` section, if present. |
|
||||
| `upstreams` | `object?` | `[upstreams]` section, if present. |
|
||||
| `dc_overrides` | `object?` | `[dc_overrides]` section, if present. |
|
||||
| `web` | `object?` | Complete authored `[web]` section, if present. The derived runtime-only `web.runtime` field is excluded. |
|
||||
| `server` | `object?` | Partial `[server]` view when editable nested fields are present. Currently only `listeners` may appear; `api`/`admin_api`, `port`, unix sockets, and other bind-identity fields are never returned. |
|
||||
|
||||
Sections absent from the config file are absent from the response (not `null`). Only the editable sections above are returned; `access` (users/secrets) and `network` (per-node addresses) are always excluded. Under `server`, only the nested field-level allowlist (`listeners`) is exposed.
|
||||
Sections absent from the config file are absent from the response (not `null`). Only the editable sections above are returned; `access` (users/secrets) and `network` (per-node addresses) are always excluded. Under `server`, only the nested field-level allowlist (`listeners`) is exposed. Changes under `[web.limits]` are valid desired configuration but remain process-deferred; the patch response reports `web.limits` in `deferred_process_fields` until restart.
|
||||
|
||||
### WEB runtime identity and lifecycle
|
||||
|
||||
The WEB control plane is process-fenced. `runtime_instance` is a random 128-bit lowercase hexadecimal value created with the process-owned WEB runtime. Session references use `ws1.<runtime_instance>.<16-lowercase-hex-id>` and close-operation references use `wo1.<runtime_instance>.<16-lowercase-hex-id>`. Treat all three as opaque. A reference from another process instance returns `409 web_runtime_mismatch`, preventing an old controller from targeting reused counters after restart.
|
||||
|
||||
`GET /v1/config` is the desired on-disk configuration view. `GET /v1/runtime/web/status` is the effective process view. Its envelope `revision` still identifies the current source graph and can therefore be newer than the active runtime generation while a reload is pending.
|
||||
|
||||
`WebStatusData` contains:
|
||||
|
||||
| Field | Type | Description |
|
||||
| --- | --- | --- |
|
||||
| `lifecycle` | `string` | `starting`, `no_web_listener`, `running`, `draining`, `drained`, or `deadline_exceeded`. |
|
||||
| `lifecycle_epoch` | `u64` | Monotonic publication epoch. |
|
||||
| `lifecycle_age_ms` | `u64` | Monotonic age of the current lifecycle publication. |
|
||||
| `available` | `bool` | Whether a readable process runtime is currently published. |
|
||||
| `reason` | `string?` | Stable unavailability reason when `available=false`. |
|
||||
| `listeners` | `string[]` | Effective bound WEB listener addresses. |
|
||||
| `effective_config_enabled` | `bool` | `web.enabled` in the API request's active runtime generation. |
|
||||
| `runtime` | `WebRuntimeStatus?` | Present while the weak process-runtime publication can be upgraded. |
|
||||
|
||||
`WebRuntimeStatus` includes `runtime_instance`, `generation_id`, immutable effective `limits`, manager/stream/budget/WebSocket/learning/debug planes, permit usage, task/counter totals, and `partial`. Plane locks are read with `try_lock`; a contended plane is omitted and named in `partial`. Status collection performs no cleanup, waits, or data-plane mutation, so fields are plane-local observations rather than one globally atomic snapshot. `runtime.manager.issuance_enabled` is the authority to check before close-all.
|
||||
|
||||
### WEB session enumeration
|
||||
|
||||
`GET /v1/runtime/web/sessions` defaults to `limit=50`, permits `1..=200`, and scans at most 1000 ordered candidates. `next_cursor` continues after the last scanned opaque session reference. `scan_truncated` reports the scan bound, `partial_sessions` counts contended per-session snapshots, and `partial` names an unavailable manager plane. The complete serialized page remains below the API response envelope because every string and row count is bounded.
|
||||
|
||||
Filters are exact unless stated otherwise:
|
||||
|
||||
| Query field | Contract |
|
||||
| --- | --- |
|
||||
| `cursor`, `session_ref` | Canonical opaque reference for the current `runtime_instance`; `session_ref` forces one-row lookup semantics and is mutually exclusive with `cursor` and `limit`. |
|
||||
| `ip` | Canonically formatted IPv4 or IPv6 address. |
|
||||
| `host` | Non-empty, at most 253 bytes. |
|
||||
| `user` | Non-empty, at most 64 bytes. |
|
||||
| `user_agent_id` | Exactly 32 lowercase hexadecimal characters. |
|
||||
| `key_id` | Exactly 16 lowercase hexadecimal characters; this is a non-secret domain-separated fingerprint. |
|
||||
| `carrier` | `https`, `https-lanes`, `websocket`, or `websocket-lanes`. |
|
||||
| `state` | `provisional`, `replacing`, `committed`, `healthy`, `closing`, `superseded`, or transient live-index `closed`. |
|
||||
|
||||
Each `SessionRow` contains `session_ref`, optional bounded `user_agent` and `user_agent_id`, plus client IP, host, user, key fingerprint, carrier/attempt/class/state, stream/task/lane/WebSocket counts, pending/control usage, age/idle timing, and optional negotiation time remaining. No bootstrap token, session bearer, raw capability, configured secret/hash, or synthetic source/KDF port is returned. `GET /v1/runtime/web/sessions/{session_ref}` returns `200` for a live row, `410` with `{state:"closed", attempt}` for a bounded retained tombstone, `404` if unknown, or `503 web_snapshot_busy` on lock contention.
|
||||
|
||||
### WEB runtime mutations
|
||||
|
||||
Every WEB runtime POST requires the currently published `runtime_instance`, exactly one `Content-Type: application/json` header, no query parameters, and a JSON object with no unknown fields. All mutations inherit API authentication, direct-peer whitelist, body limit, audit recording, and `read_only` enforcement.
|
||||
|
||||
`POST /v1/runtime/web/sessions/close` accepts:
|
||||
|
||||
```json
|
||||
{
|
||||
"runtime_instance": "0123456789abcdef0123456789abcdef",
|
||||
"selector": {
|
||||
"kind": "refs",
|
||||
"session_refs": ["ws1.0123456789abcdef0123456789abcdef.0000000000000001"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The selector is exactly one of:
|
||||
|
||||
- `refs`: `1..=200` unique current-instance session references.
|
||||
- `filter`: at least one session filter using the same fields and bounds as enumeration.
|
||||
- `all`: every session at or below the operation's submission high-water mark; rejected with `409 web_issuance_enabled` until effective issuance is disabled.
|
||||
|
||||
Only one close operation runs at a time. Work is chunked into at most 128 registry candidates between cancellation points, never awaits while holding a manager/session lock, and never targets sessions created above its high-water mark. `202` returns `ControlOperationStatus`; poll its `operation_id` through `GET /v1/runtime/web/operations/{operation_id}`. The process retains 32 statuses. Fields include `state` (`queued`, `running`, `completed`, `cancelled`, or `failed`), the high-water reference, requested/scanned/matched/signalled/conflicted counters, timestamps, and an optional sanitized failure token.
|
||||
|
||||
`POST /v1/runtime/web/debug/clear` and `POST /v1/runtime/web/carrier-learning/reset` both accept:
|
||||
|
||||
```json
|
||||
{"runtime_instance":"0123456789abcdef0123456789abcdef"}
|
||||
```
|
||||
|
||||
Debug clear swaps the retained ring under a new epoch and reports `records_cleared`, still-leased snapshot bytes, and `epoch`; old in-flight commits cannot restore removed data. Learning reset swaps bounded evidence under a new epoch and reports `entries_cleared` and `epoch`; live sessions and already frozen negotiation chains are unaffected.
|
||||
|
||||
Deterministic close-all sequence:
|
||||
|
||||
1. `PATCH /v1/config?reload=drain` with `{"web":{"enabled":false}}`.
|
||||
2. Poll the accepted reload, then `GET /v1/runtime/web/status` until `runtime.manager.issuance_enabled=false`.
|
||||
3. Submit the `all` close selector with the same status `runtime_instance`.
|
||||
4. Poll the returned operation to a terminal state.
|
||||
|
||||
`web.enabled=false` only stops new bridge/bootstrap/session issuance. It never implicitly closes active WEB sessions.
|
||||
|
||||
### `PatchConfigResponse`
|
||||
|
||||
@@ -1524,20 +1628,23 @@ The revision is verified again after preparation. With `failure_policy=rollback`
|
||||
|
||||
## WEB Proxy Management
|
||||
|
||||
The API provides partial operational control for WEB mode. It does not expose a mutable `/v1/web` resource, but it serves bounded read-only HTML diagnostics at `GET /web-status`.
|
||||
The API exposes WEB desired configuration through the common config resource, process state through `/v1/runtime/web`, and bounded read-only HTML diagnostics at `GET /web-status`. There is no separate `/v1/web` configuration resource.
|
||||
|
||||
| Operation | Current contract |
|
||||
| --- | --- |
|
||||
| Read or patch `[web]`, vhosts, profiles, decoys, timeouts, or limits | Not exposed. `GET /v1/config` omits `[web]`; a `web` key in `PATCH /v1/config` returns `400 section_not_editable`. |
|
||||
| Read or patch `[web]`, vhosts, profiles, decoys, timeouts, or limits | Supported through `GET` and `PATCH /v1/config`; `web.runtime` is derived and excluded. Tables deep-merge, arrays replace wholesale, and `web.limits` remains process-deferred. |
|
||||
| Persist `server.listeners` | Supported through `PATCH /v1/config`. Arrays replace wholesale. A changed WEB listener is process-owned and remains deferred until process restart. |
|
||||
| Apply an externally edited WEB config | Update the owning TOML source, call `POST /v1/system/reload`, then poll `GET /v1/system/reload/{id}`. |
|
||||
| Inspect restart requirements | Read `deferred_process_fields` from reload status. `server.listeners` and `web.limits` require process restart. |
|
||||
| Manage access users | Use `/v1/users`. Creating a user does not add it to `web.vhosts.profiles`; profile membership remains file-managed. |
|
||||
| Inspect WEB lifecycle, capacity, sessions, operations, learning, and debug state | Use the authenticated `GET /v1/runtime/web/*` routes documented above. |
|
||||
| Close selected or all point-in-time sessions | Use `POST /v1/runtime/web/sessions/close`; close-all first requires effective issuance to be disabled. |
|
||||
| Clear debug records or reset carrier learning | Use `POST /v1/runtime/web/debug/clear` or `/carrier-learning/reset` with the current `runtime_instance`. |
|
||||
| Manage access users | Use `/v1/users`. Creating a user does not add it to `web.vhosts.profiles`; add profile membership through the `web` config patch. |
|
||||
| Disable one user | `POST /v1/users/{username}/disable` updates admission immediately and cancels the user's active sessions. |
|
||||
| Rotate a profiled user's secret | Use `/v1/users/{username}/rotate-secret`; the config watcher rebuilds WEB capabilities from the new access snapshot. The API returns the secret, not a `tg://webproxy` link. |
|
||||
| Read WEB-specific runtime diagnostics | Use authenticated `GET /web-status`; filters cover client IP, process session ID, User-Agent, and non-secret key fingerprint, with optional grouping, expandable HTTP request-to-response details, and WebSocket handshake/message/frame rows. |
|
||||
|
||||
`web.enabled`, `web.carrier`, `web.debug`, `web.timeouts`, vhosts, profiles, and decoy snapshots are runtime-generation fields. A changed carrier applies only to newly issued bridge sessions; existing sessions retain their creation-time carrier. WEB listener inventory and trust policy, plus all `[web.limits]`, are process-owned. A successful reload can therefore activate the runtime-owned subset while reporting the process-owned subset as deferred.
|
||||
`web.enabled`, `web.carrier`, `web.debug`, `web.timeouts`, vhosts, profiles, and decoy snapshots are runtime-generation fields. A changed carrier applies only to newly issued bridge sessions; existing sessions and issued bootstrap chains retain their issuance-time policy. `web.enabled=false` stops new issuance but never closes live sessions implicitly. WEB listener inventory and trust policy, plus all `[web.limits]`, are process-owned. A successful reload can therefore activate the runtime-owned subset while reporting the process-owned subset as deferred.
|
||||
|
||||
Before deleting a user referenced by a WEB profile, remove and apply the profile first. User mutations validate the complete resulting configuration, so a dangling WEB profile is rejected rather than persisted.
|
||||
|
||||
|
||||
@@ -2558,12 +2558,19 @@ Der WEB-Modus transportiert MTProxy-Datenverkehr von Telegram Desktop über HTTP
|
||||
| --- | --- | --- | --- |
|
||||
| `enabled` | `bool` | `false` | `✔` |
|
||||
| `carrier` | `"https"`, `"https-lanes"`, `"websocket"` oder `"websocket-lanes"` | `"https"` | `✔` |
|
||||
| `carriers` | `false` oder ein nicht leeres Array eindeutiger Carrier | `false` | `✔` |
|
||||
| `carrier_learning` | `bool` | `true` | `✔` |
|
||||
| `carrier_negotiation_aggressiveness` | `"conservative"`, `"balanced"` oder `"aggressive"` | `"conservative"` | `✔` |
|
||||
| `debug` | Tabelle | deaktiviert, begrenzte Defaults | `✔` |
|
||||
| `limits` | Tabelle | begrenzte Defaults | `✘` |
|
||||
| `timeouts` | Tabelle | begrenzte Defaults | `✔` |
|
||||
| `vhosts` | Tabellen-Array | `[]` | `✔` |
|
||||
|
||||
`enabled = true` erfordert mindestens einen durch die Netzwerkrichtlinie zugelassenen WEB-Listener, einen vhost und mindestens ein Profil in jedem vhost. `https` behält den serialisierten HTTPS-Transport bei. Mit `https-lanes` erhalten Stream null und jeder logische Stream eigene Uplink-Sequenzen, Downlink-Cursor, Wiederholungen und Long Polls; dieser Carrier erfordert `max_http_handlers >= 2` und öffentliches HTTP/2 am TLS-Terminator. `websocket` transportiert alle logischen Streams über eine geordnete RFC-6455-Verbindung, während `websocket-lanes` jedem Stream ungleich null eine eigene Verbindung zuweist und Lane-Fehler isoliert. Beide WebSocket-Carrier verwenden nach der HTTPS-Sitzungserstellung `GET /api/v1/ws` und erfordern, dass der TLS-Terminator die HTTP/1.1-Upgrade-Header unverändert weiterleitet. Ein Reload wendet `carrier` nur auf neu ausgegebene Bridge-Sitzungen an. Das Deaktivieren von WEB beendet nach dem Reload die Ausgabe neuer Bridge- und Session-Zugangsdaten; zum Widerrufen aktiver Sitzungen eines einzelnen Benutzers verwenden Sie die Users-API.
|
||||
`enabled = true` erfordert mindestens einen durch die Netzwerkrichtlinie zugelassenen WEB-Listener, einen vhost und mindestens ein Profil in jedem vhost. `https` behält den serialisierten HTTPS-Transport bei und erfordert `max_http_handlers >= 2`. Mit `https-lanes` erhalten Stream null und jeder logische Stream eigene Uplink-Sequenzen, Downlink-Cursor, Wiederholungen und Long Polls; dieser Carrier erfordert `max_http_handlers >= 4` und öffentliches HTTP/2 am TLS-Terminator. `websocket` transportiert alle logischen Streams über eine geordnete RFC-6455-Verbindung, während `websocket-lanes` jedem Stream ungleich null eine eigene Verbindung zuweist und Lane-Fehler isoliert. Beide WebSocket-Carrier verwenden nach der HTTPS-Sitzungserstellung `GET /api/v1/ws` und erfordern, dass der TLS-Terminator die HTTP/1.1-Upgrade-Header unverändert weiterleitet.
|
||||
|
||||
Fehlt `carriers` oder ist es `false`, sind Auto-Negotiation und Lernen deaktiviert und `carrier` ist der einzige Modus. Ein nicht leeres `carriers`-Array aktiviert die Start-Negotiation in der konfigurierten Reihenfolge; `carrier` wird genau einmal als letzter Fallback angehängt. Leere Arrays, Duplikate und `true` werden abgelehnt. Der Client darf nur vor dem Carrier-Commit zum nächsten Kandidaten wechseln; nach dem Commit erfordert ein Carrier-Wechsel eine neue Sitzung. Ein nativer Client ohne Metadaten, einschließlich Telegram iOS, verwendet immer den konfigurierten festen `carrier`, auch bei aktivierter Negotiation. Das aktuelle iOS unterstützt nur `https`; solche Bereitstellungen müssen daher `carrier = "https"` setzen. Die CFNetwork- und Darwin-User-Agent-Klassifizierung leitet keine Carrier-Unterstützung ab. Explizite native iOS-Capabilities werden mit `{https}` geschnitten; andere explizite Client-Capabilities gelten wie gemeldet.
|
||||
|
||||
`carrier_learning` wirkt nur bei aktivierter Negotiation. Das Lernen ist prozesslokal, speicherresident, begrenzt und ausschließlich positiv: Nur ein Carrier, der den serverdefinierten Zustand healthy erreicht, liefert Evidenz. `conservative` erfordert die breiteste Evidenz und deaktiviert IP-Ranking, `balanced` verwendet mittlere User-Agent-/Profil-Schwellen sowie geeignete öffentliche IPs nur als Tie-Breaker, und `aggressive` reagiert auf die ersten begrenzten Samples. Vom Client gemeldete Fehler bleiben rein diagnostisch und erzeugen keine negative Evidenz. Ein Reload wendet die Richtlinie auf neue Negotiation-Ketten an und verwirft inkompatible gespeicherte Evidenz. Das Deaktivieren von WEB beendet die Ausgabe neuer Bridge- und Session-Zugangsdaten; zum Widerrufen aktiver Sitzungen eines einzelnen Benutzers verwenden Sie die Users-API.
|
||||
|
||||
# [web.debug]
|
||||
|
||||
@@ -2584,6 +2591,8 @@ Diese hot-reload-fähige Tabelle steuert den prozesseigenen serverseitigen WEB-D
|
||||
|
||||
Eine Änderung von `enabled` oder einem Erfassungsfeld löscht gespeicherte Datensätze und verwirft Commits, die unter der vorherigen Policy-Epoche begonnen wurden. Ändert sich nur das standardmäßige oder maximale Beobachtungsfenster, bleiben kompatible Datensätze erhalten. `full` speichert den vollständigen Body eines erkannten Carriers nur bis `web.limits.max_body_bytes`; Decoy-Bodys bleiben immer auf einen Präfix begrenzt. Ein Präfix, der nur mit einer gleichzeitig erhöhten, neustartpflichtigen Kapazität zulässig wäre, wird zusammen mit `web.debug` bis zum Neustart zurückgestellt. URI-Queries werden nie gespeichert, Werte von Credential-Headern werden ausgelassen, Body-Kopien werden von bekannten WEB-Capabilities und Bearer-Tokens bereinigt und Profilschlüssel ausschließlich als domänengetrennter Fingerprint mit 16 Hex-Zeichen dargestellt.
|
||||
|
||||
Die authentifizierte JSON-Steuerung kann den Ring mit `POST /v1/runtime/web/debug/clear` explizit löschen. Die erforderliche prozessbezogene `runtime_instance` sperrt veraltete Controller, die zurückgegebene Epoche sperrt laufende Writer und `leased_bytes` meldet Speicher, der noch von bereits gerenderten Snapshots gehalten wird.
|
||||
|
||||
# [web.limits]
|
||||
|
||||
Diese prozessweiten Obergrenzen begrenzen alle WEB-Register, Warteschlangen, Request-Bodys, statischen Snapshots und Admission-Pfade. Alle Werte werden gemeinsam validiert: Eigentümerbezogene Grenzen dürfen die globalen Grenzen nicht überschreiten, Queue-Reserven müssen den Fortschritt von Control Frames gewährleisten, Body-Reservierungen müssen in ihr globales Budget passen und alle deklarierten Byte-Grenzen müssen in `memory_envelope_bytes` passen. Jede Änderung in dieser Tabelle erfordert einen Prozessneustart.
|
||||
@@ -2597,10 +2606,15 @@ Diese prozessweiten Obergrenzen begrenzen alle WEB-Register, Warteschlangen, Req
|
||||
| `max_frames_per_body` | `usize` | `4096` | Maximale Zahl geparster oder ausgegebener Frames pro Carrier-Body. |
|
||||
| `max_http_connections` | `usize` | `1024` | Prozessweit akzeptierte WEB-HTTP-Verbindungen. |
|
||||
| `max_http_handlers` | `usize` | `512` | Prozessweit gleichzeitig ausgeführte HTTP-Handler; HTTPS-Lanes dürfen höchstens die Hälfte mit Long Polls belegen, der Rest bleibt für Session-, Uplink- und Steuerarbeit verfügbar. |
|
||||
| `max_lane_open_waits_per_session` | `usize` | `16` | Kanonische Cursor-null-Downlink-Polls, die pro Sitzung auf ein konkurrierendes Lane-`OPEN` warten dürfen. |
|
||||
| `pending_bytes_per_lane` | `usize` | `8388608` | Eingereihte und residente `DATA`-Bytes pro unabhängiger HTTPS- oder WebSocket-Lane. |
|
||||
| `pending_items_per_lane` | `usize` | `1024` | Eingereihte und residente `DATA`-Elemente pro unabhängiger HTTPS- oder WebSocket-Lane. |
|
||||
| `websocket_bytes_global` | `usize` | `268435456` | Transientes Teilbudget für WebSocket-Codecs, Messages und Write-Staging innerhalb von `pending_bytes_global`. |
|
||||
| `websocket_admission_watermark_pct` | `u8` | `75` | WebSocket-Byte-Anteil, ab dem neue Admission eine Owner-First-Verbindung ersetzen darf. |
|
||||
| `websocket_eviction_watermark_pct` | `u8` | `90` | WebSocket-Byte-Anteil, ab dem Queue-Druck die zulässige Verbindung mit dem ältesten Fortschritt verdrängen darf. |
|
||||
| `websocket_admission_watermark_pct` | `u8` | `75` | WebSocket-Byte-Watermark für neue Basis-Admission und die Fair-Share-Berechnung des deterministischen Ersatzes. |
|
||||
| `websocket_eviction_watermark_pct` | `u8` | `90` | Watermark für WebSocket-Datenallokationen, ab dem gemeinsamer Queue-Druck ein deterministisches Cleanup anfordern darf. |
|
||||
| `websocket_http_connection_reserve` | `usize` | `64` | Für WebSocket-Upgrades gesperrte HTTP-Verbindungen, die Kapazität für gewöhnliches HTTP und Decoys erhalten. |
|
||||
| `max_websocket_evictions_in_flight` | `usize` | `8` | Prozessweite Obergrenze gleichzeitiger exakter WebSocket-Verdrängungs-Claims bei Admission und Druck-Cleanup. |
|
||||
| `max_carrier_learning_entries` | `usize` | `4096` | Prozessweite Obergrenze begrenzter Carrier-Learning-Evidenzeinträge. |
|
||||
| `max_body_readers` | `usize` | `32` | Prozessweit gleichzeitig gesammelte Request-Bodys. |
|
||||
| `max_body_bytes_global` | `usize` | `67108864` | Globales Byte-Budget für gesammelte Bodys. |
|
||||
| `max_sessions_global` | `usize` | `128` | Prozessweit aktive WEB-Sitzungen. |
|
||||
@@ -2624,7 +2638,7 @@ Diese prozessweiten Obergrenzen begrenzen alle WEB-Register, Warteschlangen, Req
|
||||
| `max_static_bytes` | `usize` | `67108864` | Bytes statischer Snapshots über alle vhosts. |
|
||||
| `debug_records_capacity` | `usize` | `65536` | Maximale Zahl gespeicherter WEB-Debugdatensätze. |
|
||||
| `debug_bytes_global` | `usize` | `67108864` | Globale Byte-Obergrenze für gespeicherte und in Verarbeitung befindliche WEB-Debugdaten; mindestens 4096. |
|
||||
| `memory_envelope_bytes` | `usize` | `805306368` | Deklarierter Rahmen für HTTP-Heads, Bodys, gemeinsame Queues/WebSocket-I/O, statische Snapshots und begrenzte Debug-/Statuspuffer; maximal 4 GiB. |
|
||||
| `memory_envelope_bytes` | `usize` | `1342177280` | Deklarierter Rahmen für HTTP-Heads, Bodys, gemeinsame Queues/WebSocket-I/O, Lane-Zustand, Carrier-Learning, statische Snapshots und begrenzte Debug-/Statuspuffer; maximal 4 GiB. |
|
||||
| `new_bootstraps_per_minute` | `u32` | `1200` | Nachhaltige prozessweite Ausgaberate für Bootstraps. |
|
||||
| `new_bootstraps_burst` | `u32` | `256` | Prozessweiter Burst für die Bootstrap-Ausgabe. |
|
||||
| `new_sessions_per_minute` | `u32` | `600` | Nachhaltige prozessweite Erstellungsrate für Sitzungen. |
|
||||
@@ -2634,21 +2648,31 @@ Diese prozessweiten Obergrenzen begrenzen alle WEB-Register, Warteschlangen, Req
|
||||
|
||||
# [web.timeouts]
|
||||
|
||||
Alle Timeouts werden in Sekunden angegeben und müssen im Bereich `1..=3600` liegen. Die längste Request-Deadline muss kleiner als `http_idle_secs` sein.
|
||||
Sofern eine Zeile nichts anderes angibt, werden Timeouts in Sekunden angegeben und müssen im Bereich `1..=3600` liegen. Konfigurierte serverseitige Deadlines einzelner HTTP-Phasen müssen kleiner als `http_idle_secs` sein; geschützte Phasen behalten ihre eigenen Deadlines, sodass der Idle-Timer keine Gesamtdeadline für einen Request ist. Das clientseitige Bridge-Retry-Fenster hat eigene Grenzen.
|
||||
|
||||
| Schlüssel | Typ | Default | Hot-Reload | Beschreibung |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `header_secs` | `u64` | `10` | `✔` | Empfang eines vollständigen HTTP-Request-Heads. |
|
||||
| `body_secs` | `u64` | `30` | `✔` | Sammeln eines authentifizierten Carrier-Bodys. |
|
||||
| `stream_handshake_secs` | `u64` | `10` | `✔` | Abschluss eines inneren MTProxy-Handshakes. |
|
||||
| `stream_first_byte_secs` | `u64` | `30` | `✔` | Empfang des ersten inneren MTProxy-Bytes nach `OPEN`; validiert im Bereich `1..=300`. |
|
||||
| `long_poll_secs` | `u64` | `25` | `✔` | Maximale Dauer eines leeren Downlink-Long-Polls. |
|
||||
| `bridge_request_secs` | `u64` | `10` | `✔` | Bridge-seitige Deadline eines HTTP-Versuchs bis zum vollständigen Lesen des Response-Bodys; `/down` erhält zusätzlich `long_poll_secs`. Bereich `1..=60`. |
|
||||
| `bridge_retry_secs` | `u64` | `90` | `✔` | Absolutes Bridge-Retry-Fenster einschließlich Versuchen und Backoff; Bereich `1..=300` und nicht kleiner als `bridge_request_secs`. |
|
||||
| `carrier_probe_coalesce_ms` | `u64` | `0` | `✔` | Optionales Bridge-Warten nach `OPEN` auf passendes `DATA`; Millisekunden im Bereich `0..=10`, wobei `0` sofortiges Probing beibehält. |
|
||||
| `lane_open_wait_secs` | `u64` | `2` | `✔` | Wartezeit für einen kanonischen Cursor-null-Downlink, der sein Lane-`OPEN` überholt; höchstens `long_poll_secs`. |
|
||||
| `carrier_health_secs` | `u64` | `30` | `✔` | Beobachtungsintervall nach dem Commit, bevor ein Carrier Learning-Evidenz liefern kann. |
|
||||
| `websocket_upgrade_secs` | `u64` | `5` | `✔` | Maximale Wartezeit, bis ein akzeptiertes HTTP-Upgrade zum WebSocket wird; Bereich `1..=60`. |
|
||||
| `websocket_open_secs` | `u64` | `15` | `✔` | Absolute Deadline für die erste Carrier-Binärnachricht nach dem Upgrade; Bereich `1..=300`. |
|
||||
| `websocket_write_secs` | `u64` | `30` | `✔` | Maximale Wartezeit für einen WebSocket-Write oder Flush. |
|
||||
| `websocket_backpressure_secs` | `u64` | `30` | `✔` | Maximale Wartezeit auf Fortschritt des gemeinsamen Byte-Budgets oder einer Queue, bevor die betroffene Verbindung geschlossen wird. |
|
||||
| `websocket_eviction_secs` | `u64` | `1` | `✔` | Karenzzeit, in der ein verdrängter WebSocket Slot und Budget freigeben muss, bevor Admission fehlschlägt. |
|
||||
| `carrier_negotiation_deadlines_secs` | `[u64; 4]` | `[3, 5, 8, 12]` | `✔` | Streng steigende kumulative Offsets: Die Bridge verwendet sie vor ihrem ersten `/session`-Request, der Server bei Annahme des ersten automatischen Versuchs. Die Checkpoints für ein bis vier Kandidaten sind `[d3]`, `[d0, d3]`, `[d0, d1, d3]` und `[d0, d1, d2, d3]`; der letzte Kandidat verwendet immer `d3`. |
|
||||
| `carrier_learning_secs` | `u64` | `600` | `✔` | Feste Lebensdauer zweier prozesslokaler Evidenzfenster; Bereich `2..=86400`. |
|
||||
| `bootstrap_lifetime_secs` | `u64` | `120` | `✔` | Lebensdauer ungenutzter Bootstraps und geschlossener Token-Replay-Marker. |
|
||||
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Maximale Carrier-Inaktivität bis zum Schließen der Sitzung. |
|
||||
| `http_idle_secs` | `u64` | `75` | `✔` | Idle-Lebensdauer einer WEB-HTTP-Keep-Alive-Verbindung. |
|
||||
| `shutdown_secs` | `u64` | `15` | `✔` | Deadline für das kontrollierte Beenden von WEB. |
|
||||
| `http_idle_secs` | `u64` | `75` | `✔` | Idle-Grenze zwischen HTTP-Austauschvorgängen und bei ausbleibendem Fortschritt eines bereits ausgegebenen Response-Bodys. Explizit begrenzte Request-Body-, Long-Poll-, Decoy- und ausstehende Upgrade-Phasen behalten ihre eigenen Deadlines und werden nicht durch diesen Timer verkürzt. Der Wert wird beim Annehmen der Verbindung fixiert. |
|
||||
| `shutdown_secs` | `u64` | `15` | `✔` | Ein absolutes Budget für das Beenden des Prozesses, das von allen Listener-Acceptoren und Verbindungen sowie WEB-Sitzungs- und Hilfstask-Drains gemeinsam verwendet wird. Der aktive Wert wird beim Start des Shutdowns einmalig erfasst. |
|
||||
| `decoy_header_secs` | `u64` | `30` | `✔` | Deadline für Verbindung und Response-Head eines HTTP-Decoys. |
|
||||
|
||||
# [[web.vhosts]]
|
||||
@@ -2675,7 +2699,7 @@ Genau ein Decoy-Modus ist erforderlich:
|
||||
|
||||
| Schlüssel | Typ | Erforderlich | Default | Beschreibung |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `user` | `String` | ja | — | Vorhandener Schlüssel aus `[access.users]`. |
|
||||
| `user` | `String` | ja | — | Vorhandener Schlüssel mit 1–64 Byte aus `[access.users]`; die Grenze hält Runtime-Status und Filter beschränkt. |
|
||||
| `secret_mode` | `"plain"` oder `"dd"` | ja | — | Exakte Secret-Darstellung für Telegram Desktop. `ee` wird nicht unterstützt. |
|
||||
| `max_sessions` | `usize` | nein | `web.limits.max_sessions_global` | Aktive Sitzungen für dieses Profil. |
|
||||
| `max_streams` | `usize` | nein | `web.limits.max_streams_global` | Aktive logische Streams für dieses Profil. |
|
||||
@@ -2685,10 +2709,11 @@ Profilgrenzen müssen ungleich null sein und dürfen die zugehörigen globalen G
|
||||
|
||||
## WEB-Lebenszyklus und API-Verwaltung
|
||||
|
||||
- Config-Watcher und Generations-Reload wenden `web.enabled`, `web.carrier`, `web.debug`, `web.timeouts`, vhosts, Profile und Decoy-Snapshots ohne Prozessneustart an. Bestehende Sitzungen behalten Carrier, Grenzen und Deadlines ihres Erstellungszeitpunkts; neu ausgegebene Bridge-Sitzungen verwenden die aktive Generation.
|
||||
- Config-Watcher und Generations-Reload wenden `web.enabled`, Carrier- und Negotiation-Richtlinie, `web.debug`, `web.timeouts`, vhosts, Profile und Decoy-Snapshots ohne Prozessneustart an. Ein einzelner unveränderlicher expandierter Source-Snapshot wird validiert und aktiviert; der Watcher einer Kandidatengeneration startet erst nach deren Aktivierung. Bestehende Sitzungen und laufende Negotiation-Ketten behalten Carrier-Kandidaten, Grenzen, Timeouts und absolute Deadlines ihres Ausgabezeitpunkts; neue Bridge-Sitzungen verwenden genau eine fixierte aktive Generation.
|
||||
- Bestand und Vertrauensrichtlinie der WEB-Listener unter `server.listeners` sowie alle Werte in `web.limits` sind prozesseigen und erfordern einen Neustart.
|
||||
- Es gibt keine veränderbare Ressource `/v1/web`. `GET /web-status` stellt authentifizierte, schreibgeschützte HTML-Diagnosen bereit; `GET /v1/config` lässt `[web]` aus und `PATCH /v1/config` lehnt einen Schlüssel `web` mit `400 section_not_editable` ab.
|
||||
- Zum entfernten Anwenden einer WEB-Richtlinie ändern Sie die zuständige TOML-Datei und rufen `POST /v1/system/reload` auf. Prüfen Sie anschließend `GET /v1/system/reload/{id}` und dessen `deferred_process_fields`. Starten Sie Telemt neu, wenn das Feld `server.listeners` oder `web.limits` enthält.
|
||||
- `GET /v1/config` liefert den vollständigen verfassten `[web]`-Baum außer dem abgeleiteten Snapshot `web.runtime`. `PATCH /v1/config` akzeptiert ein dünn besetztes `web`-Objekt, führt Tabellen tief zusammen, ersetzt Arrays vollständig, validiert den gesamten Kandidaten und meldet `web.limits` bis zum Neustart in `deferred_process_fields`.
|
||||
- `GET /v1/runtime/web/status`, `/sessions`, `/sessions/{session_ref}` und `/operations/{operation_id}` stellen begrenzten, nicht geheimen Runtime-Zustand bereit. POST-Steuerungen schließen ausgewählte Sitzungen, löschen Debugdaten oder setzen Carrier-Learning zurück und verlangen die aktuelle zufällige `runtime_instance`.
|
||||
- `web.enabled = false` stoppt nach der Aktivierung neue Bootstrap-/Session-Ausgaben, schließt aber keine aktiven Sitzungen. Warten Sie für Close-all auf `manager.issuance_enabled = false`, senden Sie den asynchronen Selektor `all` und fragen Sie dessen Operation ab.
|
||||
- Vorhandene Access-Benutzer können über `/v1/users` erstellt, geändert, rotiert, aktiviert, deaktiviert und gelöscht werden. Das Erstellen eines Benutzers fügt kein WEB-Profil hinzu. Das Deaktivieren aktualisiert die Admission sofort und beendet die aktiven Sitzungen dieses Benutzers.
|
||||
- `PATCH /v1/config` kann `server.listeners` einschließlich der WEB-Listener-Felder speichern; ein geänderter WEB-Listener wird jedoch erst nach einem Prozessneustart aktiv.
|
||||
|
||||
|
||||
@@ -2558,12 +2558,19 @@ WEB mode carries Telegram Desktop MTProxy traffic through HTTPS terminated by an
|
||||
| --- | --- | --- | --- |
|
||||
| `enabled` | `bool` | `false` | `✔` |
|
||||
| `carrier` | `"https"`, `"https-lanes"`, `"websocket"`, or `"websocket-lanes"` | `"https"` | `✔` |
|
||||
| `carriers` | `false` or a non-empty array of unique carriers | `false` | `✔` |
|
||||
| `carrier_learning` | `bool` | `true` | `✔` |
|
||||
| `carrier_negotiation_aggressiveness` | `"conservative"`, `"balanced"`, or `"aggressive"` | `"conservative"` | `✔` |
|
||||
| `debug` | table | disabled, bounded defaults | `✔` |
|
||||
| `limits` | table | bounded defaults | `✘` |
|
||||
| `timeouts` | table | bounded defaults | `✔` |
|
||||
| `vhosts` | array of tables | `[]` | `✔` |
|
||||
|
||||
`enabled = true` requires at least one network-eligible WEB listener, at least one vhost, and at least one profile in every vhost. `https` preserves the serialized HTTPS transport. `https-lanes` gives stream zero and every logical stream independent uplink sequencing, downlink cursors, retries, and long polls; it requires `max_http_handlers >= 2` and public HTTP/2 on the TLS terminator. `websocket` carries all logical streams over one ordered RFC 6455 connection, while `websocket-lanes` owns one connection per non-zero logical stream and isolates lane failures. Both WebSocket carriers use `GET /api/v1/ws` after HTTPS session creation and require the TLS terminator to preserve HTTP/1.1 Upgrade headers. A reload applies `carrier` only to newly issued bridge sessions. Disabling WEB stops issuance of new bridge and session credentials after reload; use the users API to revoke one user's active sessions.
|
||||
`enabled = true` requires at least one network-eligible WEB listener, at least one vhost, and at least one profile in every vhost. `https` preserves the serialized HTTPS transport and requires `max_http_handlers >= 2`. `https-lanes` gives stream zero and every logical stream independent uplink sequencing, downlink cursors, retries, and long polls; it requires `max_http_handlers >= 4` and public HTTP/2 on the TLS terminator. `websocket` carries all logical streams over one ordered RFC 6455 connection, while `websocket-lanes` owns one connection per non-zero logical stream and isolates lane failures. Both WebSocket carriers use `GET /api/v1/ws` after HTTPS session creation and require the TLS terminator to preserve HTTP/1.1 Upgrade headers.
|
||||
|
||||
When `carriers` is missing or `false`, auto-negotiation and learning are disabled and `carrier` is the only mode. A non-empty `carriers` array enables startup-only negotiation in its configured order; `carrier` is appended exactly once as the final fallback. Empty arrays, duplicates, and `true` are rejected. The client advances candidates only before carrier commit and must create a new session to change carrier after commit. A metadata-free native client, including Telegram iOS, always uses the configured fixed `carrier`, even when negotiation is enabled. Current iOS supports only `https`, so such deployments must configure `carrier = "https"`. CFNetwork and Darwin User-Agent classification does not infer carrier support. Explicit native iOS capabilities are intersected with `{https}`; other explicit client capabilities participate as reported.
|
||||
|
||||
`carrier_learning` applies only while negotiation is enabled. Learning is process-local, in-memory, bounded, and positive-only: only a carrier that reaches the server-defined healthy state contributes evidence. `conservative` requires the broadest evidence and disables IP ranking, `balanced` admits moderate User-Agent/profile evidence plus eligible public-IP tie breaking, and `aggressive` reacts to the first bounded samples. Reported client failures remain diagnostic and never create negative evidence. Reload applies the policy to new negotiation chains and invalidates incompatible retained evidence. Disabling WEB stops issuance of new bridge and session credentials after reload; use the users API to revoke one user's active sessions.
|
||||
|
||||
# [web.debug]
|
||||
|
||||
@@ -2584,6 +2591,8 @@ This hot-reloadable table controls the process-owned server-side WEB debug recor
|
||||
|
||||
Changing `enabled` or any capture field clears retained records and rejects commits started under the previous policy epoch. Changing only the default or maximum observation window preserves compatible retained records. `full` retains a complete recognized carrier body only up to `web.limits.max_body_bytes`; decoy bodies always remain prefix-bounded. A prefix that depends on a simultaneously increased restart-only capacity is deferred with `web.debug` until restart. URI queries are never retained, credential header values are omitted, body copies are scrubbed for known WEB capabilities and bearer tokens, and profile keys are represented only by a domain-separated 16-hex fingerprint.
|
||||
|
||||
Authenticated JSON control may clear the ring explicitly with `POST /v1/runtime/web/debug/clear`; the required process `runtime_instance` fences stale controllers, the returned epoch fences in-flight writers, and `leased_bytes` reports memory still owned by already rendered snapshots.
|
||||
|
||||
# [web.limits]
|
||||
|
||||
These process-wide ceilings make every WEB registry, queue, request body, static snapshot, and admission path bounded. All values are validated together. Per-owner limits cannot exceed global limits, queue reserves must preserve control-frame progress, body reservations must fit their global budget, and all declared byte ceilings must fit `memory_envelope_bytes`. Changing any value in this table requires a process restart.
|
||||
@@ -2597,10 +2606,15 @@ These process-wide ceilings make every WEB registry, queue, request body, static
|
||||
| `max_frames_per_body` | `usize` | `4096` | Maximum frames parsed or emitted per carrier body. |
|
||||
| `max_http_connections` | `usize` | `1024` | Accepted WEB HTTP connections process-wide. |
|
||||
| `max_http_handlers` | `usize` | `512` | Concurrent HTTP handlers process-wide; HTTPS lanes may park at most half, preserving the remainder for session, uplink, and control work. |
|
||||
| `max_lane_open_waits_per_session` | `usize` | `16` | Canonical cursor-zero downlink polls allowed to wait for a racing lane `OPEN` in one session. |
|
||||
| `pending_bytes_per_lane` | `usize` | `8388608` | Queued and resident `DATA` bytes allowed for one independent HTTPS or WebSocket lane. |
|
||||
| `pending_items_per_lane` | `usize` | `1024` | Queued and resident `DATA` items allowed for one independent HTTPS or WebSocket lane. |
|
||||
| `websocket_bytes_global` | `usize` | `268435456` | Transient WebSocket codec, message, and write-staging sub-budget inside `pending_bytes_global`. |
|
||||
| `websocket_admission_watermark_pct` | `u8` | `75` | WebSocket byte percentage at which new admission may replace an owner-first victim. |
|
||||
| `websocket_eviction_watermark_pct` | `u8` | `90` | WebSocket byte percentage at which queue pressure may evict the least-recently-progressed eligible connection. |
|
||||
| `websocket_admission_watermark_pct` | `u8` | `75` | WebSocket byte watermark for new base admission and the fair-share calculation used by deterministic replacement. |
|
||||
| `websocket_eviction_watermark_pct` | `u8` | `90` | WebSocket data-allocation watermark at which shared queue pressure may request deterministic cleanup. |
|
||||
| `websocket_http_connection_reserve` | `usize` | `64` | Accepted HTTP connections unavailable to WebSocket upgrades, preserving ordinary HTTP and decoy capacity. |
|
||||
| `max_websocket_evictions_in_flight` | `usize` | `8` | Process-wide ceiling for concurrent exact WebSocket eviction claims during admission and pressure cleanup. |
|
||||
| `max_carrier_learning_entries` | `usize` | `4096` | Process-wide ceiling for bounded carrier-learning evidence entries. |
|
||||
| `max_body_readers` | `usize` | `32` | Concurrent collected request bodies process-wide. |
|
||||
| `max_body_bytes_global` | `usize` | `67108864` | Global byte reservation for collected bodies. |
|
||||
| `max_sessions_global` | `usize` | `128` | Live WEB sessions process-wide. |
|
||||
@@ -2624,7 +2638,7 @@ These process-wide ceilings make every WEB registry, queue, request body, static
|
||||
| `max_static_bytes` | `usize` | `67108864` | Static snapshot bytes across all vhosts. |
|
||||
| `debug_records_capacity` | `usize` | `65536` | Maximum retained WEB debug record count. |
|
||||
| `debug_bytes_global` | `usize` | `67108864` | Retained plus in-flight WEB debug byte ceiling; minimum 4096. |
|
||||
| `memory_envelope_bytes` | `usize` | `805306368` | Declared envelope for HTTP heads, bodies, shared queues/WebSocket I/O, static snapshots, and bounded debug/status buffers; maximum 4 GiB. |
|
||||
| `memory_envelope_bytes` | `usize` | `1342177280` | Declared envelope for HTTP heads, bodies, shared queues/WebSocket I/O, lane state, carrier learning, static snapshots, and bounded debug/status buffers; maximum 4 GiB. |
|
||||
| `new_bootstraps_per_minute` | `u32` | `1200` | Sustained process-wide bootstrap issuance rate. |
|
||||
| `new_bootstraps_burst` | `u32` | `256` | Process-wide bootstrap issuance burst. |
|
||||
| `new_sessions_per_minute` | `u32` | `600` | Sustained process-wide session creation rate. |
|
||||
@@ -2634,21 +2648,31 @@ These process-wide ceilings make every WEB registry, queue, request body, static
|
||||
|
||||
# [web.timeouts]
|
||||
|
||||
Every timeout is measured in seconds and must be within `1..=3600`. The longest request deadline must be lower than `http_idle_secs`.
|
||||
Unless a row states otherwise, timeouts are measured in seconds and must be within `1..=3600`. Configured server-side HTTP phase deadlines must be lower than `http_idle_secs`; protected phases retain their own deadlines, so the idle timer is not an aggregate request deadline. The bridge retry window is client-side and follows its own bound.
|
||||
|
||||
| Key | Type | Default | Hot-Reload | Description |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `header_secs` | `u64` | `10` | `✔` | Receive one complete HTTP request head. |
|
||||
| `body_secs` | `u64` | `30` | `✔` | Collect one authenticated carrier body. |
|
||||
| `stream_handshake_secs` | `u64` | `10` | `✔` | Complete one inner MTProxy handshake. |
|
||||
| `stream_first_byte_secs` | `u64` | `30` | `✔` | Receive the first inner MTProxy byte after `OPEN`; validated within `1..=300`. |
|
||||
| `long_poll_secs` | `u64` | `25` | `✔` | Maximum empty downlink long poll. |
|
||||
| `bridge_request_secs` | `u64` | `10` | `✔` | Bridge-side deadline for one HTTP attempt through complete response-body consumption; `/down` additionally allows `long_poll_secs`. Validated within `1..=60`. |
|
||||
| `bridge_retry_secs` | `u64` | `90` | `✔` | Absolute bridge retry window including attempts and backoff; validated within `1..=300` and no lower than `bridge_request_secs`. |
|
||||
| `carrier_probe_coalesce_ms` | `u64` | `0` | `✔` | Optional bridge wait after `OPEN` for matching `DATA`; milliseconds within `0..=10`, where `0` preserves immediate probing. |
|
||||
| `lane_open_wait_secs` | `u64` | `2` | `✔` | Wait for a canonical cursor-zero downlink that races its lane `OPEN`; no greater than `long_poll_secs`. |
|
||||
| `carrier_health_secs` | `u64` | `30` | `✔` | Post-commit observation interval required before a carrier can contribute learning evidence. |
|
||||
| `websocket_upgrade_secs` | `u64` | `5` | `✔` | Maximum wait for an accepted HTTP Upgrade to become a WebSocket; validated within `1..=60`. |
|
||||
| `websocket_open_secs` | `u64` | `15` | `✔` | Absolute deadline for the first carrier binary message after Upgrade; validated within `1..=300`. |
|
||||
| `websocket_write_secs` | `u64` | `30` | `✔` | Maximum wait for one WebSocket write or flush. |
|
||||
| `websocket_backpressure_secs` | `u64` | `30` | `✔` | Maximum wait for shared byte-budget or queue progress before closing the affected connection. |
|
||||
| `websocket_eviction_secs` | `u64` | `1` | `✔` | Grace allowed for a pressure-evicted WebSocket to release its slot and budget before admission fails. |
|
||||
| `carrier_negotiation_deadlines_secs` | `[u64; 4]` | `[3, 5, 8, 12]` | `✔` | Strictly increasing cumulative offsets used by the bridge before its first `/session` request and by the server when accepting the first automatic attempt. Checkpoints for one through four candidates are `[d3]`, `[d0, d3]`, `[d0, d1, d3]`, and `[d0, d1, d2, d3]`; the final candidate always uses `d3`. |
|
||||
| `carrier_learning_secs` | `u64` | `600` | `✔` | Fixed two-window process-local evidence lifetime; validated within `2..=86400`. |
|
||||
| `bootstrap_lifetime_secs` | `u64` | `120` | `✔` | Unused bootstrap and closed-token replay lifetime. |
|
||||
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Maximum carrier inactivity before session closure. |
|
||||
| `http_idle_secs` | `u64` | `75` | `✔` | WEB HTTP keep-alive idle lifetime. |
|
||||
| `shutdown_secs` | `u64` | `15` | `✔` | Graceful WEB shutdown deadline. |
|
||||
| `http_idle_secs` | `u64` | `75` | `✔` | Idle limit between HTTP exchanges and while an emitted response body makes no progress. Explicitly bounded request-body, long-poll, decoy, and pending-Upgrade phases keep their own deadlines instead of being truncated by this timer. The value is frozen when the connection is accepted. |
|
||||
| `shutdown_secs` | `u64` | `15` | `✔` | One absolute process-shutdown budget shared by all listener acceptors and connections plus WEB session and auxiliary-task drains. The active value is captured once when shutdown starts. |
|
||||
| `decoy_header_secs` | `u64` | `30` | `✔` | Connect and response-head deadline for an HTTP decoy. |
|
||||
|
||||
# [[web.vhosts]]
|
||||
@@ -2675,7 +2699,7 @@ Exactly one decoy mode is required:
|
||||
|
||||
| Key | Type | Required | Default | Description |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `user` | `String` | yes | — | Existing key from `[access.users]`. |
|
||||
| `user` | `String` | yes | — | Existing 1–64-byte key from `[access.users]`; the bound keeps runtime status and filters bounded. |
|
||||
| `secret_mode` | `"plain"` or `"dd"` | yes | — | Exact Telegram Desktop secret representation. `ee` is not supported. |
|
||||
| `max_sessions` | `usize` | no | `web.limits.max_sessions_global` | Live sessions for this profile. |
|
||||
| `max_streams` | `usize` | no | `web.limits.max_streams_global` | Live logical streams for this profile. |
|
||||
@@ -2685,10 +2709,11 @@ Profile limits must be non-zero and no greater than their corresponding global l
|
||||
|
||||
## WEB lifecycle and API management
|
||||
|
||||
- The config watcher and generation reload apply `web.enabled`, `web.carrier`, `web.debug`, `web.timeouts`, vhosts, profiles, and decoy snapshots without a process restart. Existing sessions keep their acquisition-time carrier, limits, and deadlines; newly issued bridge sessions use the active generation.
|
||||
- The config watcher and generation reload apply `web.enabled`, carrier and negotiation policy, `web.debug`, `web.timeouts`, vhosts, profiles, and decoy snapshots without a process restart. One immutable expanded source snapshot is validated and activated; a candidate generation's watcher starts only after that generation becomes active. Existing sessions and in-flight negotiation chains keep their issuance-time carrier candidates, limits, timeouts, and absolute deadlines; newly issued bridge sessions use one pinned active generation.
|
||||
- WEB listener inventory and trust policy under `server.listeners`, and every `web.limits` value, are process-owned and restart-required.
|
||||
- There is no mutable `/v1/web` resource. `GET /web-status` provides authenticated read-only HTML diagnostics; `GET /v1/config` omits `[web]`, and `PATCH /v1/config` rejects a `web` key with `400 section_not_editable`.
|
||||
- To manage WEB policy remotely, update the owned TOML file and call `POST /v1/system/reload`; inspect `GET /v1/system/reload/{id}` and its `deferred_process_fields`. Restart Telemt when it contains `server.listeners` or `web.limits`.
|
||||
- `GET /v1/config` returns the complete authored `[web]` tree except the derived `web.runtime` snapshot. `PATCH /v1/config` accepts a sparse `web` object, deep-merges tables, replaces arrays wholesale, validates the complete candidate, and reports `web.limits` in `deferred_process_fields` until restart.
|
||||
- `GET /v1/runtime/web/status`, `/sessions`, `/sessions/{session_ref}`, and `/operations/{operation_id}` expose bounded non-secret runtime state. POST controls close selected sessions, clear debug data, or reset carrier learning and require the current random `runtime_instance`.
|
||||
- `web.enabled = false` stops new bootstrap/session issuance after activation but does not close live sessions. For close-all, wait until status reports `manager.issuance_enabled = false`, submit the asynchronous `all` selector, and poll its operation.
|
||||
- Existing access users can be created, changed, rotated, enabled, disabled, and deleted through `/v1/users`. Creating a user does not add a WEB profile. Disabling a user immediately updates admission and cancels that user's active sessions.
|
||||
- `PATCH /v1/config` can persist `server.listeners`, including WEB listener fields, but a changed WEB listener does not become active until process restart.
|
||||
|
||||
|
||||
@@ -2484,12 +2484,19 @@ WEB-режим переносит MTProxy-трафик Telegram Desktop внут
|
||||
| --- | --- | --- | --- |
|
||||
| `enabled` | `bool` | `false` | `✔` |
|
||||
| `carrier` | `"https"`, `"https-lanes"`, `"websocket"` или `"websocket-lanes"` | `"https"` | `✔` |
|
||||
| `carriers` | `false` или непустой массив уникальных carrier | `false` | `✔` |
|
||||
| `carrier_learning` | `bool` | `true` | `✔` |
|
||||
| `carrier_negotiation_aggressiveness` | `"conservative"`, `"balanced"` или `"aggressive"` | `"conservative"` | `✔` |
|
||||
| `debug` | таблица | выключено, ограниченные defaults | `✔` |
|
||||
| `limits` | таблица | ограниченные defaults | `✘` |
|
||||
| `timeouts` | таблица | ограниченные defaults | `✔` |
|
||||
| `vhosts` | массив таблиц | `[]` | `✔` |
|
||||
|
||||
Для `enabled = true` нужен как минимум один доступный по сетевой политике WEB-listener, один vhost и один профиль в каждом vhost. `https` сохраняет сериализованный HTTPS transport. В `https-lanes` stream zero и каждый logical stream получают независимые uplink sequence, downlink cursor, retry и long poll; carrier требует `max_http_handlers >= 2` и публичного HTTP/2 на TLS-терминаторе. `websocket` переносит все logical streams через одно упорядоченное RFC 6455 connection, а `websocket-lanes` выделяет отдельное connection каждому ненулевому stream и изолирует сбои lane. Оба WebSocket carrier используют `GET /api/v1/ws` после создания HTTPS-сессии и требуют от TLS-терминатора сохранять HTTP/1.1 Upgrade headers. Reload применяет `carrier` только к новым bridge sessions. Отключение WEB после reload прекращает выдачу новых bridge- и session-credentials; для отзыва активных сессий отдельного пользователя используйте users API.
|
||||
Для `enabled = true` нужен как минимум один доступный по сетевой политике WEB-listener, один vhost и один профиль в каждом vhost. `https` сохраняет сериализованный HTTPS transport и требует `max_http_handlers >= 2`. В `https-lanes` stream zero и каждый logical stream получают независимые uplink sequence, downlink cursor, retry и long poll; carrier требует `max_http_handlers >= 4` и публичного HTTP/2 на TLS-терминаторе. `websocket` переносит все logical streams через одно упорядоченное RFC 6455 connection, а `websocket-lanes` выделяет отдельное connection каждому ненулевому stream и изолирует сбои lane. Оба WebSocket carrier используют `GET /api/v1/ws` после создания HTTPS-сессии и требуют от TLS-терминатора сохранять HTTP/1.1 Upgrade headers.
|
||||
|
||||
Если `carriers` отсутствует или равен `false`, auto-negotiation и обучение выключены, а `carrier` задаёт единственный режим. Непустой массив `carriers` включает стартовый перебор в заданном порядке; `carrier` ровно один раз добавляется последним fallback-вариантом. Пустой массив, дубликаты и `true` запрещены. Клиент может перейти к следующему кандидату только до commit carrier; для смены carrier после commit нужна новая сессия. Native-клиент без метаданных, включая Telegram iOS, всегда использует настроенный фиксированный `carrier`, даже при включённом auto-negotiation. Текущий iOS поддерживает только `https`, поэтому такой deployment должен задавать `carrier = "https"`. Классификация User-Agent CFNetwork и Darwin не определяет поддержку carrier. Явные capabilities нативного iOS пересекаются с `{https}`; capabilities остальных явных клиентов применяются как переданы.
|
||||
|
||||
`carrier_learning` действует только при включённом auto-negotiation. Обучение локально для процесса, хранится в памяти, ограничено и учитывает только положительный результат: evidence добавляет лишь carrier, достигший определённого сервером состояния healthy. `conservative` требует наиболее широкой выборки и отключает ранжирование по IP, `balanced` использует умеренные пороги для User-Agent/профиля и допустимый публичный IP только для разрешения равенства, а `aggressive` реагирует на первые ограниченные samples. Сообщённые клиентом ошибки остаются только диагностикой и не создают отрицательный evidence. Reload применяет новую policy к новым цепочкам negotiation и инвалидирует несовместимый сохранённый evidence. Отключение WEB прекращает выдачу новых bridge- и session-credentials; для отзыва активных сессий отдельного пользователя используйте users API.
|
||||
|
||||
# [web.debug]
|
||||
|
||||
@@ -2510,6 +2517,8 @@ WEB-режим переносит MTProxy-трафик Telegram Desktop внут
|
||||
|
||||
Изменение `enabled` или любого поля capture очищает сохранённые записи и отклоняет commits, начатые в предыдущую policy epoch. Изменение только стандартного или максимального окна наблюдения сохраняет совместимые записи. `full` сохраняет полное тело распознанного carrier только до `web.limits.max_body_bytes`; decoy bodies всегда остаются ограничены настроенным prefix. Prefix, который помещается только в одновременно увеличенную restart-only ёмкость, откладывается вместе с `web.debug` до перезапуска. URI queries никогда не сохраняются, значения credential headers исключаются, копии body очищаются от известных WEB capabilities и bearer tokens, а ключи профилей представлены только domain-separated fingerprint из 16 hex-символов.
|
||||
|
||||
Аутентифицированное JSON-управление может явно очистить ring через `POST /v1/runtime/web/debug/clear`: обязательный process `runtime_instance` защищает от устаревшего controller, возвращаемый epoch отсекает in-flight writers, а `leased_bytes` показывает память, всё ещё удерживаемую уже отрисовываемыми snapshots.
|
||||
|
||||
# [web.limits]
|
||||
|
||||
Эти process-wide границы ограничивают все WEB-реестры, очереди, тела запросов, статические snapshots и admission-пути. Значения проверяются совместно: per-owner лимиты не могут превышать глобальные, резервы очередей должны сохранять прогресс control frames, body-резервы должны помещаться в общий бюджет, а все заявленные байтовые границы — в `memory_envelope_bytes`. Изменение любого значения этой таблицы требует перезапуска процесса.
|
||||
@@ -2523,10 +2532,15 @@ WEB-режим переносит MTProxy-трафик Telegram Desktop внут
|
||||
| `max_frames_per_body` | `usize` | `4096` | Максимальное число frames в одном carrier body. |
|
||||
| `max_http_connections` | `usize` | `1024` | Принятые WEB HTTP connections на весь процесс. |
|
||||
| `max_http_handlers` | `usize` | `512` | Одновременно выполняемые HTTP handlers на весь процесс; HTTPS lanes могут занять long polls не более половины лимита, оставляя остаток для session, uplink и control work. |
|
||||
| `max_lane_open_waits_per_session` | `usize` | `16` | Канонические downlink polls с cursor zero, которые могут ожидать конкурирующий lane `OPEN` в одной сессии. |
|
||||
| `pending_bytes_per_lane` | `usize` | `8388608` | Байты queued и resident `DATA`, разрешённые одной независимой HTTPS- или WebSocket-lane. |
|
||||
| `pending_items_per_lane` | `usize` | `1024` | Элементы queued и resident `DATA`, разрешённые одной независимой HTTPS- или WebSocket-lane. |
|
||||
| `websocket_bytes_global` | `usize` | `268435456` | Подбюджет transient WebSocket codec, messages и write staging внутри `pending_bytes_global`. |
|
||||
| `websocket_admission_watermark_pct` | `u8` | `75` | Доля WebSocket byte-budget, после которой новый admission может вытеснить owner-first victim. |
|
||||
| `websocket_eviction_watermark_pct` | `u8` | `90` | Доля WebSocket byte-budget, после которой queue pressure может вытеснить подходящее connection с наиболее старым прогрессом. |
|
||||
| `websocket_admission_watermark_pct` | `u8` | `75` | Watermark WebSocket byte-budget для нового base admission и расчёта fair share при детерминированном replacement. |
|
||||
| `websocket_eviction_watermark_pct` | `u8` | `90` | Watermark выделения WebSocket data, после которого давление общей queue может запросить детерминированный cleanup. |
|
||||
| `websocket_http_connection_reserve` | `usize` | `64` | Число принятых HTTP connections, недоступных WebSocket upgrades и сохраняющих capacity для обычного HTTP и decoy. |
|
||||
| `max_websocket_evictions_in_flight` | `usize` | `8` | Process-wide предел одновременных точных WebSocket eviction claims при admission и pressure cleanup. |
|
||||
| `max_carrier_learning_entries` | `usize` | `4096` | Process-wide предел записей bounded carrier-learning evidence. |
|
||||
| `max_body_readers` | `usize` | `32` | Одновременно собираемые request bodies на весь процесс. |
|
||||
| `max_body_bytes_global` | `usize` | `67108864` | Глобальный байтовый резерв для собранных bodies. |
|
||||
| `max_sessions_global` | `usize` | `128` | Активные WEB-сессии на весь процесс. |
|
||||
@@ -2550,7 +2564,7 @@ WEB-режим переносит MTProxy-трафик Telegram Desktop внут
|
||||
| `max_static_bytes` | `usize` | `67108864` | Размер static snapshots всех vhosts. |
|
||||
| `debug_records_capacity` | `usize` | `65536` | Максимальное число сохранённых WEB debug records. |
|
||||
| `debug_bytes_global` | `usize` | `67108864` | Глобальная байтовая граница сохранённых и находящихся в обработке WEB debug данных; минимум 4096. |
|
||||
| `memory_envelope_bytes` | `usize` | `805306368` | Заявленный envelope для HTTP heads, bodies, общих queues/WebSocket I/O, static snapshots и bounded debug/status buffers; максимум 4 GiB. |
|
||||
| `memory_envelope_bytes` | `usize` | `1342177280` | Заявленный envelope для HTTP heads, bodies, общих queues/WebSocket I/O, состояния lanes, carrier learning, static snapshots и bounded debug/status buffers; максимум 4 GiB. |
|
||||
| `new_bootstraps_per_minute` | `u32` | `1200` | Устойчивая process-wide скорость выдачи bootstrap. |
|
||||
| `new_bootstraps_burst` | `u32` | `256` | Process-wide burst выдачи bootstrap. |
|
||||
| `new_sessions_per_minute` | `u32` | `600` | Устойчивая process-wide скорость создания сессий. |
|
||||
@@ -2560,21 +2574,31 @@ WEB-режим переносит MTProxy-трафик Telegram Desktop внут
|
||||
|
||||
# [web.timeouts]
|
||||
|
||||
Все таймауты задаются в секундах и должны входить в диапазон `1..=3600`. Самый длинный request deadline должен быть меньше `http_idle_secs`.
|
||||
Если в строке не указано иное, таймауты задаются в секундах и должны входить в диапазон `1..=3600`. Настроенные серверные deadlines отдельных HTTP-фаз должны быть меньше `http_idle_secs`; защищённые фазы сохраняют собственные deadlines, поэтому idle-таймер не является общим deadline запроса. Client-side окно повторов bridge имеет отдельные границы.
|
||||
|
||||
| Ключ | Тип | По умолчанию | Hot-Reload | Описание |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `header_secs` | `u64` | `10` | `✔` | Получение полного заголовка HTTP-запроса. |
|
||||
| `body_secs` | `u64` | `30` | `✔` | Сбор одного аутентифицированного carrier body. |
|
||||
| `stream_handshake_secs` | `u64` | `10` | `✔` | Выполнение внутреннего MTProxy handshake. |
|
||||
| `stream_first_byte_secs` | `u64` | `30` | `✔` | Получение первого внутреннего MTProxy-байта после `OPEN`; диапазон `1..=300`. |
|
||||
| `long_poll_secs` | `u64` | `25` | `✔` | Максимальная длительность пустого downlink long poll. |
|
||||
| `bridge_request_secs` | `u64` | `10` | `✔` | Deadline одной HTTP attempt в bridge до полного чтения response body; для `/down` дополнительно разрешён `long_poll_secs`. Диапазон `1..=60`. |
|
||||
| `bridge_retry_secs` | `u64` | `90` | `✔` | Абсолютное окно повторов bridge, включая attempts и backoff; диапазон `1..=300`, не меньше `bridge_request_secs`. |
|
||||
| `carrier_probe_coalesce_ms` | `u64` | `0` | `✔` | Опциональное ожидание bridge после `OPEN` для соответствующего `DATA`; миллисекунды в диапазоне `0..=10`, где `0` сохраняет немедленный probe. |
|
||||
| `lane_open_wait_secs` | `u64` | `2` | `✔` | Ожидание канонического downlink с cursor zero, опередившего свой lane `OPEN`; не больше `long_poll_secs`. |
|
||||
| `carrier_health_secs` | `u64` | `30` | `✔` | Интервал наблюдения после commit, необходимый для добавления carrier-learning evidence. |
|
||||
| `websocket_upgrade_secs` | `u64` | `5` | `✔` | Максимальное ожидание превращения принятого HTTP Upgrade в WebSocket; диапазон `1..=60`. |
|
||||
| `websocket_open_secs` | `u64` | `15` | `✔` | Абсолютный deadline первого carrier binary message после Upgrade; диапазон `1..=300`. |
|
||||
| `websocket_write_secs` | `u64` | `30` | `✔` | Максимальное ожидание одной WebSocket write или flush операции. |
|
||||
| `websocket_backpressure_secs` | `u64` | `30` | `✔` | Максимальное ожидание прогресса общего byte-budget или queue перед закрытием затронутого connection. |
|
||||
| `websocket_eviction_secs` | `u64` | `1` | `✔` | Grace period для освобождения slot и budget вытесненным WebSocket до отказа admission. |
|
||||
| `carrier_negotiation_deadlines_secs` | `[u64; 4]` | `[3, 5, 8, 12]` | `✔` | Строго возрастающие cumulative offsets: bridge применяет их перед первым запросом `/session`, сервер — при приёме первой automatic attempt. Checkpoints для одного—четырёх кандидатов: `[d3]`, `[d0, d3]`, `[d0, d1, d3]` и `[d0, d1, d2, d3]`; последний кандидат всегда использует `d3`. |
|
||||
| `carrier_learning_secs` | `u64` | `600` | `✔` | Фиксированный срок двух process-local окон evidence; диапазон `2..=86400`. |
|
||||
| `bootstrap_lifetime_secs` | `u64` | `120` | `✔` | Срок неиспользованного bootstrap и replay-marker закрытого token. |
|
||||
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Максимальная неактивность carrier до закрытия сессии. |
|
||||
| `http_idle_secs` | `u64` | `75` | `✔` | Idle lifetime WEB HTTP keep-alive connection. |
|
||||
| `shutdown_secs` | `u64` | `15` | `✔` | Deadline корректного завершения WEB. |
|
||||
| `http_idle_secs` | `u64` | `75` | `✔` | Лимит простоя между HTTP-обменами и при отсутствии прогресса уже выданного response body. Явно ограниченные фазы request body, long poll, decoy и ожидания Upgrade сохраняют собственные deadlines и не обрываются этим таймером. Значение фиксируется при приёме connection. |
|
||||
| `shutdown_secs` | `u64` | `15` | `✔` | Один абсолютный бюджет завершения процесса, общий для всех listener acceptors и connections, а также для WEB sessions и auxiliary tasks. Активное значение фиксируется один раз при начале shutdown. |
|
||||
| `decoy_header_secs` | `u64` | `30` | `✔` | Deadline подключения и получения response head от HTTP decoy. |
|
||||
|
||||
# [[web.vhosts]]
|
||||
@@ -2601,7 +2625,7 @@ Hostname нормализуется при валидации и должен п
|
||||
|
||||
| Ключ | Тип | Обязательный | По умолчанию | Описание |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `user` | `String` | да | — | Существующий ключ из `[access.users]`. |
|
||||
| `user` | `String` | да | — | Существующий ключ длиной 1–64 байта из `[access.users]`; ограничение сохраняет bounded runtime-status и фильтры. |
|
||||
| `secret_mode` | `"plain"` или `"dd"` | да | — | Точное представление секрета для Telegram Desktop. `ee` не поддерживается. |
|
||||
| `max_sessions` | `usize` | нет | `web.limits.max_sessions_global` | Активные сессии этого профиля. |
|
||||
| `max_streams` | `usize` | нет | `web.limits.max_streams_global` | Активные logical streams этого профиля. |
|
||||
@@ -2611,10 +2635,11 @@ Hostname нормализуется при валидации и должен п
|
||||
|
||||
## Lifecycle WEB и управление через API
|
||||
|
||||
- Config watcher и generation reload применяют `web.enabled`, `web.carrier`, `web.debug`, `web.timeouts`, vhosts, profiles и decoy snapshots без перезапуска процесса. Существующие сессии сохраняют carrier, лимиты и deadlines своего момента создания; новые bridge sessions используют активное поколение.
|
||||
- Config watcher и generation reload применяют `web.enabled`, policy carrier/negotiation, `web.debug`, `web.timeouts`, vhosts, profiles и decoy snapshots без перезапуска процесса. Валидируется и активируется один immutable expanded source snapshot; watcher candidate generation запускается только после активации этого поколения. Существующие сессии и начатые negotiation chains сохраняют issuance-time carrier candidates, limits, timeouts и абсолютные deadlines; новые bridge sessions используют одно зафиксированное активное поколение.
|
||||
- Состав WEB-listeners и их trust policy в `server.listeners`, а также все значения `web.limits` принадлежат процессу и требуют перезапуска.
|
||||
- Изменяемого ресурса `/v1/web` нет. `GET /web-status` предоставляет аутентифицированную read-only HTML-диагностику; `GET /v1/config` не возвращает `[web]`, а `PATCH /v1/config` отклоняет ключ `web` с `400 section_not_editable`.
|
||||
- Для удалённого применения WEB policy измените соответствующий TOML-файл и вызовите `POST /v1/system/reload`; проверьте `GET /v1/system/reload/{id}` и поле `deferred_process_fields`. Если оно содержит `server.listeners` или `web.limits`, перезапустите Telemt.
|
||||
- `GET /v1/config` возвращает полное авторское дерево `[web]`, кроме производного snapshot `web.runtime`. `PATCH /v1/config` принимает sparse object `web`, глубоко сливает tables, целиком заменяет arrays, валидирует полный candidate и указывает `web.limits` в `deferred_process_fields` до перезапуска.
|
||||
- `GET /v1/runtime/web/status`, `/sessions`, `/sessions/{session_ref}` и `/operations/{operation_id}` предоставляют bounded несекретное runtime-состояние. POST controls закрывают выбранные сессии, очищают debug или сбрасывают carrier learning и требуют текущий случайный `runtime_instance`.
|
||||
- `web.enabled = false` после активации прекращает новую выдачу bootstrap/session credentials, но не закрывает активные сессии. Для close-all дождитесь `manager.issuance_enabled = false`, отправьте асинхронный selector `all` и опрашивайте его operation.
|
||||
- Существующих access users можно создавать, изменять, ротировать, включать, выключать и удалять через `/v1/users`. Создание пользователя не добавляет WEB-профиль. Отключение пользователя немедленно обновляет admission и завершает его активные сессии.
|
||||
- `PATCH /v1/config` может сохранить `server.listeners`, включая поля WEB-listener’а, но изменённый WEB-listener активируется только после перезапуска процесса.
|
||||
|
||||
|
||||
+73
-12
@@ -28,7 +28,9 @@ Leiten Sie den vollständigen öffentlichen vhost an Telemt weiter. Wenn der TLS
|
||||
|
||||
- Der öffentliche Endpunkt ist immer `https://HOST:443`.
|
||||
- Unterstützt werden 16-Byte-MTProxy-Secrets in den Modi `plain` und `dd`. FakeTLS-Secrets mit `ee` werden im WEB-Modus nicht unterstützt.
|
||||
- `web.carrier = "https"` wählt serialisierte HTTPS-Uplinks und Long Polling. `https-lanes` wählt unabhängige HTTPS-Sequenzen und Polls pro logischem Stream. `websocket` wählt einen geordneten WebSocket für alle Streams. `websocket-lanes` wählt einen unabhängig verwalteten WebSocket für jeden logischen Stream ungleich null.
|
||||
- `web.carrier` wählt den einzigen Carrier bei deaktivierter Auto-Negotiation und den letzten Fallback bei aktivierter Negotiation. `https` verwendet serialisierte HTTPS-Uplinks und Long Polling. `https-lanes` verwendet unabhängige HTTPS-Sequenzen und Polls pro logischem Stream. `websocket` verwendet einen geordneten WebSocket für alle Streams. `websocket-lanes` verwendet einen unabhängig verwalteten WebSocket für jeden logischen Stream ungleich null.
|
||||
- Ein fehlendes `web.carriers` oder `web.carriers = false` deaktiviert Auto-Negotiation und Lernen. Ein nicht leeres Array aktiviert ausschließlich die sequenzielle Start-Negotiation; eine bereits festgeschriebene Sitzung wird nie migriert.
|
||||
- Native Clients ohne kanonische Carrier-Negotiation-Header verwenden den konfigurierten festen `carrier`, auch wenn `carriers` die Negotiation für fähige Clients aktiviert. Das aktuelle Telegram iOS unterstützt nur `https`; für metadatafreies iOS muss der Betreiber daher `web.carrier = "https"` setzen, `https-lanes` wird nicht unterstützt. User-Agent-Werte einschließlich CFNetwork oder Darwin leiten niemals Capabilities ab. Sendet ein nativer iOS-Client explizite Negotiation-Metadaten, schneidet Telemt sie mit der serverautoritativen Obergrenze `{https}` und lehnt ein leeres Ergebnis ab; andere explizite Clients verwenden ihren angegebenen Capability-Satz.
|
||||
- Capability-, Bootstrap- und Session-Zugangsdaten sind getrennte Werte mit begrenzter Lebensdauer. Carrier-Zugangsdaten sind geheim und dürfen nicht in Access-Logs erscheinen.
|
||||
- Ein Bootstrap ist ein Bearer-Token und nicht an eine Quelladresse gebunden. Client-Adresse und IP-Familie dürfen sich zwischen dem Laden der Bridge und der Sitzungserstellung ändern. Die Ausstellungsadresse bleibt dem Limit ungenutzter Bootstraps zugeordnet; die Adresse des ersten gültigen Erstellungs-Requests wird der Sitzung zugeordnet.
|
||||
- Die innere MTProxy-Authentifizierung ist auf den Benutzer und Secret-Modus des vhost-Profils beschränkt. Ein ungültiger innerer Handshake schließt nur seinen logischen Stream und gelangt niemals in den TCP-Masking-Pfad.
|
||||
@@ -91,17 +93,52 @@ max_streams = 512
|
||||
max_streams_per_session = 64
|
||||
```
|
||||
|
||||
## Serverseitige Carrier-Negotiation
|
||||
|
||||
Auto-Negotiation ist optional und bleibt deaktiviert, solange `carriers` nicht als explizites, nicht leeres Array gesetzt ist. Der konfigurierte `carrier` bleibt der letzte Fallback und wird genau einmal angehängt, auch wenn er bereits im Array steht:
|
||||
|
||||
```toml
|
||||
[web]
|
||||
enabled = true
|
||||
carrier = "https"
|
||||
carriers = ["websocket-lanes", "websocket", "https-lanes"]
|
||||
carrier_learning = true
|
||||
carrier_negotiation_aggressiveness = "conservative"
|
||||
|
||||
[web.timeouts]
|
||||
carrier_negotiation_deadlines_secs = [3, 5, 8, 12]
|
||||
carrier_health_secs = 30
|
||||
carrier_learning_secs = 600
|
||||
bridge_request_secs = 10
|
||||
bridge_retry_secs = 90
|
||||
carrier_probe_coalesce_ms = 0
|
||||
```
|
||||
|
||||
Die erzeugte Bridge sendet bei `/session` die kanonischen Header `X-Carrier-Capabilities`, `X-Carrier-Attempt` und ab dem zweiten Versuch `X-Carrier-Failure`. Jede erfolgreiche automatische Response liefert `X-Carrier-Mode`, `X-Carrier-Attempt`, `X-Carrier-Candidate-Count`, `X-Carrier-Deadline` und `X-Carrier-State`. Die Bridge startet ihre lokale kumulative Uhr unmittelbar vor dem ersten `/session`-Request; der Server friert seine separate absolute Chain-Deadline bei Annahme des ersten automatischen Versuchs ein. Beide verwenden die konfigurierten Offsets und werden bei Ersatzversuchen nicht zurückgesetzt. Für einen bis vier effektive Kandidaten lauten die Attempt-Checkpoints entsprechend `[d3]`, `[d0, d3]`, `[d0, d1, d3]` und `[d0, d1, d2, d3]`; der letzte Kandidat verwendet immer `d3`. Ein Nachfolger bleibt bis zu seinem eigenen Checkpoint zulässig. Die Zustände sind `provisional`, `committed` und `healthy`.
|
||||
|
||||
Versuche laufen streng sequenziell. Akzeptierter `OPEN`- oder `DATA`-Fortschritt schreibt den gewählten Carrier sofort fest und schließt die Ersatzgrenze endgültig. Ein authentifiziertes `409` für eine festgeschriebene Kette wiederholt deren Metadaten und ist terminal; es erlaubt keinen weiteren Versuch. Das exakte Replay von `/session` wird nur verwendet, solange dessen Ergebnis mehrdeutig ist. Nach der authentifizierten Auswahl eines provisional Carriers fordert ein Transportfehler direkt den nächsten Versuch an; wurde der vorherige Probe doch committed, antwortet der Server terminal mit `409`, statt einen unsicheren Ersatz zuzulassen. Die endgültige absolute Server-Deadline begrenzt auch einen Nachfolger, dessen Response den Client nie erreicht hat. Dynamisches Umschalten nach dem Commit wird absichtlich nicht unterstützt; dafür ist eine neue Sitzung erforderlich.
|
||||
|
||||
Jede HTTP-Operation der Bridge besitzt ein absolutes Budget `bridge_retry_secs` und höchstens neun Versuche. `bridge_request_secs` umfasst sowohl den Fetch-Response-Head als auch das vollständige Lesen des Response-Bodys; ein Downlink-Versuch erhält zusätzlich das konfigurierte Long-Poll-Intervall. Netzwerkfehler und Antworten mit `408`, `429`, `502`, `503` oder `504` verwenden begrenzten exponentiellen Backoff, während `Retry-After` das absolute Budget nicht verlängern kann. `carrier_probe_coalesce_ms = 0` sendet den ersten geordneten `OPEN`-Probe sofort. Ein Wert bis 10 ms kann passendes `DATA` aus diesem Fenster aufnehmen; multiplexierte Carrier bewahren die vollständige vorhergehende Frame-Reihenfolge, Lane-Carrier beanspruchen nur die ausgewählte Lane. Vor der Probe-Bestätigung startet kein HTTP-Downlink. Ein multiplexierter WebSocket-Upgrade kann unmittelbar nach seiner Auswahl durch `/session` beginnen und danach eingereihte Probe-Daten aufnehmen; ein Lane-WebSocket wartet auf die bekannte Stream-ID.
|
||||
|
||||
Automatische WebSockets verwenden `tproxy-auto-v1.<session-token>` beziehungsweise `tproxy-auto-lane-v1.<session-token>.<stream-id>`. Die erste akzeptierte Binärnachricht mit echtem `OPEN`- oder `DATA`-Fortschritt schreibt den Carrier fest; danach schreibt der Server eine leere binäre Commit-Bestätigung auf genau diese Verbindung. Ping/Pong schreibt keinen Carrier fest und zählt nicht als Learning-Evidenz.
|
||||
|
||||
Ein festgeschriebener Versuch wird erst healthy, wenn transportspezifische bidirektionale Evidenz für `carrier_health_secs` gültig bleibt. HTTPS erfordert akzeptiertes `DATA`, einen bestätigten nicht leeren Post-Commit-Downlink-Batch sowie authentifizierte Aktivität an oder nach der Health-Deadline. WebSocket erfordert die geschriebene exakte Commit-Bestätigung, danach akzeptiertes `OPEN` oder `DATA` desselben Owners und einen bis zum Ende des Intervalls lebenden Owner. Ein früheres Schließen ist neutral und erzeugt kein Lernergebnis.
|
||||
|
||||
Das Lernen ist prozesslokal, speicherresident, ausschließlich positiv und durch `max_carrier_learning_entries` begrenzt. Es sortiert nur vom Client unterstützte konfigurierte Kandidaten, hält den konfigurierten Fallback stets zuletzt und bewahrt bei gleichen Scores die Konfigurationsreihenfolge. User-Agent- und Profilevidenz haben Primärgewicht; eine zulässige IP dient nur als Tie-Breaker. IP-Evidenz erfordert genau eine explizite, global routbare `X-Forwarded-For`-Adresse; private, Loopback-, Link-Local-, Carrier-Grade-NAT-, Dokumentations-, Multicast- und entsprechende IPv4-Mapped-Adressen sind ausgeschlossen. Vom Client gemeldete Fehlerkategorien und Request-Latenz sind ausschließlich diagnostisch und erzeugen weder negative noch Ranking-Evidenz. `conservative` erfordert 3 User-Agent-Ergebnisse oder 8 Profilergebnisse aus 4 Kohorten und deaktiviert IP-Evidenz; `balanced` verwendet 2, 6 aus 3 Kohorten und 3 zulässige IP-Ergebnisse; `aggressive` verwendet 1, 4 aus 2 Kohorten und 1 IP-Ergebnis. Deaktiviertes Lernen oder eine geänderte Richtlinie verwirft beim Reload inkompatible Evidenz, ohne laufende Sitzungen zu verändern.
|
||||
|
||||
`https` bleibt der Default und behält das ursprüngliche serialisierte Verhalten bei. Bei `https-lanes` ist Lane null für Session-Steuerung reserviert, und jeder logische Stream ungleich null erhält eine eigene Lane. Jede Lane besitzt eigene Uplink-Sequenzen, Retry-Digests, Downlink-Cursor, nicht bestätigte Replay-Batches, Queues und einen Newest-Poll-Wins-Lebenszyklus. Ein langsamer Stream blockiert daher keinen anderen Stream auf der WEB-Protokollebene.
|
||||
|
||||
Damit entfällt die Serialisierung zwischen WEB-Streams auf Anwendungsebene. Öffentliches HTTP/2 läuft weiterhin über eine oder mehrere TCP-Verbindungen, sodass Paketverlust Head-of-Line-Blocking auf Transportebene verursachen kann; `https-lanes` ist kein HTTP/3- oder QUIC-Carrier.
|
||||
|
||||
Alle Lane-Queues bleiben innerhalb der vorhandenen Byte-/Item-Budgets pro Sitzung und Prozess. Die Bridge begrenzt jede Lane zusätzlich auf 8 MiB und 1024 eingereihte Elemente. Lane-Long-Polls dürfen höchstens die Hälfte von `web.limits.max_http_handlers` belegen, sodass Handler-Kapazität für Sitzungserstellung, Uplink, DELETE und andere Steuerarbeit verbleibt. `https-lanes` erfordert `max_http_handlers >= 2`.
|
||||
Alle Lane-Queues und residenten Response-Bodys bleiben innerhalb der vorhandenen Byte-/Item-Budgets pro Sitzung und Prozess. Telemt begrenzt jede Lane zusätzlich durch `pending_bytes_per_lane` und `pending_items_per_lane`; die erzeugte Bridge begrenzt ihre entsprechenden Queues auf 8 MiB und 1024 Elemente. Lane-Long-Polls dürfen höchstens die Hälfte von `web.limits.max_http_handlers` belegen, sodass Handler-Kapazität für Sitzungserstellung, Uplink, DELETE und andere Steuerarbeit verbleibt. `https` erfordert `max_http_handlers >= 2`, `https-lanes` erfordert `max_http_handlers >= 4`.
|
||||
|
||||
Die Pfade `/api/v1/up` und `/api/v1/down` ändern sich nicht. Bei `https-lanes` enthält jeder Request an diese Pfade genau einen kanonischen dezimalen `X-Lane-ID`-Header. Die Uplink-Sequenz beginnt pro Lane unabhängig bei `1`, der Downlink-Cursor bei `0`. Lane null akzeptiert nur Session-`PONG`; jeder Frame einer Lane ungleich null muss dieselbe Stream-ID tragen, und eine neue Lane muss mit `OPEN` beginnen. Nachdem eingereihte und nicht bestätigte Downlink-Daten einer geschlossenen Lane vollständig abgearbeitet sind, antwortet Telemt leer mit `X-Lane-Closed: 1`, und die Bridge beendet deren Polling. Wiederholungen bleiben byte-identisch und spielen die ursprüngliche Bestätigung oder den Downlink-Batch erneut aus.
|
||||
Die Pfade `/api/v1/up` und `/api/v1/down` ändern sich nicht. Bei `https-lanes` enthält jeder Request an diese Pfade genau einen kanonischen dezimalen `X-Lane-ID`-Header. Die Uplink-Sequenz beginnt pro Lane unabhängig bei `1`, der Downlink-Cursor bei `0`. Lane null akzeptiert nur Session-`PONG`; jeder Frame einer Lane ungleich null muss dieselbe Stream-ID tragen, und eine neue Lane muss mit `OPEN` beginnen. Ein kanonischer Cursor-null-Downlink, der kurz vor dem `OPEN` seiner Lane eintrifft, wartet bis zu `lane_open_wait_secs`, ohne Lane-Zustand anzulegen; Grenzen pro Sitzung und prozessweite Hilfs-Permits begrenzen diese Wartefälle. Nach Ablauf folgt eine leere `204`-Response, während eine fehlende Lane mit fortgeschrittenem Cursor weiterhin als Protokollfehler über den Decoy-Pfad behandelt wird. Nachdem eingereihte und nicht bestätigte Downlink-Daten einer geschlossenen Lane vollständig abgearbeitet sind, antwortet Telemt leer mit `X-Lane-Closed: 1`, und die Bridge beendet deren Polling. Wiederholungen bleiben byte-identisch und spielen die ursprüngliche Bestätigung oder den Downlink-Batch erneut aus.
|
||||
|
||||
Beide WebSocket-Carrier erstellen und löschen die übergeordnete Sitzung weiterhin über HTTPS und verwenden danach einen strikten Upgrade-Request ohne Body an `GET /api/v1/ws`. `websocket` übermittelt in `Sec-WebSocket-Protocol` exakt `tproxy-v1.<session-token>`; binäre Messages sind geordnete Carrier-Batches, und ein Protokoll-, Deadline- oder Verbindungsfehler schließt die gesamte übergeordnete Sitzung. `websocket-lanes` übermittelt exakt `tproxy-lane-v1.<session-token>.<stream-id>`, wobei die Stream-ID kanonisch dezimal im Bereich `1..=16777215` steht. Die erste binäre Message muss mit `OPEN` beginnen, alle Frames müssen diese Stream-ID verwenden und ein Fehler nach dem Upgrade schließt nur diese Lane. Es gibt keinen Lane-null-WebSocket: HTTPS transportiert `HELLO` und `WELCOME`, während RFC-6455-Ping/Pong die Verbindungsliveness gewährleistet.
|
||||
|
||||
WebSocket-Codec-Puffer und laufende Read-/Write-Messages teilen das prozesseigene Budget `pending_bytes_global` mit den Carrier-Queues und sind zusätzlich durch `websocket_bytes_global` begrenzt. Admission reserviert `websocket_http_connection_reserve` angenommene Verbindungen für gewöhnliches HTTP und Decoys. Unter Druck erfolgt die Verdrängung zuerst beim selben Owner und danach nach dem ältesten Fortschritt; Pre-Upgrade- und tote Verbindungen stehen vor aktiven Lanes und multiplexierten Sitzungen. Nach `long_poll_secs` ohne Peer-Aktivität wird auch bei kontinuierlichem Downlink-Verkehr ein Transport-Ping gesendet; fehlende Peer-Aktivität während des doppelten, beim Verbindungsaufbau festgelegten Intervalls macht die Verbindung zum Cleanup-Kandidaten.
|
||||
Vor HTTP `101` wird eine WebSocket-Lane-Reservierung an die exakte Prozessverbindung und Lane-Inkarnation gebunden; ein akzeptiertes `OPEN` überträgt die Ownership auf die exakte Stream-Inkarnation, bevor deren Backend-Task laufen kann. Ein verspäteter Poll, Close oder Reservierungs-Drop eines älteren Sockets kann einen Ersatz mit derselben numerischen Lane-ID weder bestätigen noch schließen oder freigeben.
|
||||
|
||||
WebSocket-Codec-Puffer und laufende Read-/Write-Messages teilen das prozesseigene Budget `pending_bytes_global` mit den Carrier-Queues und sind zusätzlich durch `websocket_bytes_global` begrenzt. Admission reserviert `websocket_http_connection_reserve` angenommene Verbindungen für gewöhnliches HTTP und Decoys. Bei einem Admission-Ersatz werden zuerst global tote aktive Verbindungen ausgewählt; danach gelten die Lokalitätsstufen gleiche Sitzung, gleicher Profil-Owner und gleiche Client-IP. Ein davon unabhängiges gesundes Opfer ist nur zulässig, wenn der Anforderer unter seinem fairen Byte-Anteil und der Owner des Opfers darüber liegt. Innerhalb einer Lokalitätsstufe stehen beanspruchte oder auf WebSocket hochgestufte Verbindungen vor aktiven Lanes und diese vor aktiven multiplexierten Sitzungen; letzter Fortschritt, Erstellungsreihenfolge und Verbindungs-ID lösen Gleichstände deterministisch auf. Das Cleanup bei Speicherdruck verwendet dieselbe Dead-first- und Lebenszyklusreihenfolge und bevorzugt Owner über ihrem fairen Anteil, setzt die Verdrängung aber auch fort, wenn alle Owner ihren Anteil einhalten. `max_websocket_evictions_in_flight` begrenzt gleichzeitige exakte Verdrängungs-Claims. Upgrade-, Erstnachrichten-, Write-, Backpressure- und Eviction-Deadlines stammen unveränderlich aus der Parent-Sitzung. Nach `long_poll_secs` ohne Peer-Aktivität wird auch bei kontinuierlichem Downlink-Verkehr ein Transport-Ping gesendet; fehlende Peer-Aktivität während des doppelten, beim Verbindungsaufbau festgelegten Intervalls macht eine aktive Verbindung zum Cleanup-Kandidaten.
|
||||
|
||||
Jeder Authentifizierungs-, Shape-, Lane-Reservierungs- oder Kapazitätsfehler vor dem Upgrade folgt dem bereinigten Decoy-Pfad und legt keinen WebSocket-spezifischen Status offen. Das exakte Subprotokoll enthält den Session-Bearer und darf nicht protokolliert werden.
|
||||
|
||||
@@ -198,22 +235,29 @@ Im Frontend oder im Abschnitt `defaults` muss für das standardmäßige WebSocke
|
||||
| --- | --- |
|
||||
| Bestand der WEB-Listener, Bind-Adresse und Vertrauensrichtlinie | Prozesseigen; Telemt neu starten. |
|
||||
| Jeder Wert in `[web.limits]` | Prozesseigener Speicher- und Ressourcenvertrag; Telemt neu starten. |
|
||||
| `web.enabled`, `web.carrier`, `web.debug`, Timeouts, vhosts, Profile und Decoys | Werden vom Config-Watcher oder durch einen Runtime-Generations-Reload angewendet. |
|
||||
| Bestehende HTTP-Verbindungen und WEB-Sitzungen | Behalten Carrier, Grenzen und Session-Deadlines ihres Erstellungszeitpunkts; neu ausgegebene Bridge-Sitzungen verwenden den aktiven Carrier. WebSocket-Write-, Backpressure- und Eviction-Vorgänge lesen die aktiven hot-reload-fähigen Deadlines. Neue logische Streams verwenden die aktive Relay-Generation. |
|
||||
| Beenden des Prozesses | Verwendet den zuletzt geladenen Wert von `web.timeouts.shutdown_secs`. |
|
||||
| `web.enabled`, Carrier-/Negotiation-Richtlinie, `web.debug`, Timeouts, vhosts, Profile und Decoys | Werden vom Config-Watcher oder durch einen Runtime-Generations-Reload angewendet. |
|
||||
| Bestehende HTTP-Verbindungen und WEB-Sitzungen | Behalten HTTP-Idle-Grenze, Carrier-Kandidaten, Grenzen, Body-Timeout, Lebensdauer des Replay-Markers geschlossener Token sowie absolute Session-/Negotiation-Deadlines ihres Erstellungszeitpunkts; jede ausgegebene Bridge enthält ihre Request-, Retry- und Probe-Coalescing-Werte. WebSocket-Upgrade-, Open-, Write-, Backpressure- und Eviction-Vorgänge verwenden die unveränderlichen Deadlines der Parent-Sitzung. Neue Bridges verwenden die aktive Richtlinie, neue logische Streams die aktive Relay-Generation. |
|
||||
| Beenden des Prozesses | Erfasst den zuletzt geladenen Wert von `web.timeouts.shutdown_secs` einmalig und verwendet dieselbe absolute Deadline für Listener-Acceptoren und Verbindungen sowie WEB-Sitzungen und Hilfstasks. Aufeinanderfolgende Komponenten erhalten keine separaten vollständigen Budgets. |
|
||||
|
||||
Jeder logische Stream behält die Client-IP seiner Sitzung und besitzt während der gesamten Relay-Lebensdauer einen prozessweit eindeutigen, von null verschiedenen synthetischen Quellport. Damit bleibt für Direct- und Middle-End-KDF-Routing ein stabiles, kollisionsfreies Quell-/Ziel-Tupel erhalten.
|
||||
|
||||
Die HTTP-Idle-Erfassung schützt nur explizit begrenzte Request-Body-, Long-Poll-, Decoy-Verbindungs-/Response-Head- und ausstehende Upgrade-Phasen. Die eigene Deadline der Operation bleibt exakt; besteht ihre Lease in diesem Moment noch, gewährt der Verbindungs-Watchdog dem eingeplanten Task höchstens ein Connection-Idle-Intervall zur Veröffentlichung seines Timeouts/Ergebnisses, bevor er die Verbindung erzwingend schließt. Zwischen Austauschvorgängen und nach Bereitstellung eines Response-Heads setzt Fortschritt den Idle-Timer zurück, während ein blockierter Response-Body weiterhin durch den Idle-Timeout begrenzt bleibt. Der Abschluss einer älteren Phase kann den Deadline-Schutz einer neueren Phase nicht freigeben.
|
||||
|
||||
Ein `OPEN` reserviert die begrenzte Eigentümerschaft für logischen Stream und Tupel, verbraucht jedoch noch kein `max_connections`-Permit der Relay-Generation. Telemt erwirbt dieses Permit erst nach dem ersten inneren Byte; die unveränderliche First-Byte-Deadline und Stream-Grenzen begrenzen stille Opens, und erschöpfte Kapazität schließt anschließend nur den betroffenen Stream.
|
||||
|
||||
## Verwaltung über die API
|
||||
|
||||
API-Verwaltung ist verfügbar, aber absichtlich eingeschränkt. Es gibt keine veränderbare Ressource `/v1/web`; der API-Listener stellt die schreibgeschützte HTML-Debug-Ansicht unter `/web-status` bereit.
|
||||
WEB-Konfiguration, Runtime-Status und begrenzte Runtime-Steuerung verwenden denselben authentifizierten API-Listener. `/web-status` bleibt eine schreibgeschützte HTML-Diagnose; zustandsverändernde Operationen existieren ausschließlich unter `/v1/runtime/web`.
|
||||
|
||||
| Operation | API-Unterstützung |
|
||||
| --- | --- |
|
||||
| `[web]`, vhosts, Profile, Decoys, Timeouts oder Limits lesen oder ändern | Nein. `GET /v1/config` lässt `[web]` aus; `PATCH /v1/config` antwortet für `web` mit `400 section_not_editable`. |
|
||||
| `[web]`, vhosts, Profile, Decoys, Timeouts oder Limits lesen oder ändern | Ja, über `GET` oder `PATCH /v1/config`. Der abgeleitete Snapshot `web.runtime` wird weder ausgegeben noch kann er geschrieben werden. Verschachtelte Tabellen werden feldweise zusammengeführt; Arrays ersetzen das bisherige Array vollständig. Jede Änderung an `[web.limits]` wird als gewünschte Konfiguration angenommen, aber bis zum Prozessneustart als zurückgestellt gemeldet. |
|
||||
| `server.listeners` speichern | Ja, über `PATCH /v1/config`; ein geänderter WEB-Listener bleibt jedoch bis zum Prozessneustart zurückgestellt. |
|
||||
| Außerhalb der API geänderte WEB-Konfiguration anwenden | Ja, über `POST /v1/system/reload` und anschließende Abfrage des Vorgangsstatus. |
|
||||
| Begrenzte serverseitige WEB-Request- und Lifecycle-Details untersuchen | Ja, über ein authentifiziertes `GET /web-status`. |
|
||||
| Lifecycle, Kapazitätsebenen, Learning-/Debug-Zustand und aktive Sitzungen untersuchen | Ja, über `GET /v1/runtime/web/status` und `/v1/runtime/web/sessions`. |
|
||||
| Ausgewählte aktive WEB-Sitzungen schließen | Ja, über die asynchrone Operation `POST /v1/runtime/web/sessions/close`. |
|
||||
| Debug-Datensätze löschen oder Carrier-Learning zurücksetzen | Ja, über die entsprechenden Runtime-POST-Endpunkte. |
|
||||
| `[access.users]` verwalten | Ja, über `/v1/users`. Das Erstellen eines Benutzers erzeugt kein WEB-Profil. |
|
||||
| Einen Benutzer widerrufen | Ja. `/v1/users/{username}/disable` aktualisiert die Admission sofort und beendet die aktiven Sitzungen dieses Benutzers. |
|
||||
|
||||
@@ -230,6 +274,20 @@ read_only = false
|
||||
|
||||
Die API-Whitelist prüft den direkten TCP-Peer und vertraut `X-Forwarded-For` nicht. Änderungen an `[server.api]` selbst erfordern einen Prozessneustart.
|
||||
|
||||
### Runtime-Status und Steuerung
|
||||
|
||||
`GET /v1/runtime/web/status` liefert immer den veröffentlichten Lifecycle (`starting`, `no_web_listener`, `running`, `draining`, `drained` oder `deadline_exceeded`), dessen Epoche und Alter, die effektiven Listener-Adressen und die Verfügbarkeit. Solange die prozesseigene WEB-Runtime lebt, ergänzt `runtime` die zufällige 128-Bit-`runtime_instance`, die aktive Generation, unveränderliche Limits, ebenenlokale Kapazitätszähler, Carrier-Learning-/Debug-Epochen und Summen. Die Statuserfassung liest jede Ebene nicht blockierend: Eine umkämpfte Ebene wird ausgelassen und in `partial` benannt; der Endpunkt wartet nie auf die Datenebene, bereinigt sie nicht und verändert sie nicht.
|
||||
|
||||
`GET /v1/runtime/web/sessions` liefert standardmäßig höchstens 50 und bei gesetztem `limit` höchstens 200 Sitzungen. Der geordnete Scan ist auf 1000 Kandidaten begrenzt. `cursor` und `session_ref` verwenden die undurchsichtige kanonische Form `ws1.<runtime-instance>.<lowercase-hex-id>`; ein exakter `session_ref` darf nicht mit `cursor` oder `limit` kombiniert werden. Filter sind `ip`, `host`, `user`, `user_agent_id`, `key_id`, `carrier` und `state`; doppelte oder unbekannte Query-Felder werden abgelehnt. Der Detailpfad lautet `GET /v1/runtime/web/sessions/{session_ref}`. Ein gespeicherter Tombstone einer geschlossenen Sitzung ergibt `410`; ein umkämpfter exakter Snapshot ergibt `503 web_snapshot_busy`. Antworten enthalten nur begrenzte, nicht geheime Metadaten und niemals Bootstrap-/Session-Bearer, Capabilities, Secret-Hashes oder synthetische KDF-Ports.
|
||||
|
||||
Jeder Runtime-POST verlangt exakt `Content-Type: application/json`, lehnt unbekannte JSON-Felder ab, beachtet API-Authentifizierung, Whitelist und `read_only` und enthält die aktuelle `runtime_instance` als ABA-Sperre. Verfügbare Steuerungen:
|
||||
|
||||
- `POST /v1/runtime/web/sessions/close` mit genau einem Selektor: `{"kind":"refs","session_refs":[...]}`, `{"kind":"filter",...}` oder `{"kind":"all"}`. Exakte Referenzen sind auf 200 begrenzt, ein Filter darf nicht leer sein, nur eine Close-Operation darf laufen, und `all` wird abgelehnt, solange die effektive Ausgabe aktiviert ist. Die `202`-Antwort liefert `operation_id`; fragen Sie `GET /v1/runtime/web/operations/{operation_id}` ab. Die Operation scannt in Blöcken von 128 nur Sitzungen bis einschließlich ihres beim Start fixierten High-Water-Marks.
|
||||
- `POST /v1/runtime/web/debug/clear` mit `{"runtime_instance":"..."}`. Die Antwort meldet gelöschte Datensätze, weiterhin von bereits gerenderten Snapshots gehaltene Bytes und die neue Epoche. Laufende Writer der alten Epoche können den Ring nicht erneut füllen.
|
||||
- `POST /v1/runtime/web/carrier-learning/reset` mit derselben Body-Form. Der Endpunkt löscht gespeicherte prozesslokale Evidenz und erhöht die Learning-Epoche; bereits fixierte Versuchsketten und aktive Sitzungen bleiben unverändert.
|
||||
|
||||
Für ein deterministisches Close-all patchen Sie `{"web":{"enabled":false}}` mit aktiviertem Runtime-Reload, warten auf `runtime.manager.issuance_enabled = false`, senden den Selektor `all` mit derselben `runtime_instance` und fragen die Operation bis zu einem Endzustand ab. Das Deaktivieren von WEB stoppt neue Bootstrap-/Session-Ausgabe, schließt bestehende Sitzungen aber niemals implizit.
|
||||
|
||||
### Serverseitige WEB-Debug-Ansicht
|
||||
|
||||
Aktivieren Sie die begrenzte Erfassung in der zuständigen Konfigurationsdatei:
|
||||
@@ -248,7 +306,7 @@ default_window_secs = 180
|
||||
max_window_secs = 3600
|
||||
```
|
||||
|
||||
Öffnen Sie `http://127.0.0.1:9091/web-status` mit derselben Whitelist direkter Peers und demselben exakten `Authorization`-Header wie für die API. Ein abschließender Slash wird akzeptiert. Nur `GET` ist zulässig. Die Seite unterstützt die Filter `window_secs`, kanonische `ip`, numerische `session`, `user_agent` ohne Beachtung der Groß-/Kleinschreibung und `key`. Wiederholen Sie `group_by=ip`, `group_by=session`, `group_by=user_agent` oder `group_by=key`, um gruppierte Zusammenfassungen zu erstellen; `limit` ist auf `1..=1000` beschränkt. HTTP-Zeilen lassen sich vom Request bis zur Response zu Methode, Pfad, bereinigten Headern, Body-Metadaten oder -Bytes, Zeitpunkten, Frames und typisierten Lifecycle-Ereignissen aufklappen. Für WebSocket kommen der bereinigte Handshake `GET` → `101` sowie begrenzte Angaben pro Message zu Richtung, Message-Typ, Payload-/Body-Erfassung, Verarbeitungszeit, Verbindungs-/Lane-ID und geparsten inneren Frames hinzu. Rohe Subprotokolle und Session-Tokens werden nie gespeichert.
|
||||
Öffnen Sie `http://127.0.0.1:9091/web-status` mit derselben Whitelist direkter Peers und demselben exakten `Authorization`-Header wie für die API. Ein abschließender Slash wird akzeptiert. Nur `GET` ist zulässig. Die Seite unterstützt die Filter `window_secs`, kanonische `ip`, numerische `session`, `user_agent` ohne Beachtung der Groß-/Kleinschreibung und `key`. Wiederholen Sie `group_by=ip`, `group_by=session`, `group_by=user_agent` oder `group_by=key`, um gruppierte Zusammenfassungen zu erstellen; `limit` ist auf `1..=1000` beschränkt. HTTP-Zeilen lassen sich vom Request bis zur Response zu Methode, Pfad, bereinigten Headern, Body-Metadaten oder -Bytes, Zeitpunkten, Frames und typisierten Lifecycle-Ereignissen einschließlich Carrier-Versuch, Commit, Healthy und gemeldetem Fehler aufklappen. Für WebSocket kommen der bereinigte Handshake `GET` → `101` sowie begrenzte Angaben pro Message zu Richtung, Message-Typ, Payload-/Body-Erfassung, Verarbeitungszeit, Verbindungs-/Lane-ID und geparsten inneren Frames hinzu. Rohe Subprotokolle und Session-Tokens werden nie gespeichert.
|
||||
|
||||
Der prozesseigene Ring übersteht den Austausch einer Runtime-Generation. Änderungen der Erfassungs-Policy löschen inkompatible gespeicherte Datensätze; reine Änderungen des Beobachtungsfensters tun dies nicht. Der Ring ist standardmäßig auf 65536 Datensätze und 64 MiB gespeicherte plus in Verarbeitung befindliche Daten begrenzt, die HTML-Response auf 8 MiB und die Gruppierung auf 1024 Gruppen; gleichzeitig dürfen höchstens zwei Response-Bodys Seiten-Permits halten. Ändern Sie `web.limits.debug_records_capacity` oder `web.limits.debug_bytes_global` nur zusammen mit einem Prozessneustart. Ein hot-reload-fähiger Präfix, der nur in eine gleichzeitig erhöhte neustartpflichtige Kapazität passt, wird bis zu diesem Neustart zurückgestellt.
|
||||
|
||||
@@ -267,7 +325,7 @@ curl -sS http://127.0.0.1:9091/v1/system/reload/RELOAD_ID \
|
||||
-H "Authorization: ${TELEMT_API_AUTH}"
|
||||
```
|
||||
|
||||
Der terminale Status `succeeded` bestätigt die Runtime-Aktivierung. Ein geänderter `web.carrier` wird von neu ausgegebenen Bridge-Sitzungen verwendet; bestehende Sitzungen werden nicht migriert. Enthält `deferred_process_fields` den Wert `server.listeners` oder `web.limits`, ist die Datei gültig und gespeichert, diese Einstellungen erfordern aber weiterhin einen Telemt-Neustart.
|
||||
Der terminale Status `succeeded` bestätigt die Runtime-Aktivierung. Geänderte Carrier-, Kandidaten-, Deadline- oder Learning-Richtlinien werden von neu ausgegebenen Bridge-Sitzungen verwendet; bestehende Sitzungen und laufende Versuchsketten werden nicht migriert. Enthält `deferred_process_fields` den Wert `server.listeners` oder `web.limits`, ist die Datei gültig und gespeichert, diese Einstellungen erfordern aber weiterhin einen Telemt-Neustart.
|
||||
|
||||
Operationen für Access-Benutzer verwenden die vorhandenen Endpunkte, zum Beispiel:
|
||||
|
||||
@@ -291,7 +349,7 @@ Der vollständige Vertrag für Requests, Revisionen, Fehler und alle Benutzer-En
|
||||
- Deaktivieren Sie am TLS-Terminator die Protokollierung von Request-Target und Authorization oder verwenden Sie ein geprüftes, redigiertes Format. Raw Queries enthalten Bridge-Capabilities und `Authorization` enthält Bootstrap- oder Session-Bearer-Zugangsdaten.
|
||||
- Verwenden Sie pro vhost eine stabile öffentliche Adresse. Wenn DNS mehrere Ingress-Adressen liefert, muss jede Bereitstellung die Adresse ihres externen Pfads verwenden.
|
||||
- Bootstrap- und Session-Register sind prozesslokal. Ein Multi-Prozess- oder Multi-Host-Upstream-Pool benötigt Affinität für den vollständigen vhost: Bridge-GET, Sitzungserstellung, Uplink, Downlink und DELETE. Ein einzelner Telemt-Prozess benötigt keine zusätzliche Affinität.
|
||||
- Ein ungenutzter Bootstrap übersteht einen Konfigurations-Reload nur, wenn die exakte Profilidentität aktiv bleibt: Host, `public_addr`, Benutzer, Secret-Modus, Carrier und Capability. Bereits erstellte Sitzungen behalten ihren unveränderlichen Carrier und ihre Profilidentität und bleiben lifecycle-bounded.
|
||||
- Ein ungenutzter Bootstrap übersteht einen Konfigurations-Reload nur, wenn die exakte Profilidentität aktiv bleibt: Host, `public_addr`, Benutzer, Secret-Modus, Carrier-Kandidaten, Negotiation-Deadlines und Capability. Bereits erstellte Sitzungen behalten ihren unveränderlichen Carrier und ihre Profilidentität und bleiben lifecycle-bounded.
|
||||
- Der Decoy gehört zum Anti-Probing-Vertrag. Prüfen Sie sein gewöhnliches 404-Verhalten und die Antwortzeiten über den öffentlichen TLS-Endpunkt, bevor Sie Links verteilen.
|
||||
|
||||
## Erstprüfung
|
||||
@@ -304,6 +362,7 @@ Der vollständige Vertrag für Requests, Revisionen, Fehler und alle Benutzer-En
|
||||
6. Bestätigen Sie für `websocket` eine `101`-Response, binären Relay-Datenverkehr und RFC-6455-Ping/Pong nach 25 Sekunden. Testen Sie für `websocket-lanes` mindestens zwei gleichzeitige Stream-Sockets und prüfen Sie, dass das Schließen oder Beschädigen einer Lane weder Geschwister noch die übergeordnete Sitzung schließt.
|
||||
7. Testen Sie einen Reconnect und mindestens einen Long Poll über 25 Sekunden, um sicherzustellen, dass Frontend-Timeouts den Carrier nicht abbrechen.
|
||||
8. Prüfen Sie Benutzer- und logische MTProxy-Verbindungslimits anhand der Logical-Stream-Zähler und nicht anhand der Zahl der HTTP-Verbindungen.
|
||||
9. Prüfen Sie bei aktivierter Auto-Negotiation die konfigurierte Reihenfolge, das Replay exakt desselben Versuchs nach einer absichtlich verlorenen Response, das terminale Verhalten nach dem Commit sowie die Lifecycle-Zeilen `carrier_committed` und `carrier_healthy` in `/web-status`. Prüfen Sie, dass ein nativer Client ohne Metadaten den festen `carrier` ohne automatische Response-Header verwendet und explizite Capabilities unverändert bleiben.
|
||||
|
||||
## Fehlerbehebung
|
||||
|
||||
@@ -311,6 +370,8 @@ Der vollständige Vertrag für Requests, Revisionen, Fehler und alle Benutzer-En
|
||||
| --- | --- |
|
||||
| WEB-Konfiguration ist auf dem Datenträger gültig, aber das Listener-Verhalten hat sich nicht geändert | Prüfen Sie `deferred_process_fields`; Listener- und `[web.limits]`-Änderungen erfordern einen Neustart. |
|
||||
| Carrier-Requests erreichen den Decoy | Prüfen Sie den exakten vhost, den Secret-Modus des Links, das CIDR des direkten Proxys und genau einen syntaktisch gültigen `X-Forwarded-For`-Wert. |
|
||||
| Ein konkurrierender `https-lanes`-Downlink erreicht den Decoy mit `404` | Prüfen Sie, dass er mit `X-Down-Cursor: 0` beginnt, bewahren Sie `X-Lane-ID` und setzen Sie `lane_open_wait_secs` über den beobachteten Abstand zwischen Downlink und `OPEN`. Fortgeschrittene Cursor fehlender Lanes schlagen absichtlich fail-closed fehl. |
|
||||
| Auto-Negotiation wechselt weiter, nachdem Daten bereits akzeptiert wurden | Das ist ungültig. Prüfen Sie das authentifizierte `X-Carrier-State`-Replay und das Carrier-Commit-Lifecycle-Ereignis; `committed` oder `healthy` ist terminal und erfordert eine neue Sitzung. |
|
||||
| Long Polls werden nach einem festen Intervall getrennt | Setzen Sie Client-, Server-, Sende- und Lese-Timeouts von NGINX/HAProxy über `web.timeouts.long_poll_secs`. |
|
||||
| WebSocket-Upgrade erreicht statt `101` den Decoy | Bewahren Sie HTTP/1.1 `Connection: Upgrade`, `Upgrade: websocket`, das einzelne exakte `Sec-WebSocket-Protocol` und den kanonischen bodylosen Request `/api/v1/ws`. Prüfen Sie außerdem Carrier-/Session-Kompatibilität und die Prozess-Verbindungsreserve. |
|
||||
| Ein `websocket-lanes`-Stream wurde geschlossen, Geschwister bleiben aber verbunden | Dies ist die beabsichtigte Fehlergrenze. Prüfen Sie die Message-/Frame-Zeilen dieser Lane in `/web-status`; fehlerhafte oder lane-fremde Frames, Write-Timeouts und Backend-Close schließen nur die betroffene Lane. |
|
||||
|
||||
+73
-12
@@ -28,7 +28,9 @@ Route the complete public vhost to Telemt. Splitting only recognized carrier pat
|
||||
|
||||
- The public endpoint is always `https://HOST:443`.
|
||||
- `plain` and `dd` 16-byte MTProxy secrets are supported. `ee` FakeTLS secrets are not supported by WEB mode.
|
||||
- `web.carrier = "https"` selects serialized HTTPS uplink and long polling. `https-lanes` selects independent HTTPS sequencing and polling per logical stream. `websocket` selects one ordered WebSocket for all streams. `websocket-lanes` selects one independently owned WebSocket per non-zero logical stream.
|
||||
- `web.carrier` selects the sole carrier when auto-negotiation is disabled and the final fallback when it is enabled. `https` uses serialized HTTPS uplink and long polling. `https-lanes` uses independent HTTPS sequencing and polling per logical stream. `websocket` uses one ordered WebSocket for all streams. `websocket-lanes` uses one independently owned WebSocket per non-zero logical stream.
|
||||
- Missing `web.carriers` or `web.carriers = false` disables auto-negotiation and learning. A non-empty array enables startup-only sequential negotiation; it never migrates an already committed session.
|
||||
- Native clients without canonical carrier-negotiation headers use the configured fixed `carrier`, even when `carriers` enables negotiation for capable clients. Current Telegram iOS supports only `https`, so an operator serving metadata-free iOS clients must set `web.carrier = "https"`; it does not support `https-lanes`. User-Agent values, including CFNetwork or Darwin, never infer capabilities. When a native iOS request does send explicit negotiation metadata, Telemt intersects it with the server-authoritative `{https}` ceiling and rejects an empty result; other explicit clients use their advertised capability set.
|
||||
- Capability, bootstrap, and session credentials are separate bounded-lifetime values. Carrier credentials must be treated as secrets and must not appear in access logs.
|
||||
- A bootstrap is a bearer credential, not a source-address-bound token. The client address and IP family may change between bridge loading and session creation. The issuing address retains unused-bootstrap accounting, while the address on the first valid creation request owns the session.
|
||||
- Inner MTProxy authentication is restricted to the user and secret mode selected by the vhost profile. Invalid inner handshakes close only their logical stream and never enter the TCP masking path.
|
||||
@@ -91,17 +93,52 @@ max_streams = 512
|
||||
max_streams_per_session = 64
|
||||
```
|
||||
|
||||
## Server-side carrier negotiation
|
||||
|
||||
Auto-negotiation is optional and disabled unless `carriers` is an explicit non-empty array. The configured `carrier` remains the final fallback and is appended exactly once, even when it also appears in the array:
|
||||
|
||||
```toml
|
||||
[web]
|
||||
enabled = true
|
||||
carrier = "https"
|
||||
carriers = ["websocket-lanes", "websocket", "https-lanes"]
|
||||
carrier_learning = true
|
||||
carrier_negotiation_aggressiveness = "conservative"
|
||||
|
||||
[web.timeouts]
|
||||
carrier_negotiation_deadlines_secs = [3, 5, 8, 12]
|
||||
carrier_health_secs = 30
|
||||
carrier_learning_secs = 600
|
||||
bridge_request_secs = 10
|
||||
bridge_retry_secs = 90
|
||||
carrier_probe_coalesce_ms = 0
|
||||
```
|
||||
|
||||
The generated bridge sends canonical `X-Carrier-Capabilities`, `X-Carrier-Attempt`, and, after the first attempt, `X-Carrier-Failure` headers on `/session`. Every successful automatic response returns `X-Carrier-Mode`, `X-Carrier-Attempt`, `X-Carrier-Candidate-Count`, `X-Carrier-Deadline`, and `X-Carrier-State`. The bridge starts its local cumulative clock immediately before the first `/session` request; the server freezes its separate absolute chain deadline when it accepts the first automatic attempt. Both use the configured offsets, and neither resets across replacement attempts. For one through four effective candidates, the attempt checkpoints are respectively `[d3]`, `[d0, d3]`, `[d0, d1, d3]`, and `[d0, d1, d2, d3]`; the final candidate always owns `d3`. A successor remains admissible until its own checkpoint. The states are `provisional`, `committed`, and `healthy`.
|
||||
|
||||
Attempts are strictly sequential. Accepted `OPEN` or `DATA` progress commits the chosen carrier immediately and permanently closes the replacement boundary. A `409` for an authenticated committed chain echoes the committed metadata and is terminal; it is not permission to advance. Exact `/session` replay is used only while that response is ambiguous. Once an authenticated response has selected a provisional carrier, a transport failure requests the next attempt directly; if the previous probe actually committed, the server answers with the terminal `409` instead of permitting an unsafe replacement. The server's final absolute deadline also bounds a successor response that the client never received. Post-commit dynamic switching is deliberately unsupported: reconnect with a new session instead.
|
||||
|
||||
Each bridge HTTP operation has an absolute `bridge_retry_secs` budget and at most nine attempts. `bridge_request_secs` covers both the Fetch response head and complete response body; a downlink attempt additionally receives the configured long-poll interval. Network failures and `408`, `429`, `502`, `503`, or `504` responses use bounded exponential backoff, while `Retry-After` cannot extend the absolute budget. `carrier_probe_coalesce_ms = 0` sends the first ordered `OPEN` probe immediately. A value up to 10 ms may include matching `DATA` that arrives in that window; multiplexed carriers preserve the complete preceding frame order, while lane carriers claim only the selected lane. No HTTP downlink starts before the probe acknowledgement. Multiplexed WebSocket Upgrade may begin as soon as `/session` selects it and then absorbs queued probe data; a lane WebSocket waits until its stream ID is known.
|
||||
|
||||
Automatic WebSockets use `tproxy-auto-v1.<session-token>` or `tproxy-auto-lane-v1.<session-token>.<stream-id>`. The first accepted binary message containing real `OPEN` or `DATA` progress commits the carrier; the server then writes an empty binary commit acknowledgement to that exact connection. Ping/Pong does not commit a carrier and does not count as learning evidence.
|
||||
|
||||
A committed attempt becomes healthy only after transport-specific bidirectional evidence remains valid for `carrier_health_secs`. HTTPS requires accepted `DATA`, an acknowledged non-empty post-commit downlink batch, and authenticated activity at or after the health deadline. WebSocket requires the exact commit acknowledgement to be written, subsequent accepted `OPEN` or `DATA` from the same owner, and that owner to remain live through the interval. Closing earlier is neutral and records no learning result.
|
||||
|
||||
Learning is process-local, in-memory, positive-only, and bounded by `max_carrier_learning_entries`. It ranks only client-supported configured candidates, keeps the configured fallback last, and uses configured order for equal scores. User-Agent and profile evidence have primary weight; an eligible IP is only a tie-breaker. IP evidence requires exactly one explicit, globally routable `X-Forwarded-For` address; private, loopback, link-local, carrier-grade NAT, documentation, multicast, and IPv4-mapped equivalents are excluded. Client-reported failure categories and request latency are diagnostics, not negative or ranking evidence. `conservative` requires 3 User-Agent outcomes or 8 profile outcomes across 4 cohorts and disables IP evidence; `balanced` uses 2, 6 across 3, and 3 eligible-IP outcomes; `aggressive` uses 1, 4 across 2, and 1 eligible-IP outcome. Disabling learning or changing its policy on reload clears incompatible evidence without changing in-flight sessions.
|
||||
|
||||
`https` remains the default and preserves the original serialized behavior. `https-lanes` assigns lane zero to session control and one lane to every non-zero logical stream. Each lane has its own uplink sequence, retry digest, downlink cursor, unacknowledged replay batch, queue, and newest-poll-wins lifecycle. A slow stream therefore does not block another stream at the WEB protocol layer.
|
||||
|
||||
This removes application-level serialization between WEB streams. Public HTTP/2 still runs over one or more TCP connections, so packet loss can cause transport-level head-of-line blocking; `https-lanes` is not an HTTP/3 or QUIC carrier.
|
||||
|
||||
All lane queues remain inside the existing per-session and process-wide byte/item budgets. The bridge also limits each lane to 8 MiB and 1024 queued items. Telemt permits lane long polls to occupy at most half of `web.limits.max_http_handlers`, preserving handler capacity for session creation, uplink, DELETE, and other control work. `https-lanes` requires `max_http_handlers >= 2`.
|
||||
All lane queues and resident response bodies remain inside the existing per-session and process-wide byte/item budgets. Telemt additionally limits each lane to `pending_bytes_per_lane` and `pending_items_per_lane`; the generated bridge caps its corresponding queues at 8 MiB and 1024 items. Telemt permits lane long polls to occupy at most half of `web.limits.max_http_handlers`, preserving handler capacity for session creation, uplink, DELETE, and other control work. `https` requires `max_http_handlers >= 2`, and `https-lanes` requires `max_http_handlers >= 4`.
|
||||
|
||||
The `/api/v1/up` and `/api/v1/down` paths do not change. In `https-lanes`, every request on those paths carries one canonical decimal `X-Lane-ID`. Uplink sequence starts at `1` and downlink cursor at `0` independently for each lane. Lane zero accepts only session `PONG`; every frame in a non-zero lane must have the same stream ID, and a new lane must begin with `OPEN`. After a closed lane's queued and unacknowledged downlink data is drained, Telemt returns an empty response with `X-Lane-Closed: 1`, and the bridge stops polling it. Retries remain byte-identical and replay the original acknowledgement or downlink batch.
|
||||
The `/api/v1/up` and `/api/v1/down` paths do not change. In `https-lanes`, every request on those paths carries one canonical decimal `X-Lane-ID`. Uplink sequence starts at `1` and downlink cursor at `0` independently for each lane. Lane zero accepts only session `PONG`; every frame in a non-zero lane must have the same stream ID, and a new lane must begin with `OPEN`. A canonical cursor-zero downlink that reaches Telemt just before its lane `OPEN` waits up to `lane_open_wait_secs` without creating lane state; per-session and process auxiliary permits bound these waits. Expiry returns an empty `204`, while a missing lane with an advanced cursor remains a protocol failure routed through the decoy. After a closed lane's queued and unacknowledged downlink data is drained, Telemt returns an empty response with `X-Lane-Closed: 1`, and the bridge stops polling it. Retries remain byte-identical and replay the original acknowledgement or downlink batch.
|
||||
|
||||
Both WebSocket carriers still create and delete the parent session over HTTPS. They then use a strict bodyless `GET /api/v1/ws` Upgrade request. `websocket` offers exactly `tproxy-v1.<session-token>` in `Sec-WebSocket-Protocol`; binary messages are ordered carrier batches, and a protocol, deadline, or connection failure closes the complete parent session. `websocket-lanes` offers exactly `tproxy-lane-v1.<session-token>.<stream-id>`, where the stream ID is canonical decimal in `1..=16777215`. Its first binary message must begin with `OPEN`, every frame must use that stream ID, and failure after upgrade closes only that lane. There is no lane-zero WebSocket: HTTPS carries `HELLO` and `WELCOME`, while RFC 6455 Ping/Pong supplies connection liveness.
|
||||
|
||||
WebSocket codec buffers and in-flight read/write messages share the process-owned `pending_bytes_global` budget with carrier queues and are additionally bounded by `websocket_bytes_global`. Admission leaves `websocket_http_connection_reserve` accepted connections for ordinary HTTP and decoys. Under pressure, replacement is owner-first, then least-recently-progressed with pre-Upgrade and dead connections ahead of live lanes and multiplexed sessions. A transport Ping is sent after `long_poll_secs` without peer activity, including during continuous downlink traffic; missing peer activity for twice that creation-time interval makes a connection eligible for cleanup.
|
||||
Before HTTP `101`, a WebSocket-lane reservation binds to the exact process connection and lane incarnation; an accepted `OPEN` transfers ownership to the exact stream incarnation before its backend task can run. A late poll, close, or reservation drop from an older socket cannot acknowledge, close, or release a replacement that reused the same numeric lane ID.
|
||||
|
||||
WebSocket codec buffers and in-flight read/write messages share the process-owned `pending_bytes_global` budget with carrier queues and are additionally bounded by `websocket_bytes_global`. Admission leaves `websocket_http_connection_reserve` accepted connections for ordinary HTTP and decoys. Admission replacement selects dead active connections globally first, then uses same-session, same-profile-owner, and same-client-IP locality. An unrelated healthy victim is eligible only when the requester is below its fair byte share and the victim owner is above it. Within one locality, claimed or upgraded connections precede active lanes, active lanes precede active multiplexed sessions, and least-recent progress, creation order, and connection ID provide deterministic tie breaking. Memory-pressure cleanup uses the same dead-first and lifecycle ordering, preferring over-share owners without stalling when every owner is at or below its share. `max_websocket_evictions_in_flight` bounds concurrent exact eviction claims. Upgrade, first-message, write, backpressure, and eviction deadlines are frozen from the parent session. A transport Ping is sent after `long_poll_secs` without peer activity, including during continuous downlink traffic; missing peer activity for twice that creation-time interval makes an active connection eligible for cleanup.
|
||||
|
||||
Every pre-Upgrade authentication, shape, lane-reservation, or capacity failure follows the sanitized decoy path instead of exposing a WebSocket-specific status. The exact subprotocol contains the session bearer and must not be logged.
|
||||
|
||||
@@ -198,22 +235,29 @@ The frontend or `defaults` section must also set `timeout client 65s` or longer
|
||||
| --- | --- |
|
||||
| WEB listener inventory, bind address, and trust policy | Process-owned; restart Telemt. |
|
||||
| Any `[web.limits]` value | Process-owned memory/resource contract; restart Telemt. |
|
||||
| `web.enabled`, `web.carrier`, `web.debug`, timeouts, vhosts, profiles, and decoys | Applied by the config watcher or a runtime generation reload. |
|
||||
| Existing HTTP connections and WEB sessions | Keep their acquisition-time carrier, limits, and session deadlines; newly issued bridge sessions use the active carrier. WebSocket write, backpressure, and eviction operations read the active hot-reloaded deadlines. New logical streams use the active relay generation. |
|
||||
| Process shutdown | Uses the latest reloaded `web.timeouts.shutdown_secs`. |
|
||||
| `web.enabled`, carrier/negotiation policy, `web.debug`, timeouts, vhosts, profiles, and decoys | Applied by the config watcher or a runtime generation reload. |
|
||||
| Existing HTTP connections and WEB sessions | Keep their acquisition-time HTTP idle limit, carrier candidates, limits, body timeout, closed-token replay lifetime, and absolute session/negotiation deadlines; each issued bridge embeds its request, retry, and probe-coalescing values. WebSocket upgrade, open, write, backpressure, and eviction operations use the parent session's frozen deadlines. Newly issued bridges use the active policy, while new logical streams use the active relay generation. |
|
||||
| Process shutdown | Captures the latest reloaded `web.timeouts.shutdown_secs` once and shares that single absolute deadline across listener acceptors and connections plus WEB sessions and auxiliary tasks. The waits do not receive sequential per-component budgets. |
|
||||
|
||||
Each logical stream keeps its session's creation-time client IP and owns a process-unique, non-zero synthetic source port for the complete relay lifetime. This preserves one stable, non-colliding source/destination tuple for Direct and Middle-End KDF routing.
|
||||
|
||||
HTTP idle accounting protects only explicitly bounded request-body, long-poll, decoy connect/response-head, and pending-Upgrade phases. The operation's own deadline remains exact; if its lease is still present at that instant, the connection watchdog allows at most one connection-idle interval for the scheduled task to publish its timeout/result before forcing closure. Between exchanges, and after a response head is ready, progress resets the idle clock while a stalled response body remains idle-bounded. Completion of an older phase cannot release the deadline protection owned by a newer phase.
|
||||
|
||||
An `OPEN` reserves the bounded logical-stream and tuple ownership but does not consume the relay generation's `max_connections` permit. Telemt acquires that permit only after the first inner byte arrives; the frozen first-byte deadline and stream limits bound silent opens, and capacity exhaustion then closes only the affected stream.
|
||||
|
||||
## API management
|
||||
|
||||
API management is available, but it is intentionally partial. There is no mutable `/v1/web` resource; the API listener exposes the read-only HTML debug view at `/web-status`.
|
||||
WEB configuration, runtime status, and bounded runtime controls share the authenticated API listener. `/web-status` remains a read-only HTML diagnostic view; state-changing operations exist only under `/v1/runtime/web`.
|
||||
|
||||
| Operation | API support |
|
||||
| --- | --- |
|
||||
| Read or patch `[web]`, vhosts, profiles, decoys, timeouts, or limits | No. `GET /v1/config` omits `[web]`; `PATCH /v1/config` returns `400 section_not_editable` for `web`. |
|
||||
| Read or patch `[web]`, vhosts, profiles, decoys, timeouts, or limits | Yes, through `GET` or `PATCH /v1/config`. The derived `web.runtime` snapshot is never returned or writable. Nested tables merge field-by-field; arrays replace the previous array wholesale. Every `[web.limits]` change is accepted as desired configuration but reported as deferred until process restart. |
|
||||
| Persist `server.listeners` | Yes, through `PATCH /v1/config`, but a changed WEB listener remains deferred until process restart. |
|
||||
| Apply an externally edited WEB configuration | Yes, through `POST /v1/system/reload`, then inspect the operation status. |
|
||||
| Inspect bounded server-side WEB request and lifecycle details | Yes, through authenticated `GET /web-status`. |
|
||||
| Inspect lifecycle, capacity planes, learning/debug state, and live sessions | Yes, through `GET /v1/runtime/web/status` and `/v1/runtime/web/sessions`. |
|
||||
| Close selected live WEB sessions | Yes, through the asynchronous `POST /v1/runtime/web/sessions/close` operation. |
|
||||
| Clear debug records or reset carrier learning | Yes, through the corresponding runtime POST endpoints. |
|
||||
| Manage `[access.users]` | Yes, through `/v1/users`. User creation does not create a WEB profile. |
|
||||
| Revoke one user | Yes. `/v1/users/{username}/disable` updates admission immediately and cancels that user's active sessions. |
|
||||
|
||||
@@ -230,6 +274,20 @@ read_only = false
|
||||
|
||||
The API whitelist checks the direct TCP peer and does not trust `X-Forwarded-For`. Changes to `[server.api]` itself require a process restart.
|
||||
|
||||
### Runtime status and control
|
||||
|
||||
`GET /v1/runtime/web/status` always returns the published lifecycle (`starting`, `no_web_listener`, `running`, `draining`, `drained`, or `deadline_exceeded`), its epoch and age, effective listener addresses, and availability. When the process-owned WEB runtime is alive, `runtime` adds its random 128-bit `runtime_instance`, active generation, immutable limits, plane-local capacity counters, carrier-learning/debug epochs, and totals. Status collection uses non-blocking plane reads: a contended plane is omitted and named in `partial`; the endpoint never waits for, cleans up, or mutates the data plane.
|
||||
|
||||
`GET /v1/runtime/web/sessions` returns at most 50 sessions by default and at most 200 when `limit` is supplied. Its ordered scan is capped at 1000 candidates. `cursor` and `session_ref` use the opaque canonical form `ws1.<runtime-instance>.<lowercase-hex-id>`; exact `session_ref` is mutually exclusive with `cursor` and `limit`. Filters are `ip`, `host`, `user`, `user_agent_id`, `key_id`, `carrier`, and `state`; duplicate or unknown query fields are rejected. The detail route is `GET /v1/runtime/web/sessions/{session_ref}`. A retained closed-session tombstone returns `410`; a contended exact snapshot returns `503 web_snapshot_busy`. Responses expose bounded non-secret metadata and never expose bootstrap/session bearers, capabilities, secret hashes, or synthetic/KDF ports.
|
||||
|
||||
Every runtime POST requires `Content-Type: application/json` exactly, rejects unknown JSON fields, obeys API authentication, whitelist, and `read_only`, and carries the current `runtime_instance` as an ABA fence. Available controls are:
|
||||
|
||||
- `POST /v1/runtime/web/sessions/close` with one selector: `{"kind":"refs","session_refs":[...]}`, `{"kind":"filter",...}`, or `{"kind":"all"}`. Exact refs are limited to 200, a filter must be non-empty, only one close operation may run, and `all` is rejected while effective issuance remains enabled. The `202` response returns `operation_id`; poll `GET /v1/runtime/web/operations/{operation_id}`. The operation scans only sessions at or below its submission high-water mark in chunks of 128.
|
||||
- `POST /v1/runtime/web/debug/clear` with `{"runtime_instance":"..."}`. The response reports cleared records, bytes still leased by already rendered snapshots, and the new epoch. In-flight writers from the old epoch cannot repopulate the ring.
|
||||
- `POST /v1/runtime/web/carrier-learning/reset` with the same body shape. It clears retained process-local evidence and advances the learning epoch; already frozen attempt chains and live sessions are unchanged.
|
||||
|
||||
For a deterministic close-all, patch `{"web":{"enabled":false}}` with runtime reload enabled, wait until `runtime.manager.issuance_enabled` is `false`, submit the `all` selector using that same `runtime_instance`, and poll the operation to a terminal state. Disabling WEB stops new bootstrap/session issuance but never implicitly closes existing sessions.
|
||||
|
||||
### Server-side WEB debug view
|
||||
|
||||
Enable bounded collection in the owned configuration file:
|
||||
@@ -248,7 +306,7 @@ default_window_secs = 180
|
||||
max_window_secs = 3600
|
||||
```
|
||||
|
||||
Open `http://127.0.0.1:9091/web-status` with the same direct-peer whitelist and exact `Authorization` header used by the API. A trailing slash is accepted. Only `GET` is allowed. The page supports `window_secs`, canonical `ip`, numeric `session`, case-insensitive `user_agent`, and `key` filters. Repeat `group_by=ip`, `group_by=session`, `group_by=user_agent`, or `group_by=key` to build grouped summaries; `limit` is restricted to `1..=1000`. HTTP rows expand from request through response with method, path, sanitized headers, body metadata or bytes, timing points, parsed frames, and typed lifecycle events. WebSocket operation adds the sanitized `GET` to `101` handshake plus bounded per-message direction, message type, payload/body capture, processing time, connection/lane identifiers, and parsed inner frames. Raw subprotocols and session tokens are never retained.
|
||||
Open `http://127.0.0.1:9091/web-status` with the same direct-peer whitelist and exact `Authorization` header used by the API. A trailing slash is accepted. Only `GET` is allowed. The page supports `window_secs`, canonical `ip`, numeric `session`, case-insensitive `user_agent`, and `key` filters. Repeat `group_by=ip`, `group_by=session`, `group_by=user_agent`, or `group_by=key` to build grouped summaries; `limit` is restricted to `1..=1000`. HTTP rows expand from request through response with method, path, sanitized headers, body metadata or bytes, timing points, parsed frames, and typed lifecycle events, including carrier attempt, commit, healthy, and reported-failure transitions. WebSocket operation adds the sanitized `GET` to `101` handshake plus bounded per-message direction, message type, payload/body capture, processing time, connection/lane identifiers, and parsed inner frames. Raw subprotocols and session tokens are never retained.
|
||||
|
||||
The process-owned ring survives runtime generation replacement. Capture-policy changes clear incompatible retained records; window-only changes do not. The ring defaults to 65536 records and 64 MiB retained plus in-flight bytes, the HTML response is capped at 8 MiB, grouping is capped at 1024 groups, and no more than two response bodies retain page permits concurrently. Change `web.limits.debug_records_capacity` or `web.limits.debug_bytes_global` only with a process restart. A hot prefix that fits only a simultaneously increased restart-only capacity is deferred until that restart.
|
||||
|
||||
@@ -267,7 +325,7 @@ curl -sS http://127.0.0.1:9091/v1/system/reload/RELOAD_ID \
|
||||
-H "Authorization: ${TELEMT_API_AUTH}"
|
||||
```
|
||||
|
||||
A terminal `succeeded` status confirms runtime activation. A changed `web.carrier` is used by newly issued bridge sessions; existing sessions are not migrated. If `deferred_process_fields` contains `server.listeners` or `web.limits`, the file is valid and persisted but those settings still require a Telemt restart.
|
||||
A terminal `succeeded` status confirms runtime activation. Changed carrier, candidate, deadline, or learning policy is used by newly issued bridge sessions; existing sessions and in-flight attempt chains are not migrated. If `deferred_process_fields` contains `server.listeners` or `web.limits`, the file is valid and persisted but those settings still require a Telemt restart.
|
||||
|
||||
Access-user operations use the existing endpoints, for example:
|
||||
|
||||
@@ -291,7 +349,7 @@ See the complete [Control API contract](../Architecture/API/API.md) for request
|
||||
- Disable request-target and authorization logging at the TLS terminator, or use a verified redacted format. Raw queries contain bridge capabilities and `Authorization` contains bootstrap or session bearer credentials.
|
||||
- Keep one stable public address per vhost. If DNS returns several ingress addresses, each deployment must use the address matching its external path.
|
||||
- Bootstrap and session registries are process-local. A multi-process or multi-host upstream pool requires affinity for the complete vhost: bridge GET, session creation, uplink, downlink, and DELETE. A single Telemt process needs no extra affinity.
|
||||
- An unused bootstrap survives a configuration reload only when the exact profile identity remains active: host, `public_addr`, user, secret mode, carrier, and capability. Existing created sessions retain their immutable carrier and profile identity and remain lifecycle-bounded.
|
||||
- An unused bootstrap survives a configuration reload only when the exact profile identity remains active: host, `public_addr`, user, secret mode, carrier candidates, negotiation deadlines, and capability. Existing created sessions retain their immutable carrier and profile identity and remain lifecycle-bounded.
|
||||
- The decoy is part of the anti-probing contract. Verify its ordinary 404 behavior and response timing through the public TLS endpoint before distributing links.
|
||||
|
||||
## Initial verification
|
||||
@@ -304,6 +362,7 @@ See the complete [Control API contract](../Architecture/API/API.md) for request
|
||||
6. For `websocket`, confirm one `101` response, binary relay traffic, and RFC 6455 Ping/Pong beyond 25 seconds. For `websocket-lanes`, exercise at least two simultaneous stream sockets and verify that closing or corrupting one lane does not close its sibling or parent session.
|
||||
7. Exercise reconnect and at least one long poll beyond 25 seconds to prove the frontend timeouts do not truncate the carrier.
|
||||
8. Verify user and logical MTProxy connection limits using logical-stream counters, not the number of HTTP connections.
|
||||
9. When auto-negotiation is enabled, verify the configured sequence, exact-attempt replay after an intentionally lost response, terminal behavior after commit, and `carrier_committed`/`carrier_healthy` lifecycle rows in `/web-status`. Verify that a metadata-free native client uses the fixed `carrier` without automatic response headers and that explicit capabilities remain unchanged.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
@@ -311,6 +370,8 @@ See the complete [Control API contract](../Architecture/API/API.md) for request
|
||||
| --- | --- |
|
||||
| WEB configuration is valid on disk but listener behavior did not change | Inspect reload `deferred_process_fields`; listener and `[web.limits]` changes require restart. |
|
||||
| Carrier requests reach the decoy | Verify exact vhost, link secret mode, direct proxy CIDR, and one parseable `X-Forwarded-For` value. |
|
||||
| A racing `https-lanes` downlink reaches the decoy with `404` | Confirm it starts at `X-Down-Cursor: 0`, preserve `X-Lane-ID`, and set `lane_open_wait_secs` above the observed down-before-`OPEN` skew. Advanced cursors for missing lanes intentionally fail closed. |
|
||||
| Auto-negotiation advances after traffic was already accepted | This is not valid behavior. Inspect the authenticated `X-Carrier-State` replay and the carrier commit lifecycle row; a committed or healthy response is terminal and requires a new session. |
|
||||
| Long polls disconnect near a fixed interval | Raise NGINX/HAProxy client, server, send, and read timeouts above `web.timeouts.long_poll_secs`. |
|
||||
| WebSocket Upgrade reaches the decoy instead of returning `101` | Preserve HTTP/1.1 `Connection: Upgrade`, `Upgrade: websocket`, the single exact `Sec-WebSocket-Protocol`, and the canonical bodyless `/api/v1/ws` request. Also check carrier/session compatibility and the process connection reserve. |
|
||||
| One `websocket-lanes` stream closes while siblings stay connected | This is the intended failure boundary. Inspect that lane's message/frame rows in `/web-status`; malformed, cross-lane, write-timeout, and backend-close paths terminate only the affected lane. |
|
||||
|
||||
+73
-12
@@ -28,7 +28,9 @@ WEB-listener Telemt
|
||||
|
||||
- Публичный endpoint всегда имеет вид `https://HOST:443`.
|
||||
- Поддерживаются 16-байтовые MTProxy-секреты `plain` и `dd`. FakeTLS-секреты `ee` в WEB-режиме не поддерживаются.
|
||||
- `web.carrier = "https"` выбирает сериализованные HTTPS uplink и long polling. `https-lanes` выбирает независимые HTTPS sequencing и polling для каждого logical stream. `websocket` выбирает один упорядоченный WebSocket для всех streams. `websocket-lanes` выбирает отдельный WebSocket с независимым ownership для каждого ненулевого logical stream.
|
||||
- `web.carrier` выбирает единственный carrier при выключенном auto-negotiation и последний fallback при включённом. `https` использует сериализованные HTTPS uplink и long polling. `https-lanes` использует независимые HTTPS sequencing и polling для каждого logical stream. `websocket` использует один упорядоченный WebSocket для всех streams. `websocket-lanes` использует отдельный WebSocket с независимым ownership для каждого ненулевого logical stream.
|
||||
- Отсутствующий `web.carriers` или `web.carriers = false` отключает auto-negotiation и обучение. Непустой массив включает только стартовый последовательный перебор; уже committed session никогда не мигрирует.
|
||||
- Нативные клиенты без канонических headers carrier negotiation используют настроенный фиксированный `carrier`, даже когда `carriers` включает negotiation для поддерживающих его клиентов. Текущий Telegram iOS поддерживает только `https`, поэтому для metadata-free iOS оператор должен задать `web.carrier = "https"`; `https-lanes` этим клиентом не поддерживается. User-Agent, включая CFNetwork или Darwin, никогда не выводит capabilities неявно. Если нативный iOS всё же отправляет явные negotiation metadata, Telemt пересекает их с server-authoritative ceiling `{https}` и отклоняет пустой результат; остальные явные клиенты используют заявленный capability set.
|
||||
- Capability, bootstrap и session credentials — отдельные значения с ограниченным сроком жизни. Carrier credentials считаются секретами и не должны попадать в access logs.
|
||||
- Bootstrap является bearer credential, а не token с привязкой к source address. Адрес клиента и его IP-семейство могут измениться между загрузкой bridge и созданием session. Адрес выдачи продолжает учитываться в лимите неиспользованных bootstrap, а владельцем session становится адрес первого корректного запроса создания.
|
||||
- Внутренняя MTProxy-аутентификация ограничена пользователем и режимом секрета, выбранными профилем vhost. Некорректный внутренний handshake закрывает только свой logical stream и никогда не попадает в TCP masking path.
|
||||
@@ -91,17 +93,52 @@ max_streams = 512
|
||||
max_streams_per_session = 64
|
||||
```
|
||||
|
||||
## Server-side negotiation carrier
|
||||
|
||||
Auto-negotiation необязателен и выключен, пока `carriers` не задан явным непустым массивом. Настроенный `carrier` остаётся последним fallback и добавляется ровно один раз, даже если уже присутствует в массиве:
|
||||
|
||||
```toml
|
||||
[web]
|
||||
enabled = true
|
||||
carrier = "https"
|
||||
carriers = ["websocket-lanes", "websocket", "https-lanes"]
|
||||
carrier_learning = true
|
||||
carrier_negotiation_aggressiveness = "conservative"
|
||||
|
||||
[web.timeouts]
|
||||
carrier_negotiation_deadlines_secs = [3, 5, 8, 12]
|
||||
carrier_health_secs = 30
|
||||
carrier_learning_secs = 600
|
||||
bridge_request_secs = 10
|
||||
bridge_retry_secs = 90
|
||||
carrier_probe_coalesce_ms = 0
|
||||
```
|
||||
|
||||
Сгенерированный bridge отправляет канонические headers `X-Carrier-Capabilities`, `X-Carrier-Attempt` и, после первой попытки, `X-Carrier-Failure` в запросе `/session`. Каждый успешный automatic response возвращает `X-Carrier-Mode`, `X-Carrier-Attempt`, `X-Carrier-Candidate-Count`, `X-Carrier-Deadline` и `X-Carrier-State`. Bridge запускает локальный cumulative clock непосредственно перед первым запросом `/session`, а сервер фиксирует отдельный absolute chain deadline при приёме первой automatic attempt. Оба используют настроенные offsets и не сбрасываются при replacement. Для одного, двух, трёх и четырёх effective candidates checkpoints attempts равны соответственно `[d3]`, `[d0, d3]`, `[d0, d1, d3]` и `[d0, d1, d2, d3]`; финальному candidate всегда принадлежит `d3`. Successor остаётся допустимым до собственного checkpoint. Состояния: `provisional`, `committed` и `healthy`.
|
||||
|
||||
Попытки строго последовательны. Принятый прогресс `OPEN` или `DATA` немедленно фиксирует выбранный carrier и окончательно закрывает границу replacement. Аутентифицированный `409` для committed chain повторяет metadata зафиксированного carrier и является terminal response, а не разрешением перейти дальше. Точный replay `/session` применяется только пока результат этого запроса неоднозначен. После аутентифицированного выбора provisional carrier transport failure сразу запрашивает следующую attempt; если предыдущий probe всё же успел committed, сервер возвращает terminal `409` и не разрешает небезопасный replacement. Финальный абсолютный deadline на сервере также ограничивает lifetime successor, ответ которого клиент не получил. Динамическое post-commit переключение намеренно не поддерживается: для смены carrier требуется новая сессия.
|
||||
|
||||
Каждая HTTP-операция bridge имеет абсолютный budget `bridge_retry_secs` и не более девяти attempts. `bridge_request_secs` охватывает Fetch response head и полное чтение response body; для downlink attempt дополнительно разрешён настроенный long-poll interval. Network failures и ответы `408`, `429`, `502`, `503` или `504` используют bounded exponential backoff, а `Retry-After` не может расширить абсолютный budget. При `carrier_probe_coalesce_ms = 0` первый упорядоченный probe с `OPEN` отправляется немедленно. Значение до 10 мс позволяет включить соответствующий `DATA`, пришедший в этом окне; multiplexed carriers сохраняют весь предшествующий порядок frames, а lane carriers забирают только выбранную lane. HTTP downlink не запускается до acknowledgement probe. Multiplexed WebSocket Upgrade может начаться сразу после его выбора ответом `/session` и затем включить queued probe data; lane WebSocket ждёт известного stream ID.
|
||||
|
||||
Automatic WebSocket использует `tproxy-auto-v1.<session-token>` или `tproxy-auto-lane-v1.<session-token>.<stream-id>`. Первое принятое binary message с реальным прогрессом `OPEN` или `DATA` фиксирует carrier; затем сервер пишет пустой binary commit ACK именно в это connection. Ping/Pong не фиксирует carrier и не считается learning evidence.
|
||||
|
||||
Committed attempt становится healthy, только когда transport-specific двунаправленный evidence остаётся корректным в течение `carrier_health_secs`. HTTPS требует принятый `DATA`, подтверждённый непустой post-commit downlink batch и аутентифицированную активность не раньше health deadline. WebSocket требует записи точного commit ACK, последующего принятого `OPEN` или `DATA` от того же owner и сохранения этого owner живым до конца интервала. Более раннее закрытие нейтрально и не записывает результат обучения.
|
||||
|
||||
Обучение process-local, in-memory, positive-only и ограничено `max_carrier_learning_entries`. Оно ранжирует только поддерживаемые клиентом настроенные candidates, всегда оставляет fallback последним и сохраняет настроенный порядок при равных scores. Evidence User-Agent и профиля имеет основной вес; допустимый IP служит только tie-breaker. Для IP evidence требуется ровно один явный глобально маршрутизируемый `X-Forwarded-For`; private, loopback, link-local, carrier-grade NAT, documentation, multicast и их IPv4-mapped эквиваленты исключаются. Категории ошибок от клиента и request latency используются только для диагностики и не создают отрицательный или ranking evidence. `conservative` требует 3 outcomes User-Agent или 8 outcomes профиля в 4 cohorts и отключает IP evidence; `balanced` использует соответственно 2, 6 в 3 cohorts и 3 outcomes допустимого IP; `aggressive` — 1, 4 в 2 cohorts и 1 outcome IP. Выключение обучения или смена policy при reload очищает несовместимый evidence, не меняя уже начатые сессии.
|
||||
|
||||
`https` остаётся default и сохраняет исходное сериализованное поведение. В `https-lanes` lane zero отведена под session control, а каждому ненулевому logical stream соответствует своя lane. У каждой lane собственные uplink sequence, retry digest, downlink cursor, unacknowledged replay batch, очередь и lifecycle newest-poll-wins. Поэтому медленный stream не блокирует другой stream на уровне WEB-протокола.
|
||||
|
||||
Это устраняет сериализацию между WEB-streams на уровне приложения. Публичный HTTP/2 всё ещё работает поверх одного или нескольких TCP-connections, поэтому потеря пакетов может вызвать transport-level head-of-line blocking; `https-lanes` не является HTTP/3- или QUIC-carrier.
|
||||
|
||||
Все lane queues входят в существующие per-session и process-wide byte/item budgets. Bridge дополнительно ограничивает одну lane 8 MiB и 1024 элементами. Lane long polls могут занимать не более половины `web.limits.max_http_handlers`, оставляя handler capacity для session creation, uplink, DELETE и другой control work. Для `https-lanes` требуется `max_http_handlers >= 2`.
|
||||
Все lane queues и resident response bodies входят в существующие per-session и process-wide byte/item budgets. Telemt дополнительно ограничивает одну lane значениями `pending_bytes_per_lane` и `pending_items_per_lane`; сгенерированный bridge ограничивает свои очереди 8 MiB и 1024 элементами. Lane long polls могут занимать не более половины `web.limits.max_http_handlers`, оставляя handler capacity для session creation, uplink, DELETE и другой control work. Для `https` требуется `max_http_handlers >= 2`, для `https-lanes` — `max_http_handlers >= 4`.
|
||||
|
||||
Paths `/api/v1/up` и `/api/v1/down` не меняются. В `https-lanes` каждый запрос к ним содержит один канонический десятичный `X-Lane-ID`. Uplink sequence начинается с `1`, а downlink cursor — с `0` независимо для каждой lane. Lane zero принимает только session `PONG`; все frames ненулевой lane должны иметь тот же stream ID, а новая lane должна начинаться с `OPEN`. После отправки всей queued и unacknowledged downlink data закрытой lane Telemt возвращает пустой ответ с `X-Lane-Closed: 1`, и bridge прекращает её polling. Retry остаются byte-identical и повторяют исходный acknowledgement или downlink batch.
|
||||
Paths `/api/v1/up` и `/api/v1/down` не меняются. В `https-lanes` каждый запрос к ним содержит один канонический десятичный `X-Lane-ID`. Uplink sequence начинается с `1`, а downlink cursor — с `0` независимо для каждой lane. Lane zero принимает только session `PONG`; все frames ненулевой lane должны иметь тот же stream ID, а новая lane должна начинаться с `OPEN`. Канонический downlink с cursor zero, пришедший немного раньше `OPEN` своей lane, ждёт до `lane_open_wait_secs` без создания lane state; число таких ожиданий ограничено per-session и process auxiliary permits. Истечение таймаута возвращает пустой `204`, а отсутствующая lane с продвинутым cursor остаётся protocol failure и уходит в decoy. После отправки всей queued и unacknowledged downlink data закрытой lane Telemt возвращает пустой ответ с `X-Lane-Closed: 1`, и bridge прекращает её polling. Retry остаются byte-identical и повторяют исходный acknowledgement или downlink batch.
|
||||
|
||||
Оба WebSocket carrier по-прежнему создают и удаляют parent session через HTTPS, после чего используют строгий bodyless Upgrade-запрос `GET /api/v1/ws`. `websocket` передаёт в `Sec-WebSocket-Protocol` ровно `tproxy-v1.<session-token>`; binary messages являются упорядоченными carrier batches, а ошибка протокола, deadline или connection закрывает всю parent session. `websocket-lanes` передаёт ровно `tproxy-lane-v1.<session-token>.<stream-id>`, где stream ID записан каноническим десятичным числом из диапазона `1..=16777215`. Первое binary message должно начинаться с `OPEN`, все frames должны содержать этот stream ID, а сбой после Upgrade закрывает только данную lane. Lane-zero WebSocket отсутствует: HTTPS переносит `HELLO` и `WELCOME`, а liveness connection обеспечивает RFC 6455 Ping/Pong.
|
||||
|
||||
WebSocket codec buffers и находящиеся в обработке read/write messages делят process-owned `pending_bytes_global` с carrier queues и дополнительно ограничены `websocket_bytes_global`. Admission оставляет `websocket_http_connection_reserve` принятых connections для обычного HTTP и decoy. При pressure вытеснение сначала выбирает того же owner, затем connection с наиболее старым прогрессом; pre-Upgrade и dead connections идут раньше активных lanes и multiplexed sessions. После `long_poll_secs` без peer activity отправляется transport Ping, в том числе при непрерывном downlink traffic, а отсутствие peer activity в течение удвоенного creation-time интервала делает connection кандидатом на cleanup.
|
||||
До HTTP `101` reservation WebSocket lane привязывается к точным process connection и incarnation lane; принятый `OPEN` передаёт ownership точному incarnation stream до запуска его backend task. Поздний poll, close или drop reservation от старого socket не может подтвердить, закрыть или освободить replacement, повторно использующий тот же числовой lane ID.
|
||||
|
||||
WebSocket codec buffers и находящиеся в обработке read/write messages делят process-owned `pending_bytes_global` с carrier queues и дополнительно ограничены `websocket_bytes_global`. Admission оставляет `websocket_http_connection_reserve` принятых connections для обычного HTTP и decoy. При admission replacement сначала глобально выбираются dead active connections, затем применяются уровни locality: та же session, тот же profile owner и тот же client IP. Не связанный с ними healthy victim допустим только когда requester использует меньше своей fair byte share, а owner victim — больше. Внутри одного уровня locality claimed или upgraded connections идут перед active lanes, lanes — перед active multiplexed sessions; дальнейший порядок детерминируют время последнего прогресса, создания и connection ID. Cleanup при memory pressure использует тот же dead-first и lifecycle-порядок, предпочитая owners выше fair share, но продолжает eviction, если все owners находятся на своей share или ниже. `max_websocket_evictions_in_flight` ограничивает одновременные точные eviction claims. Deadlines Upgrade, первого message, write, backpressure и eviction заморожены из parent session. После `long_poll_secs` без peer activity отправляется transport Ping, в том числе при непрерывном downlink traffic, а отсутствие peer activity в течение удвоенного creation-time интервала делает active connection кандидатом на cleanup.
|
||||
|
||||
Любая ошибка authentication, shape, lane reservation или capacity до Upgrade следует по очищенному decoy path и не раскрывает WebSocket-специфичный status. Точный subprotocol содержит session bearer и не должен попадать в logs.
|
||||
|
||||
@@ -198,22 +235,29 @@ backend telemt_web
|
||||
| --- | --- |
|
||||
| Состав WEB-listeners, bind address и trust policy | Принадлежат процессу; перезапустите Telemt. |
|
||||
| Любое значение `[web.limits]` | Process-owned контракт памяти и ресурсов; перезапустите Telemt. |
|
||||
| `web.enabled`, `web.carrier`, `web.debug`, timeouts, vhosts, profiles и decoys | Применяются config watcher или runtime generation reload. |
|
||||
| Существующие HTTP connections и WEB sessions | Сохраняют carrier, лимиты и session deadlines своего момента создания; новые bridge sessions получают активный carrier. WebSocket write, backpressure и eviction operations читают активные hot-reloaded deadlines. Новые logical streams используют активное relay generation. |
|
||||
| Завершение процесса | Использует последнее применённое значение `web.timeouts.shutdown_secs`. |
|
||||
| `web.enabled`, policy carrier/negotiation, `web.debug`, timeouts, vhosts, profiles и decoys | Применяются config watcher или runtime generation reload. |
|
||||
| Существующие HTTP connections и WEB sessions | Сохраняют HTTP idle limit, carrier candidates, лимиты, body timeout, lifetime replay-marker закрытого token и абсолютные session/negotiation deadlines своего момента создания; каждый выданный bridge содержит собственные request, retry и probe-coalescing значения. WebSocket Upgrade, open, write, backpressure и eviction operations используют замороженные deadlines parent session. Новые bridges получают активную policy, а новые logical streams используют активное relay generation. |
|
||||
| Завершение процесса | Один раз фиксирует последнее применённое значение `web.timeouts.shutdown_secs` и использует единый абсолютный deadline для listener acceptors и connections, WEB sessions и auxiliary tasks. Последовательные компоненты не получают отдельные полные бюджеты. |
|
||||
|
||||
Каждый logical stream сохраняет client IP своей сессии и владеет уникальным в пределах процесса ненулевым synthetic source port до завершения relay. Это сохраняет один стабильный непересекающийся source/destination tuple для Direct и Middle-End KDF routing.
|
||||
|
||||
HTTP idle accounting защищает только явно ограниченные фазы request body, long poll, подключения/response head decoy и ожидания Upgrade. Собственный deadline операции остаётся точным; если в этот момент её lease ещё существует, connection watchdog даёт запланированной задаче не более одного connection-idle interval для публикации timeout/result, после чего принудительно закрывает connection. Между обменами и после готовности response head прогресс сбрасывает idle-таймер, а зависший response body остаётся ограничен idle timeout. Завершение старой фазы не может снять deadline-защиту, которой уже владеет новая фаза.
|
||||
|
||||
`OPEN` резервирует bounded ownership logical stream и tuple, но не занимает permit `max_connections` relay generation. Telemt получает этот permit только после первого внутреннего байта; замороженный first-byte deadline и stream limits ограничивают silent opens, а исчерпание capacity закрывает только затронутый stream.
|
||||
|
||||
## Управление через API
|
||||
|
||||
Управление через API доступно, но намеренно ограничено. Изменяемого ресурса `/v1/web` нет; API-listener предоставляет read-only HTML debug view по адресу `/web-status`.
|
||||
Конфигурация WEB, статус runtime и bounded runtime-управление доступны на одном аутентифицированном API-listener. `/web-status` остаётся read-only HTML-диагностикой; операции, изменяющие состояние, существуют только под `/v1/runtime/web`.
|
||||
|
||||
| Операция | Поддержка API |
|
||||
| --- | --- |
|
||||
| Чтение или изменение `[web]`, vhosts, profiles, decoys, timeouts или limits | Нет. `GET /v1/config` не возвращает `[web]`; `PATCH /v1/config` отвечает `400 section_not_editable` на ключ `web`. |
|
||||
| Чтение или изменение `[web]`, vhosts, profiles, decoys, timeouts или limits | Да, через `GET` или `PATCH /v1/config`. Производный snapshot `web.runtime` не возвращается и недоступен для записи. Вложенные tables сливаются по полям; arrays целиком заменяют прежний array. Любое изменение `[web.limits]` принимается как desired configuration, но помечается deferred до перезапуска процесса. |
|
||||
| Сохранение `server.listeners` | Да, через `PATCH /v1/config`, но изменённый WEB-listener остаётся deferred до перезапуска процесса. |
|
||||
| Применение WEB-конфигурации, изменённой вне API | Да, через `POST /v1/system/reload` с последующей проверкой статуса операции. |
|
||||
| Просмотр bounded серверных WEB request- и lifecycle-деталей | Да, через аутентифицированный `GET /web-status`. |
|
||||
| Просмотр lifecycle, capacity planes, состояния learning/debug и активных сессий | Да, через `GET /v1/runtime/web/status` и `/v1/runtime/web/sessions`. |
|
||||
| Закрытие выбранных активных WEB-сессий | Да, через асинхронную операцию `POST /v1/runtime/web/sessions/close`. |
|
||||
| Очистка debug-записей или сброс carrier learning | Да, через соответствующие runtime POST endpoints. |
|
||||
| Управление `[access.users]` | Да, через `/v1/users`. Создание пользователя не создаёт WEB-профиль. |
|
||||
| Отзыв отдельного пользователя | Да. `/v1/users/{username}/disable` немедленно обновляет admission и завершает активные сессии пользователя. |
|
||||
|
||||
@@ -230,6 +274,20 @@ read_only = false
|
||||
|
||||
API whitelist проверяет непосредственный TCP peer и не доверяет `X-Forwarded-For`. Изменения самой секции `[server.api]` требуют перезапуска процесса.
|
||||
|
||||
### Статус и управление runtime
|
||||
|
||||
`GET /v1/runtime/web/status` всегда возвращает опубликованный lifecycle (`starting`, `no_web_listener`, `running`, `draining`, `drained` или `deadline_exceeded`), его epoch и возраст, эффективные адреса listeners и доступность. Пока process-owned WEB runtime существует, поле `runtime` добавляет случайный 128-битный `runtime_instance`, активное поколение, неизменяемые limits, capacity counters отдельных planes, epochs carrier-learning/debug и суммарные counters. Status собирается неблокирующим чтением каждого plane: занятый plane пропускается и указывается в `partial`; endpoint никогда не ожидает data plane, не выполняет cleanup и не изменяет его.
|
||||
|
||||
`GET /v1/runtime/web/sessions` возвращает не более 50 сессий по умолчанию и не более 200 при заданном `limit`. Упорядоченный scan ограничен 1000 кандидатами. `cursor` и `session_ref` имеют opaque canonical вид `ws1.<runtime-instance>.<lowercase-hex-id>`; точный `session_ref` нельзя сочетать с `cursor` или `limit`. Доступны фильтры `ip`, `host`, `user`, `user_agent_id`, `key_id`, `carrier` и `state`; повторяющиеся или неизвестные query fields отклоняются. Детальная операция — `GET /v1/runtime/web/sessions/{session_ref}`. Сохранённый tombstone закрытой сессии возвращает `410`; занятый точный snapshot — `503 web_snapshot_busy`. Ответы содержат только bounded несекретные metadata и никогда не раскрывают bootstrap/session bearers, capabilities, hashes секретов или synthetic/KDF ports.
|
||||
|
||||
Каждый runtime POST требует ровно `Content-Type: application/json`, отклоняет неизвестные JSON fields, наследует API authentication, whitelist и `read_only`, а также содержит текущий `runtime_instance` как ABA-fence. Доступные операции:
|
||||
|
||||
- `POST /v1/runtime/web/sessions/close` с одним selector: `{"kind":"refs","session_refs":[...]}`, `{"kind":"filter",...}` или `{"kind":"all"}`. Точные refs ограничены 200, filter должен быть непустым, одновременно выполняется не более одной close operation, а `all` отклоняется, пока effective issuance включён. Ответ `202` содержит `operation_id`; опрашивайте `GET /v1/runtime/web/operations/{operation_id}`. Операция chunks по 128 сканирует только сессии не выше submission high-water mark.
|
||||
- `POST /v1/runtime/web/debug/clear` с `{"runtime_instance":"..."}`. Ответ содержит число удалённых записей, bytes, всё ещё удерживаемые уже отрисовываемыми snapshots, и новый epoch. In-flight writers старого epoch не могут снова заполнить ring.
|
||||
- `POST /v1/runtime/web/carrier-learning/reset` с тем же body. Операция очищает сохранённый process-local evidence и увеличивает learning epoch; уже замороженные attempt chains и активные сессии не изменяются.
|
||||
|
||||
Для детерминированного close-all отправьте patch `{"web":{"enabled":false}}` с включённым runtime reload, дождитесь `runtime.manager.issuance_enabled = false`, отправьте selector `all` с тем же `runtime_instance` и опрашивайте operation до terminal state. Отключение WEB прекращает новую выдачу bootstrap/session credentials, но никогда не закрывает существующие сессии неявно.
|
||||
|
||||
### Серверная WEB-отладка
|
||||
|
||||
Включите bounded сбор в конфигурационном файле, которому принадлежит эта секция:
|
||||
@@ -248,7 +306,7 @@ default_window_secs = 180
|
||||
max_window_secs = 3600
|
||||
```
|
||||
|
||||
Откройте `http://127.0.0.1:9091/web-status`, используя те же whitelist непосредственных peers и точный header `Authorization`, что и для API. Завершающий slash разрешён. Допускается только `GET`. Страница поддерживает фильтры `window_secs`, канонический `ip`, числовой `session`, регистронезависимый `user_agent` и `key`. Повторяйте `group_by=ip`, `group_by=session`, `group_by=user_agent` или `group_by=key` для построения сгруппированных сводок; `limit` ограничен диапазоном `1..=1000`. HTTP rows раскрываются от request до response с method, path, очищенными headers, метаданными или байтами body, timing points, frames и типизированными lifecycle events. Для WebSocket добавляются очищенный handshake `GET` → `101` и bounded per-message direction, message type, payload/body capture, processing time, connection/lane identifiers и разобранные inner frames. Raw subprotocol и session tokens никогда не сохраняются.
|
||||
Откройте `http://127.0.0.1:9091/web-status`, используя те же whitelist непосредственных peers и точный header `Authorization`, что и для API. Завершающий slash разрешён. Допускается только `GET`. Страница поддерживает фильтры `window_secs`, канонический `ip`, числовой `session`, регистронезависимый `user_agent` и `key`. Повторяйте `group_by=ip`, `group_by=session`, `group_by=user_agent` или `group_by=key` для построения сгруппированных сводок; `limit` ограничен диапазоном `1..=1000`. HTTP rows раскрываются от request до response с method, path, очищенными headers, метаданными или байтами body, timing points, frames и типизированными lifecycle events, включая carrier attempt, commit, healthy и reported-failure transitions. Для WebSocket добавляются очищенный handshake `GET` → `101` и bounded per-message direction, message type, payload/body capture, processing time, connection/lane identifiers и разобранные inner frames. Raw subprotocol и session tokens никогда не сохраняются.
|
||||
|
||||
Process-owned кольцевой буфер переживает замену runtime generation. Изменения capture policy очищают несовместимые сохранённые записи; изменения только окна наблюдения этого не делают. По умолчанию кольцо ограничено 65536 записями и 64 MiB сохранённых плюс находящихся в обработке данных, HTML-response — 8 MiB, grouping — 1024 группами; одновременно page permits могут удерживать не более двух response bodies. Изменяйте `web.limits.debug_records_capacity` или `web.limits.debug_bytes_global` только с перезапуском процесса. Hot prefix, который помещается только в одновременно увеличенную restart-only ёмкость, откладывается до этого перезапуска.
|
||||
|
||||
@@ -267,7 +325,7 @@ curl -sS http://127.0.0.1:9091/v1/system/reload/RELOAD_ID \
|
||||
-H "Authorization: ${TELEMT_API_AUTH}"
|
||||
```
|
||||
|
||||
Терминальный статус `succeeded` подтверждает активацию runtime. Изменённый `web.carrier` используют новые bridge sessions; существующие сессии не мигрируют. Если `deferred_process_fields` содержит `server.listeners` или `web.limits`, файл валиден и сохранён, но эти настройки всё ещё требуют перезапуска Telemt.
|
||||
Терминальный статус `succeeded` подтверждает активацию runtime. Изменённые carrier, candidates, deadlines или learning policy используют новые bridge sessions; существующие сессии и начатые attempt chains не мигрируют. Если `deferred_process_fields` содержит `server.listeners` или `web.limits`, файл валиден и сохранён, но эти настройки всё ещё требуют перезапуска Telemt.
|
||||
|
||||
Операции с access users используют существующие endpoints, например:
|
||||
|
||||
@@ -291,7 +349,7 @@ curl -sS -X POST http://127.0.0.1:9091/v1/users/web-user/rotate-secret \
|
||||
- Отключите логирование request target и authorization на TLS-терминаторе либо используйте проверенный формат с редактированием. Raw queries содержат bridge capabilities, а `Authorization` — bootstrap или session bearer credentials.
|
||||
- Сохраняйте один стабильный публичный адрес на vhost. Если DNS возвращает несколько ingress addresses, каждый deployment должен использовать адрес своего внешнего пути.
|
||||
- Bootstrap- и session-registries локальны для процесса. Для multi-process или multi-host upstream pool нужна affinity всего vhost: bridge GET, создание сессии, uplink, downlink и DELETE. Одному процессу Telemt дополнительная affinity не нужна.
|
||||
- Неиспользованный bootstrap переживает reload конфигурации, только если остаётся активной точная identity профиля: host, `public_addr`, user, secret mode, carrier и capability. Уже созданные sessions сохраняют неизменные carrier и identity профиля и остаются lifecycle-bounded.
|
||||
- Неиспользованный bootstrap переживает reload конфигурации, только если остаётся активной точная identity профиля: host, `public_addr`, user, secret mode, carrier candidates, negotiation deadlines и capability. Уже созданные sessions сохраняют неизменные carrier и identity профиля и остаются lifecycle-bounded.
|
||||
- Decoy входит в anti-probing contract. До распространения ссылок проверьте через публичный TLS endpoint его обычный ответ 404 и response timing.
|
||||
|
||||
## Первичная проверка
|
||||
@@ -304,6 +362,7 @@ curl -sS -X POST http://127.0.0.1:9091/v1/users/web-user/rotate-secret \
|
||||
6. Для `websocket` подтвердите один response `101`, binary relay traffic и RFC 6455 Ping/Pong после 25 секунд. Для `websocket-lanes` проверьте как минимум два одновременных stream sockets и убедитесь, что закрытие или повреждение одной lane не закрывает sibling или parent session.
|
||||
7. Проверьте reconnect и как минимум один long poll длительнее 25 секунд, чтобы frontend timeouts не обрывали carrier.
|
||||
8. Проверяйте лимиты пользователя и logical MTProxy connections по logical-stream counters, а не по числу HTTP connections.
|
||||
9. При включённом auto-negotiation проверьте настроенную последовательность, replay точно той же попытки после намеренно потерянного response, terminal-поведение после commit и lifecycle rows `carrier_committed`/`carrier_healthy` в `/web-status`. Убедитесь, что нативный клиент без metadata использует фиксированный `carrier` без automatic response headers, а явные capabilities остаются неизменными.
|
||||
|
||||
## Диагностика
|
||||
|
||||
@@ -311,6 +370,8 @@ curl -sS -X POST http://127.0.0.1:9091/v1/users/web-user/rotate-secret \
|
||||
| --- | --- |
|
||||
| WEB-конфигурация валидна на диске, но поведение listener’а не изменилось | Проверьте `deferred_process_fields`; listener и `[web.limits]` требуют перезапуска. |
|
||||
| Carrier-запросы попадают в decoy | Проверьте точный vhost, secret mode ссылки, CIDR непосредственного proxy и единственное корректно разбираемое значение `X-Forwarded-For`. |
|
||||
| Downlink `https-lanes`, участвующий в гонке, попадает в decoy с `404` | Убедитесь, что он начинается с `X-Down-Cursor: 0`, сохраняйте `X-Lane-ID` и задайте `lane_open_wait_secs` выше наблюдаемого разрыва down-before-`OPEN`. Продвинутый cursor отсутствующей lane намеренно закрывается fail-closed. |
|
||||
| Auto-negotiation переходит дальше после уже принятого трафика | Такое поведение некорректно. Проверьте аутентифицированный replay `X-Carrier-State` и lifecycle row commit carrier; ответ `committed` или `healthy` terminal и требует новой сессии. |
|
||||
| Long polls разрываются через фиксированный интервал | Поднимите client, server, send и read timeouts NGINX/HAProxy выше `web.timeouts.long_poll_secs`. |
|
||||
| WebSocket Upgrade попадает в decoy вместо `101` | Сохраните HTTP/1.1 `Connection: Upgrade`, `Upgrade: websocket`, единственный точный `Sec-WebSocket-Protocol` и канонический bodyless request `/api/v1/ws`. Также проверьте соответствие carrier/session и process connection reserve. |
|
||||
| Один stream `websocket-lanes` закрылся, а siblings остались подключены | Это штатная failure boundary. Проверьте message/frame rows этой lane в `/web-status`; malformed, cross-lane, write-timeout и backend-close закрывают только затронутую lane. |
|
||||
|
||||
@@ -87,6 +87,64 @@ async fn read_managed_config_strips_access() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_managed_config_exposes_web_without_runtime_or_access_secrets() {
|
||||
let (path, _directory) = temp_config(concat!(
|
||||
"[web]\nenabled = false\ncarrier = \"https\"\n",
|
||||
"[web.debug]\nenabled = true\ndefault_window_secs = 180\n",
|
||||
"[access.users]\nbob = \"00000000000000000000000000000000\"\n",
|
||||
));
|
||||
|
||||
let (value, _revision) = read_managed_config(&path).await.unwrap();
|
||||
let table = value.as_table().unwrap();
|
||||
|
||||
assert!(table.contains_key("web"));
|
||||
assert!(table["web"].get("debug").is_some());
|
||||
assert!(table["web"].get("runtime").is_none());
|
||||
assert!(!table.contains_key("access"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn patch_web_debug_is_hot_and_limits_are_process_deferred() {
|
||||
let (path, _directory) = temp_config("[web]\nenabled = false\n");
|
||||
let debug_patch: Json = serde_json::json!({
|
||||
"web": {"debug": {"enabled": true, "capture_headers": false}}
|
||||
});
|
||||
let debug = apply_patch_to_path(&path, &debug_patch, None)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!debug.process_restart_required);
|
||||
assert!(debug.changed.iter().any(|section| section == "web"));
|
||||
|
||||
let limits_patch: Json = serde_json::json!({
|
||||
"web": {"limits": {"max_http_connections": 2049}}
|
||||
});
|
||||
let limits = apply_patch_to_path(&path, &limits_patch, None)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(limits.process_restart_required);
|
||||
assert!(
|
||||
limits
|
||||
.deferred_process_fields
|
||||
.iter()
|
||||
.any(|field| field == "web.limits")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_web_patch_does_not_modify_the_source() {
|
||||
let (path, _directory) = temp_config("[web]\nenabled = false\n");
|
||||
let original = tokio::fs::read_to_string(&path).await.unwrap();
|
||||
let patch: Json = serde_json::json!({
|
||||
"web": {"debug": {"default_window_secs": 181, "max_window_secs": 180}}
|
||||
});
|
||||
|
||||
let error = apply_patch_to_path(&path, &patch, None).await.unwrap_err();
|
||||
|
||||
assert_eq!(error.status, hyper::StatusCode::BAD_REQUEST);
|
||||
assert_eq!(tokio::fs::read_to_string(&path).await.unwrap(), original);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_managed_config_returns_only_editable_sections() {
|
||||
// Full server (api/port) and network must not leak. Listeners-only server
|
||||
|
||||
@@ -262,12 +262,12 @@ pub(super) async fn load_config_from_disk(config_path: &Path) -> Result<ProxyCon
|
||||
.map_err(|e| ApiFailure::internal(format!("failed to load config: {}", e)))
|
||||
}
|
||||
|
||||
pub(super) async fn load_config_for_reload(config_path: &Path) -> Result<ProxyConfig, ApiFailure> {
|
||||
let config_path = config_path.to_path_buf();
|
||||
tokio::task::spawn_blocking(move || ProxyConfig::load(config_path))
|
||||
.await
|
||||
.map_err(|error| ApiFailure::internal(format!("failed to join config loader: {}", error)))?
|
||||
.map_err(|error| ApiFailure::bad_request(format!("invalid runtime config: {}", error)))
|
||||
pub(super) async fn load_config_for_reload(
|
||||
config_path: &Path,
|
||||
) -> Result<(ProxyConfig, String), ApiFailure> {
|
||||
let loaded = load_config_snapshot(config_path, true).await?;
|
||||
let revision = compute_snapshot_revision(&loaded);
|
||||
Ok((loaded.config, revision))
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
@@ -307,6 +307,7 @@ pub(super) const EDITABLE_SECTIONS: &[&str] = &[
|
||||
"censorship",
|
||||
"upstreams",
|
||||
"dc_overrides",
|
||||
"web",
|
||||
];
|
||||
|
||||
/// Nested fields under `[server]` that may be read/patched via the config API.
|
||||
|
||||
+41
-4
@@ -30,6 +30,7 @@ use crate::startup::StartupTracker;
|
||||
use crate::stats::Stats;
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
use crate::web::control::WebRuntimePublication;
|
||||
use crate::web::trace::WebTraceStore;
|
||||
|
||||
mod config_edit;
|
||||
@@ -48,6 +49,8 @@ mod runtime_stats;
|
||||
mod runtime_watch;
|
||||
mod runtime_zero;
|
||||
mod users;
|
||||
// WEB runtime status and bounded controls remain separate from general API DTOs.
|
||||
mod web_runtime;
|
||||
mod web_status;
|
||||
|
||||
use config_store::{
|
||||
@@ -125,6 +128,7 @@ pub(super) struct ApiShared {
|
||||
pub(super) reload_control: ReloadControl,
|
||||
pub(super) active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
pub(super) web_trace: Arc<WebTraceStore>,
|
||||
pub(super) web_runtime_rx: watch::Receiver<WebRuntimePublication>,
|
||||
}
|
||||
|
||||
impl ApiShared {
|
||||
@@ -159,6 +163,7 @@ impl ApiShared {
|
||||
reload_control: self.reload_control.clone(),
|
||||
active_runtime: self.active_runtime.clone(),
|
||||
web_trace: self.web_trace.clone(),
|
||||
web_runtime_rx: self.web_runtime_rx.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -196,9 +201,15 @@ async fn submit_reload_from_disk(
|
||||
request: ReloadRequest,
|
||||
) -> Result<(ReloadAccepted, String), ApiFailure> {
|
||||
let _guard = mutation_lock.lock().await;
|
||||
ensure_expected_revision(config_path, expected_revision).await?;
|
||||
let revision = current_revision(config_path).await?;
|
||||
let config = Arc::new(load_config_for_reload(config_path).await?);
|
||||
let (config, revision) = load_config_for_reload(config_path).await?;
|
||||
if expected_revision.is_some_and(|expected| expected != revision) {
|
||||
return Err(ApiFailure::new(
|
||||
StatusCode::CONFLICT,
|
||||
"revision_conflict",
|
||||
"Config revision mismatch",
|
||||
));
|
||||
}
|
||||
let config = Arc::new(config);
|
||||
let accepted = reload_control
|
||||
.submit(config, revision.clone(), request)
|
||||
.await
|
||||
@@ -218,6 +229,9 @@ async fn submit_reload_from_disk(
|
||||
}
|
||||
|
||||
fn allowed_methods_for_path(path: &str) -> Option<&'static str> {
|
||||
if let Some(allow) = web_runtime::allowed_methods(path) {
|
||||
return Some(allow);
|
||||
}
|
||||
match path {
|
||||
"/v1/health"
|
||||
| "/v1/health/ready"
|
||||
@@ -285,6 +299,7 @@ pub async fn serve(
|
||||
mut active_runtime_rx: watch::Receiver<Option<Arc<ArcSwap<RuntimeGeneration>>>>,
|
||||
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||
web_trace: Arc<WebTraceStore>,
|
||||
web_runtime_rx: watch::Receiver<WebRuntimePublication>,
|
||||
) {
|
||||
let active_runtime = loop {
|
||||
if let Some(active_runtime) = active_runtime_rx.borrow().clone() {
|
||||
@@ -351,6 +366,7 @@ pub async fn serve(
|
||||
reload_control,
|
||||
active_runtime,
|
||||
web_trace,
|
||||
web_runtime_rx,
|
||||
});
|
||||
|
||||
spawn_runtime_watchers(
|
||||
@@ -498,9 +514,30 @@ async fn handle(
|
||||
let body_limit = api_cfg.request_body_limit_bytes;
|
||||
|
||||
let result: Result<Response<Full<Bytes>>, ApiFailure> = async {
|
||||
if web_runtime::is_route(normalized_path) {
|
||||
let web_mutation = method == Method::POST;
|
||||
let result = web_runtime::handle(
|
||||
method,
|
||||
normalized_path,
|
||||
query.as_deref(),
|
||||
req,
|
||||
shared.as_ref(),
|
||||
cfg.as_ref(),
|
||||
request_id,
|
||||
body_limit,
|
||||
)
|
||||
.await;
|
||||
if web_mutation && let Err(error) = &result {
|
||||
shared.runtime_events.record(
|
||||
"api.web.control.failed",
|
||||
format!("path={} code={}", normalized_path, error.code),
|
||||
);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
match (method.as_str(), normalized_path) {
|
||||
("GET", "/web-status") => {
|
||||
Ok(web_status::render(query.as_deref(), &shared.web_trace, &cfg.web.debug).await)
|
||||
Ok(web_status::render(query.as_deref(), &shared.web_trace).await)
|
||||
}
|
||||
("GET", "/v1/health") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
|
||||
@@ -0,0 +1,444 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use http_body_util::Full;
|
||||
use hyper::body::{Bytes, Incoming};
|
||||
use hyper::header::CONTENT_TYPE;
|
||||
use hyper::{Method, Request, Response, StatusCode};
|
||||
use serde::Serialize;
|
||||
|
||||
use super::config_store::current_revision;
|
||||
use super::http_utils::{read_json, success_response};
|
||||
use super::model::ApiFailure;
|
||||
use super::{ALLOW_GET, ALLOW_POST, ApiShared};
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::web::control::{WebRuntimeLifecycle, WebRuntimePublication};
|
||||
use crate::web::manager::{ControlError, SessionDetail, WebProcessRuntime};
|
||||
|
||||
// Exact JSON DTOs and strict query parsing stay independent from route dispatch.
|
||||
mod request;
|
||||
use request::{
|
||||
CloseRequest, RuntimeInstanceRequest, parse_session_query, parse_session_ref,
|
||||
valid_runtime_instance,
|
||||
};
|
||||
|
||||
const STATUS_PATH: &str = "/v1/runtime/web/status";
|
||||
const SESSIONS_PATH: &str = "/v1/runtime/web/sessions";
|
||||
const CLOSE_PATH: &str = "/v1/runtime/web/sessions/close";
|
||||
const DEBUG_CLEAR_PATH: &str = "/v1/runtime/web/debug/clear";
|
||||
const LEARNING_RESET_PATH: &str = "/v1/runtime/web/carrier-learning/reset";
|
||||
const SESSION_DETAIL_PREFIX: &str = "/v1/runtime/web/sessions/";
|
||||
const OPERATION_PREFIX: &str = "/v1/runtime/web/operations/";
|
||||
const MAX_CONTROL_BODY_BYTES: usize = 64 * 1024;
|
||||
|
||||
/// Returns the exact allowed method set for a WEB runtime route.
|
||||
pub(super) fn allowed_methods(path: &str) -> Option<&'static str> {
|
||||
match path {
|
||||
STATUS_PATH | SESSIONS_PATH => Some(ALLOW_GET),
|
||||
CLOSE_PATH | DEBUG_CLEAR_PATH | LEARNING_RESET_PATH => Some(ALLOW_POST),
|
||||
_ if detail_ref(path).is_some() || operation_ref(path).is_some() => Some(ALLOW_GET),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether the normalized API path belongs to WEB runtime control.
|
||||
pub(super) fn is_route(path: &str) -> bool {
|
||||
allowed_methods(path).is_some()
|
||||
}
|
||||
|
||||
/// Dispatches one authenticated WEB runtime status or control request.
|
||||
pub(super) async fn handle(
|
||||
method: Method,
|
||||
path: &str,
|
||||
query: Option<&str>,
|
||||
request: Request<Incoming>,
|
||||
shared: &ApiShared,
|
||||
config: &ProxyConfig,
|
||||
_request_id: u64,
|
||||
body_limit: usize,
|
||||
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
match (method.as_str(), path) {
|
||||
("GET", STATUS_PATH) => {
|
||||
reject_query(query)?;
|
||||
let publication = shared.web_runtime_rx.borrow().clone();
|
||||
let runtime = publication.runtime.upgrade();
|
||||
let data = WebStatusData::new(publication, runtime.as_deref(), config.web.enabled);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", SESSIONS_PATH) => {
|
||||
let runtime = readable_runtime(shared)?;
|
||||
let request = parse_session_query(&runtime, query)?;
|
||||
let page = runtime.list_sessions(request);
|
||||
Ok(success_response(StatusCode::OK, page, revision))
|
||||
}
|
||||
("GET", _) if detail_ref(path).is_some() => {
|
||||
reject_query(query)?;
|
||||
let runtime = readable_runtime(shared)?;
|
||||
let session_ref = detail_ref(path).expect("route guard checked detail reference");
|
||||
let trace_session_id = parse_session_ref(&runtime, session_ref)?;
|
||||
match runtime.session_detail(trace_session_id) {
|
||||
SessionDetail::Active(row) => Ok(success_response(StatusCode::OK, row, revision)),
|
||||
SessionDetail::Gone { attempt } => Ok(success_response(
|
||||
StatusCode::GONE,
|
||||
GoneSessionData {
|
||||
session_ref: session_ref.to_string(),
|
||||
state: "closed",
|
||||
attempt,
|
||||
},
|
||||
revision,
|
||||
)),
|
||||
SessionDetail::Busy => Err(snapshot_busy()),
|
||||
SessionDetail::NotFound => Err(ApiFailure::new(
|
||||
StatusCode::NOT_FOUND,
|
||||
"web_session_not_found",
|
||||
"WEB session was not found",
|
||||
)),
|
||||
}
|
||||
}
|
||||
("GET", _) if operation_ref(path).is_some() => {
|
||||
reject_query(query)?;
|
||||
let runtime = readable_runtime(shared)?;
|
||||
let operation_id = operation_ref(path).expect("route guard checked operation id");
|
||||
let status = runtime
|
||||
.control_operation(operation_id)
|
||||
.map_err(control_failure)?;
|
||||
Ok(success_response(StatusCode::OK, status, revision))
|
||||
}
|
||||
("POST", CLOSE_PATH) => {
|
||||
require_mutable(config)?;
|
||||
reject_query(query)?;
|
||||
require_json_content_type(&request)?;
|
||||
let request = read_json::<CloseRequest>(
|
||||
request.into_body(),
|
||||
body_limit.min(MAX_CONTROL_BODY_BYTES),
|
||||
)
|
||||
.await?;
|
||||
let runtime = control_runtime(shared)?;
|
||||
require_runtime_instance(&runtime, &request.runtime_instance)?;
|
||||
let selector = request.selector.resolve(&runtime)?;
|
||||
let status = runtime
|
||||
.start_close_operation(&request.runtime_instance, selector)
|
||||
.map_err(control_failure)?;
|
||||
shared.runtime_events.record(
|
||||
"api.web.sessions.close.accepted",
|
||||
format!(
|
||||
"operation_id={} requested={}",
|
||||
status.operation_id, status.requested
|
||||
),
|
||||
);
|
||||
Ok(success_response(StatusCode::ACCEPTED, status, revision))
|
||||
}
|
||||
("POST", DEBUG_CLEAR_PATH) => {
|
||||
require_mutable(config)?;
|
||||
reject_query(query)?;
|
||||
require_json_content_type(&request)?;
|
||||
let request = read_json::<RuntimeInstanceRequest>(
|
||||
request.into_body(),
|
||||
body_limit.min(MAX_CONTROL_BODY_BYTES),
|
||||
)
|
||||
.await?;
|
||||
let runtime = control_runtime(shared)?;
|
||||
require_runtime_instance(&runtime, &request.runtime_instance)?;
|
||||
let outcome = runtime.clear_debug().map_err(control_failure)?;
|
||||
let data = DebugClearData {
|
||||
runtime_instance: runtime.runtime_instance().to_string(),
|
||||
records_cleared: outcome.records_cleared,
|
||||
leased_bytes: outcome.leased_bytes,
|
||||
epoch: outcome.epoch,
|
||||
};
|
||||
shared.runtime_events.record(
|
||||
"api.web.debug.clear.ok",
|
||||
format!("records={} epoch={}", data.records_cleared, data.epoch),
|
||||
);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("POST", LEARNING_RESET_PATH) => {
|
||||
require_mutable(config)?;
|
||||
reject_query(query)?;
|
||||
require_json_content_type(&request)?;
|
||||
let request = read_json::<RuntimeInstanceRequest>(
|
||||
request.into_body(),
|
||||
body_limit.min(MAX_CONTROL_BODY_BYTES),
|
||||
)
|
||||
.await?;
|
||||
let runtime = control_runtime(shared)?;
|
||||
require_runtime_instance(&runtime, &request.runtime_instance)?;
|
||||
let outcome = runtime
|
||||
.reset_carrier_learning()
|
||||
.map_err(|_| runtime_unavailable(WebRuntimeLifecycle::Draining))?;
|
||||
let data = LearningResetData {
|
||||
runtime_instance: runtime.runtime_instance().to_string(),
|
||||
entries_cleared: outcome.entries_cleared,
|
||||
epoch: outcome.epoch,
|
||||
};
|
||||
shared.runtime_events.record(
|
||||
"api.web.carrier_learning.reset.ok",
|
||||
format!("entries={} epoch={}", data.entries_cleared, data.epoch),
|
||||
);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
_ => Err(ApiFailure::method_not_allowed(
|
||||
allowed_methods(path).unwrap_or(ALLOW_GET),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct WebStatusData {
|
||||
lifecycle: &'static str,
|
||||
lifecycle_epoch: u64,
|
||||
lifecycle_age_ms: u64,
|
||||
available: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
reason: Option<&'static str>,
|
||||
listeners: Vec<String>,
|
||||
effective_config_enabled: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
runtime: Option<crate::web::manager::WebRuntimeStatus>,
|
||||
}
|
||||
|
||||
impl WebStatusData {
|
||||
fn new(
|
||||
publication: WebRuntimePublication,
|
||||
runtime: Option<&WebProcessRuntime>,
|
||||
effective_config_enabled: bool,
|
||||
) -> Self {
|
||||
let available = runtime.is_some()
|
||||
&& matches!(
|
||||
publication.lifecycle,
|
||||
WebRuntimeLifecycle::Running | WebRuntimeLifecycle::Draining
|
||||
);
|
||||
let reason = if available {
|
||||
None
|
||||
} else {
|
||||
Some(match publication.lifecycle {
|
||||
WebRuntimeLifecycle::Starting => "starting",
|
||||
WebRuntimeLifecycle::NoWebListener => "no_web_listener",
|
||||
WebRuntimeLifecycle::Running => "runtime_released",
|
||||
WebRuntimeLifecycle::Draining => "runtime_released",
|
||||
WebRuntimeLifecycle::Drained => "drained",
|
||||
WebRuntimeLifecycle::DeadlineExceeded => "deadline_exceeded",
|
||||
})
|
||||
};
|
||||
Self {
|
||||
lifecycle: publication.lifecycle.as_str(),
|
||||
lifecycle_epoch: publication.epoch,
|
||||
lifecycle_age_ms: millis(Instant::now().saturating_duration_since(publication.since)),
|
||||
available,
|
||||
reason,
|
||||
listeners: publication
|
||||
.listeners
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect(),
|
||||
effective_config_enabled,
|
||||
runtime: runtime.map(WebProcessRuntime::try_status),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct GoneSessionData {
|
||||
session_ref: String,
|
||||
state: &'static str,
|
||||
attempt: u8,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct DebugClearData {
|
||||
runtime_instance: String,
|
||||
records_cleared: usize,
|
||||
leased_bytes: usize,
|
||||
epoch: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct LearningResetData {
|
||||
runtime_instance: String,
|
||||
entries_cleared: usize,
|
||||
epoch: u64,
|
||||
}
|
||||
|
||||
fn readable_runtime(shared: &ApiShared) -> Result<Arc<WebProcessRuntime>, ApiFailure> {
|
||||
let publication = shared.web_runtime_rx.borrow().clone();
|
||||
if !matches!(
|
||||
publication.lifecycle,
|
||||
WebRuntimeLifecycle::Running | WebRuntimeLifecycle::Draining
|
||||
) {
|
||||
return Err(runtime_unavailable(publication.lifecycle));
|
||||
}
|
||||
publication
|
||||
.runtime
|
||||
.upgrade()
|
||||
.ok_or_else(|| runtime_unavailable(publication.lifecycle))
|
||||
}
|
||||
|
||||
fn control_runtime(shared: &ApiShared) -> Result<Arc<WebProcessRuntime>, ApiFailure> {
|
||||
let publication = shared.web_runtime_rx.borrow().clone();
|
||||
if publication.lifecycle != WebRuntimeLifecycle::Running {
|
||||
return Err(runtime_unavailable(publication.lifecycle));
|
||||
}
|
||||
publication
|
||||
.runtime
|
||||
.upgrade()
|
||||
.ok_or_else(|| runtime_unavailable(publication.lifecycle))
|
||||
}
|
||||
|
||||
fn runtime_unavailable(lifecycle: WebRuntimeLifecycle) -> ApiFailure {
|
||||
ApiFailure::new(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"web_runtime_unavailable",
|
||||
format!("WEB runtime is unavailable: {}", lifecycle.as_str()),
|
||||
)
|
||||
}
|
||||
|
||||
fn require_mutable(config: &ProxyConfig) -> Result<(), ApiFailure> {
|
||||
if config.server.api.read_only {
|
||||
return Err(ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_json_content_type<B>(request: &Request<B>) -> Result<(), ApiFailure> {
|
||||
let mut values = request.headers().get_all(CONTENT_TYPE).iter();
|
||||
let exact = values
|
||||
.next()
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.is_some_and(|value| value == "application/json")
|
||||
&& values.next().is_none();
|
||||
if !exact {
|
||||
return Err(ApiFailure::new(
|
||||
StatusCode::UNSUPPORTED_MEDIA_TYPE,
|
||||
"unsupported_media_type",
|
||||
"Content-Type must be exactly application/json",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_runtime_instance(
|
||||
runtime: &WebProcessRuntime,
|
||||
runtime_instance: &str,
|
||||
) -> Result<(), ApiFailure> {
|
||||
if !valid_runtime_instance(runtime_instance) {
|
||||
return Err(ApiFailure::bad_request(
|
||||
"runtime_instance must be 32 lowercase hexadecimal characters",
|
||||
));
|
||||
}
|
||||
if runtime.runtime_instance() != runtime_instance {
|
||||
return Err(ApiFailure::new(
|
||||
StatusCode::CONFLICT,
|
||||
"web_runtime_mismatch",
|
||||
"WEB runtime instance no longer matches",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn control_failure(error: ControlError) -> ApiFailure {
|
||||
match error {
|
||||
ControlError::StaleInstance => ApiFailure::new(
|
||||
StatusCode::CONFLICT,
|
||||
"web_runtime_mismatch",
|
||||
"WEB runtime instance no longer matches",
|
||||
),
|
||||
ControlError::InvalidSelector | ControlError::InvalidOperation => {
|
||||
ApiFailure::bad_request("Invalid WEB control request")
|
||||
}
|
||||
ControlError::IssuanceEnabled => ApiFailure::new(
|
||||
StatusCode::CONFLICT,
|
||||
"web_issuance_enabled",
|
||||
"Close-all requires effective WEB issuance to be disabled",
|
||||
),
|
||||
ControlError::OperationInProgress => ApiFailure::new(
|
||||
StatusCode::CONFLICT,
|
||||
"web_operation_in_progress",
|
||||
"Another WEB close operation is active",
|
||||
),
|
||||
ControlError::OperationNotFound => ApiFailure::new(
|
||||
StatusCode::NOT_FOUND,
|
||||
"web_operation_not_found",
|
||||
"WEB control operation was not found",
|
||||
),
|
||||
ControlError::Closed => runtime_unavailable(WebRuntimeLifecycle::Draining),
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot_busy() -> ApiFailure {
|
||||
ApiFailure::new(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"web_snapshot_busy",
|
||||
"WEB runtime snapshot is temporarily busy",
|
||||
)
|
||||
}
|
||||
|
||||
fn reject_query(query: Option<&str>) -> Result<(), ApiFailure> {
|
||||
if query.is_some_and(|query| !query.is_empty()) {
|
||||
return Err(ApiFailure::bad_request(
|
||||
"This endpoint does not accept query parameters",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn detail_ref(path: &str) -> Option<&str> {
|
||||
path.strip_prefix(SESSION_DETAIL_PREFIX)
|
||||
.filter(|value| !value.is_empty() && !value.contains('/') && *value != "close")
|
||||
}
|
||||
|
||||
fn operation_ref(path: &str) -> Option<&str> {
|
||||
path.strip_prefix(OPERATION_PREFIX)
|
||||
.filter(|value| !value.is_empty() && !value.contains('/'))
|
||||
}
|
||||
|
||||
fn millis(duration: std::time::Duration) -> u64 {
|
||||
duration.as_millis().min(u128::from(u64::MAX)) as u64
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use hyper::header::HeaderValue;
|
||||
|
||||
#[test]
|
||||
fn route_table_keeps_status_read_only_and_controls_post_only() {
|
||||
assert_eq!(allowed_methods(STATUS_PATH), Some(ALLOW_GET));
|
||||
assert_eq!(allowed_methods(SESSIONS_PATH), Some(ALLOW_GET));
|
||||
assert_eq!(allowed_methods(CLOSE_PATH), Some(ALLOW_POST));
|
||||
assert_eq!(allowed_methods(DEBUG_CLEAR_PATH), Some(ALLOW_POST));
|
||||
assert_eq!(allowed_methods(LEARNING_RESET_PATH), Some(ALLOW_POST));
|
||||
assert_eq!(
|
||||
allowed_methods("/v1/runtime/web/sessions/ws1.instance.0000000000000001"),
|
||||
Some(ALLOW_GET)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn control_content_type_is_exact_and_single() {
|
||||
let exact = Request::builder()
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(require_json_content_type(&exact).is_ok());
|
||||
|
||||
let parameterized = Request::builder()
|
||||
.header(CONTENT_TYPE, "application/json; charset=utf-8")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(require_json_content_type(¶meterized).is_err());
|
||||
|
||||
let mut duplicated = Request::builder()
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(())
|
||||
.unwrap();
|
||||
duplicated
|
||||
.headers_mut()
|
||||
.append(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
assert!(require_json_content_type(&duplicated).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,391 @@
|
||||
use std::collections::BTreeSet;
|
||||
use std::net::IpAddr;
|
||||
|
||||
use hyper::StatusCode;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::config::WebCarrier;
|
||||
use crate::web::manager::{
|
||||
CloseOperationSelector, SessionFilter, SessionListRequest, SessionRefError, WebProcessRuntime,
|
||||
};
|
||||
|
||||
use super::super::model::ApiFailure;
|
||||
|
||||
const DEFAULT_SESSION_LIMIT: usize = 50;
|
||||
const MAX_SESSION_LIMIT: usize = 200;
|
||||
|
||||
/// Exact process-instance fence for one runtime mutation.
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(super) struct RuntimeInstanceRequest {
|
||||
/// Random process identifier copied from WEB runtime status.
|
||||
pub(super) runtime_instance: String,
|
||||
}
|
||||
|
||||
/// One process-fenced asynchronous close request.
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(super) struct CloseRequest {
|
||||
/// Random process identifier copied from WEB runtime status.
|
||||
pub(super) runtime_instance: String,
|
||||
/// Exact point-in-time close selector.
|
||||
pub(super) selector: CloseSelectorRequest,
|
||||
}
|
||||
|
||||
/// Strict tagged selector accepted by the WEB close endpoint.
|
||||
#[derive(Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub(super) enum CloseSelectorRequest {
|
||||
/// Closes an explicit bounded set of logical sessions.
|
||||
Refs {
|
||||
/// Unique current-instance opaque session references.
|
||||
session_refs: Vec<String>,
|
||||
},
|
||||
/// Closes the point-in-time sessions matching every supplied field.
|
||||
Filter {
|
||||
#[serde(default)]
|
||||
session_ref: Option<String>,
|
||||
#[serde(default)]
|
||||
ip: Option<String>,
|
||||
#[serde(default)]
|
||||
host: Option<String>,
|
||||
#[serde(default)]
|
||||
user: Option<String>,
|
||||
#[serde(default)]
|
||||
user_agent_id: Option<String>,
|
||||
#[serde(default)]
|
||||
key_id: Option<String>,
|
||||
#[serde(default)]
|
||||
carrier: Option<WebCarrier>,
|
||||
#[serde(default)]
|
||||
state: Option<String>,
|
||||
},
|
||||
/// Closes every point-in-time session below the submission high-water mark.
|
||||
All {},
|
||||
}
|
||||
|
||||
impl CloseSelectorRequest {
|
||||
/// Validates external identifiers and resolves them to manager-owned values.
|
||||
pub(super) fn resolve(
|
||||
self,
|
||||
runtime: &WebProcessRuntime,
|
||||
) -> Result<CloseOperationSelector, ApiFailure> {
|
||||
match self {
|
||||
Self::Refs { session_refs } => resolve_refs(runtime, session_refs),
|
||||
Self::Filter {
|
||||
session_ref,
|
||||
ip,
|
||||
host,
|
||||
user,
|
||||
user_agent_id,
|
||||
key_id,
|
||||
carrier,
|
||||
state,
|
||||
} => {
|
||||
validate_filter_strings(
|
||||
host.as_deref(),
|
||||
user.as_deref(),
|
||||
key_id.as_deref(),
|
||||
state.as_deref(),
|
||||
)?;
|
||||
let trace_session_id = session_ref
|
||||
.as_deref()
|
||||
.map(|value| parse_session_ref(runtime, value))
|
||||
.transpose()?;
|
||||
let client_ip = ip.as_deref().map(parse_canonical_ip).transpose()?;
|
||||
let filter = SessionFilter {
|
||||
trace_session_id,
|
||||
client_ip,
|
||||
host,
|
||||
user,
|
||||
user_agent_id: user_agent_id
|
||||
.as_deref()
|
||||
.map(parse_user_agent_id)
|
||||
.transpose()?,
|
||||
key_id,
|
||||
carrier,
|
||||
state,
|
||||
};
|
||||
if filter.is_empty() {
|
||||
return Err(ApiFailure::bad_request(
|
||||
"filter selector requires at least one filter",
|
||||
));
|
||||
}
|
||||
Ok(CloseOperationSelector::Filter(filter))
|
||||
}
|
||||
Self::All {} => Ok(CloseOperationSelector::All),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_refs(
|
||||
runtime: &WebProcessRuntime,
|
||||
session_refs: Vec<String>,
|
||||
) -> Result<CloseOperationSelector, ApiFailure> {
|
||||
if session_refs.is_empty() || session_refs.len() > 200 {
|
||||
return Err(ApiFailure::bad_request(
|
||||
"session_refs must contain 1..200 references",
|
||||
));
|
||||
}
|
||||
let mut resolved = Vec::with_capacity(session_refs.len());
|
||||
let mut unique = BTreeSet::new();
|
||||
for session_ref in session_refs {
|
||||
let id = parse_session_ref(runtime, &session_ref)?;
|
||||
if !unique.insert(id) {
|
||||
return Err(ApiFailure::bad_request(
|
||||
"session_refs must not contain duplicates",
|
||||
));
|
||||
}
|
||||
resolved.push(id);
|
||||
}
|
||||
Ok(CloseOperationSelector::Refs(resolved))
|
||||
}
|
||||
|
||||
/// Parses one duplicate-free bounded session-list query.
|
||||
pub(super) fn parse_session_query(
|
||||
runtime: &WebProcessRuntime,
|
||||
raw: Option<&str>,
|
||||
) -> Result<SessionListRequest, ApiFailure> {
|
||||
let mut limit = DEFAULT_SESSION_LIMIT;
|
||||
let mut cursor = None;
|
||||
let mut filter = SessionFilter::default();
|
||||
let mut seen = BTreeSet::new();
|
||||
for (name, value) in url::form_urlencoded::parse(raw.unwrap_or_default().as_bytes()) {
|
||||
if !seen.insert(name.to_string()) {
|
||||
return Err(ApiFailure::bad_request(format!("{} must not repeat", name)));
|
||||
}
|
||||
match name.as_ref() {
|
||||
"limit" => {
|
||||
limit = value
|
||||
.parse::<usize>()
|
||||
.ok()
|
||||
.filter(|value| (1..=MAX_SESSION_LIMIT).contains(value))
|
||||
.ok_or_else(|| ApiFailure::bad_request("limit must be within 1..200"))?;
|
||||
}
|
||||
"cursor" => cursor = Some(parse_session_ref(runtime, &value)?),
|
||||
"session_ref" => {
|
||||
let id = parse_session_ref(runtime, &value)?;
|
||||
filter.trace_session_id = Some(id);
|
||||
cursor = id.checked_sub(1);
|
||||
limit = 1;
|
||||
}
|
||||
"ip" => {
|
||||
filter.client_ip = Some(parse_canonical_ip(&value)?);
|
||||
}
|
||||
"host" => filter.host = Some(value.into_owned()),
|
||||
"user" => filter.user = Some(value.into_owned()),
|
||||
"user_agent_id" => filter.user_agent_id = Some(parse_user_agent_id(&value)?),
|
||||
"key_id" => filter.key_id = Some(value.into_owned()),
|
||||
"carrier" => filter.carrier = Some(parse_carrier(&value)?),
|
||||
"state" => filter.state = Some(value.into_owned()),
|
||||
_ => {
|
||||
return Err(ApiFailure::bad_request(format!(
|
||||
"unknown query field `{}`",
|
||||
name
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
if filter.trace_session_id.is_some() && (seen.contains("cursor") || seen.contains("limit")) {
|
||||
return Err(ApiFailure::bad_request(
|
||||
"session_ref must not be combined with cursor or limit",
|
||||
));
|
||||
}
|
||||
validate_filter_strings(
|
||||
filter.host.as_deref(),
|
||||
filter.user.as_deref(),
|
||||
filter.key_id.as_deref(),
|
||||
filter.state.as_deref(),
|
||||
)?;
|
||||
Ok(SessionListRequest {
|
||||
limit,
|
||||
cursor,
|
||||
filter,
|
||||
})
|
||||
}
|
||||
|
||||
/// Maps one opaque session-reference failure to the stable API error contract.
|
||||
pub(super) fn parse_session_ref(
|
||||
runtime: &WebProcessRuntime,
|
||||
session_ref: &str,
|
||||
) -> Result<u64, ApiFailure> {
|
||||
runtime
|
||||
.parse_session_ref(session_ref)
|
||||
.map_err(|error| match error {
|
||||
SessionRefError::Invalid => ApiFailure::bad_request("Invalid WEB session reference"),
|
||||
SessionRefError::StaleInstance => ApiFailure::new(
|
||||
StatusCode::CONFLICT,
|
||||
"web_runtime_mismatch",
|
||||
"WEB session reference belongs to another runtime instance",
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_carrier(value: &str) -> Result<WebCarrier, ApiFailure> {
|
||||
WebCarrier::ALL
|
||||
.into_iter()
|
||||
.find(|carrier| carrier.as_str() == value)
|
||||
.ok_or_else(|| {
|
||||
ApiFailure::bad_request(
|
||||
"carrier must be https, https-lanes, websocket, or websocket-lanes",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_filter_strings(
|
||||
host: Option<&str>,
|
||||
user: Option<&str>,
|
||||
key_id: Option<&str>,
|
||||
state: Option<&str>,
|
||||
) -> Result<(), ApiFailure> {
|
||||
if host.is_some_and(|value| value.is_empty() || value.len() > 253) {
|
||||
return Err(ApiFailure::bad_request("host must contain 1..253 bytes"));
|
||||
}
|
||||
if user.is_some_and(|value| value.is_empty() || value.len() > 64) {
|
||||
return Err(ApiFailure::bad_request("user must contain 1..64 bytes"));
|
||||
}
|
||||
if key_id.is_some_and(|value| !lower_hex(value, 16)) {
|
||||
return Err(ApiFailure::bad_request(
|
||||
"key_id must be 16 lowercase hexadecimal characters",
|
||||
));
|
||||
}
|
||||
if state.is_some_and(|value| {
|
||||
!matches!(
|
||||
value,
|
||||
"provisional"
|
||||
| "replacing"
|
||||
| "committed"
|
||||
| "healthy"
|
||||
| "closing"
|
||||
| "superseded"
|
||||
| "closed"
|
||||
)
|
||||
}) {
|
||||
return Err(ApiFailure::bad_request("Invalid WEB session state"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_user_agent_id(value: &str) -> Result<[u8; 16], ApiFailure> {
|
||||
if !lower_hex(value, 32) {
|
||||
return Err(ApiFailure::bad_request(
|
||||
"user_agent_id must be 32 lowercase hexadecimal characters",
|
||||
));
|
||||
}
|
||||
let mut id = [0; 16];
|
||||
hex::decode_to_slice(value, &mut id).map_err(|_| {
|
||||
ApiFailure::bad_request("user_agent_id must be 32 lowercase hexadecimal characters")
|
||||
})?;
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
fn lower_hex(value: &str, length: usize) -> bool {
|
||||
value.len() == length
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
}
|
||||
|
||||
/// Returns whether a process instance uses its canonical lowercase form.
|
||||
pub(super) fn valid_runtime_instance(value: &str) -> bool {
|
||||
lower_hex(value, 32)
|
||||
}
|
||||
|
||||
fn parse_canonical_ip(value: &str) -> Result<IpAddr, ApiFailure> {
|
||||
let ip = value
|
||||
.parse::<IpAddr>()
|
||||
.map_err(|_| ApiFailure::bad_request("ip must be a canonical IP address"))?;
|
||||
if ip.to_string() != value {
|
||||
return Err(ApiFailure::bad_request("ip must use canonical formatting"));
|
||||
}
|
||||
Ok(ip)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn filter_identifiers_are_exact_lowercase_hex() {
|
||||
assert!(lower_hex("0123456789abcdef", 16));
|
||||
assert!(!lower_hex("0123456789ABCDEF", 16));
|
||||
assert!(!lower_hex("0123", 16));
|
||||
assert!(valid_runtime_instance("0123456789abcdef0123456789abcdef"));
|
||||
assert!(!valid_runtime_instance("0123456789ABCDEF0123456789ABCDEF"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mutation_dtos_reject_unknown_fields() {
|
||||
let runtime_instance = "0123456789abcdef0123456789abcdef";
|
||||
assert!(
|
||||
serde_json::from_value::<RuntimeInstanceRequest>(serde_json::json!({
|
||||
"runtime_instance": runtime_instance,
|
||||
"extra": true,
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
serde_json::from_value::<CloseRequest>(serde_json::json!({
|
||||
"runtime_instance": runtime_instance,
|
||||
"selector": {"kind": "all", "extra": true},
|
||||
}))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_ips_require_canonical_text() {
|
||||
assert!(parse_canonical_ip("2001:db8::1").is_ok());
|
||||
assert!(parse_canonical_ip("2001:0db8::1").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn state_filter_accepts_every_emitted_session_state() {
|
||||
for state in [
|
||||
"provisional",
|
||||
"replacing",
|
||||
"committed",
|
||||
"healthy",
|
||||
"closing",
|
||||
"superseded",
|
||||
"closed",
|
||||
] {
|
||||
assert!(
|
||||
validate_filter_strings(None, None, None, Some(state)).is_ok(),
|
||||
"state {state} must be accepted"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exact_session_query_rejects_pagination_fields_in_any_order() {
|
||||
let generation = crate::maestro::generation::test_runtime_generation(
|
||||
1,
|
||||
crate::config::ProxyConfig::default(),
|
||||
);
|
||||
let runtime = WebProcessRuntime::start(std::sync::Arc::new(arc_swap::ArcSwap::from(
|
||||
generation.clone(),
|
||||
)));
|
||||
let session_ref = runtime.session_ref(1);
|
||||
let cursor = runtime.session_ref(2);
|
||||
|
||||
assert!(
|
||||
parse_session_query(
|
||||
&runtime,
|
||||
Some(&format!("session_ref={session_ref}&cursor={cursor}")),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
parse_session_query(
|
||||
&runtime,
|
||||
Some(&format!("limit=2&session_ref={session_ref}")),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
@@ -41,10 +41,9 @@ impl AsRef<[u8]> for RenderedPage {
|
||||
pub(super) async fn render(
|
||||
raw_query: Option<&str>,
|
||||
store: &Arc<WebTraceStore>,
|
||||
policy: &WebDebugConfig,
|
||||
) -> Response<Full<Bytes>> {
|
||||
store.apply_policy(policy);
|
||||
let query = match parse_query(raw_query, policy) {
|
||||
let status = store.status();
|
||||
let query = match parse_query(raw_query, &status.policy) {
|
||||
Ok(query) => query,
|
||||
Err(error) => return html_error(StatusCode::BAD_REQUEST, "Invalid query", &error),
|
||||
};
|
||||
@@ -62,7 +61,6 @@ pub(super) async fn render(
|
||||
0
|
||||
};
|
||||
let records = store.snapshot_matching(|record| record_matches(record, &query, since_millis));
|
||||
let status = store.status();
|
||||
let mut html = String::with_capacity(MAX_PAGE_BYTES);
|
||||
push_page_start(&mut html);
|
||||
html.push_str("<h1>WEB status</h1>");
|
||||
|
||||
@@ -46,7 +46,6 @@ async fn renderer_filters_groups_and_sets_control_plane_security_headers() {
|
||||
let response = render(
|
||||
Some("ip=192.0.2.40&session=42&key=0123456789abcdef&group_by=ip&group_by=key"),
|
||||
&store,
|
||||
&policy,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
@@ -69,18 +68,36 @@ async fn render_permits_remain_owned_by_inflight_response_bodies() {
|
||||
let limits = crate::config::WebLimitsConfig::default();
|
||||
let store = WebTraceStore::new(policy.clone(), &limits);
|
||||
|
||||
let first = render(None, &store, &policy).await;
|
||||
let second = render(None, &store, &policy).await;
|
||||
let busy = render(None, &store, &policy).await;
|
||||
let first = render(None, &store).await;
|
||||
let second = render(None, &store).await;
|
||||
let busy = render(None, &store).await;
|
||||
assert_eq!(busy.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
|
||||
drop(first);
|
||||
let admitted = render(None, &store, &policy).await;
|
||||
let admitted = render(None, &store).await;
|
||||
assert_eq!(admitted.status(), StatusCode::OK);
|
||||
drop(second);
|
||||
drop(admitted);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_renderer_cannot_restore_an_old_debug_policy() {
|
||||
let stale_policy = WebDebugConfig::default();
|
||||
let active_policy = WebDebugConfig {
|
||||
enabled: true,
|
||||
capture_headers: false,
|
||||
..Default::default()
|
||||
};
|
||||
let limits = crate::config::WebLimitsConfig::default();
|
||||
let store = WebTraceStore::new(stale_policy.clone(), &limits);
|
||||
store.apply_policy(2, &active_policy);
|
||||
|
||||
let response = render(None, &store).await;
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
assert_eq!(store.status().policy.as_ref(), &active_policy);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_truncation_preserves_utf8_boundary_and_cap() {
|
||||
let mut html = "\u{044f}".repeat(MAX_PAGE_BYTES);
|
||||
|
||||
@@ -256,6 +256,46 @@ fn reload_applies_hot_change_on_first_observed_snapshot() {
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn candidate_watcher_waits_for_activation_and_reconciles_disk() {
|
||||
let initial_tag = "10101010101010101010101010101010";
|
||||
let disk_tag = "20202020202020202020202020202020";
|
||||
let path = temp_config_path("telemt_hot_reload_activation_gate");
|
||||
write_reload_config(&path, Some(initial_tag), None);
|
||||
let initial = Arc::new(ProxyConfig::load(&path).unwrap());
|
||||
write_reload_config(&path, Some(disk_tag), None);
|
||||
let cancellation = tokio_util::sync::CancellationToken::new();
|
||||
let (activation_tx, activation_rx) = watch::channel(false);
|
||||
let (mut config_rx, _log_rx, watcher) = spawn_config_watcher(
|
||||
path.clone(),
|
||||
initial,
|
||||
None,
|
||||
None,
|
||||
cancellation.clone(),
|
||||
Some(activation_rx),
|
||||
);
|
||||
let watcher = tokio::spawn(watcher);
|
||||
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(
|
||||
config_rx.borrow().general.ad_tag.as_deref(),
|
||||
Some(initial_tag)
|
||||
);
|
||||
activation_tx.send_replace(true);
|
||||
tokio::time::timeout(Duration::from_secs(2), config_rx.changed())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
config_rx.borrow_and_update().general.ad_tag.as_deref(),
|
||||
Some(disk_tag)
|
||||
);
|
||||
|
||||
cancellation.cancel();
|
||||
watcher.await.unwrap();
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reload_keeps_hot_apply_when_non_hot_fields_change() {
|
||||
let initial_tag = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
|
||||
@@ -213,20 +213,47 @@ pub fn spawn_config_watcher(
|
||||
detected_ip_v4: Option<IpAddr>,
|
||||
detected_ip_v6: Option<IpAddr>,
|
||||
cancellation: tokio_util::sync::CancellationToken,
|
||||
) -> (watch::Receiver<Arc<ProxyConfig>>, watch::Receiver<LogLevel>) {
|
||||
mut activation: Option<watch::Receiver<bool>>,
|
||||
) -> (
|
||||
watch::Receiver<Arc<ProxyConfig>>,
|
||||
watch::Receiver<LogLevel>,
|
||||
impl std::future::Future<Output = ()> + Send + 'static,
|
||||
) {
|
||||
let initial_level = initial.general.log_level.clone();
|
||||
let (config_tx, config_rx) = watch::channel(initial);
|
||||
let (log_tx, log_rx) = watch::channel(initial_level);
|
||||
|
||||
let config_path = normalize_watch_path(&config_path);
|
||||
let initial_loaded = ProxyConfig::load_with_metadata(&config_path).ok();
|
||||
let initial_manifest = initial_loaded
|
||||
.as_ref()
|
||||
.map(|loaded| WatchManifest::from_source_files(&loaded.source_files))
|
||||
.unwrap_or_else(|| WatchManifest::from_source_files(std::slice::from_ref(&config_path)));
|
||||
let initial_snapshot_hash = initial_loaded.as_ref().map(|loaded| loaded.rendered_hash);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let task = async move {
|
||||
if let Some(activation) = activation.as_mut() {
|
||||
loop {
|
||||
if *activation.borrow_and_update() {
|
||||
break;
|
||||
}
|
||||
tokio::select! {
|
||||
result = activation.changed() => {
|
||||
if result.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
_ = cancellation.cancelled() => return,
|
||||
}
|
||||
}
|
||||
}
|
||||
let initial_loaded = ProxyConfig::load_with_metadata(&config_path).ok();
|
||||
let initial_manifest = initial_loaded
|
||||
.as_ref()
|
||||
.map(|loaded| WatchManifest::from_source_files(&loaded.source_files))
|
||||
.unwrap_or_else(|| {
|
||||
WatchManifest::from_source_files(std::slice::from_ref(&config_path))
|
||||
});
|
||||
let initial_matches_disk = initial_loaded
|
||||
.as_ref()
|
||||
.is_some_and(|loaded| config_equal(config_tx.borrow().as_ref(), &loaded.config));
|
||||
let initial_snapshot_hash = initial_loaded
|
||||
.as_ref()
|
||||
.filter(|_| initial_matches_disk)
|
||||
.map(|loaded| loaded.rendered_hash);
|
||||
let (notify_tx, mut notify_rx) = mpsc::channel::<()>(4);
|
||||
let manifest_state = Arc::new(StdRwLock::new(WatchManifest::default()));
|
||||
let mut reload_state = ReloadState::new(initial_snapshot_hash);
|
||||
@@ -304,6 +331,9 @@ pub fn spawn_config_watcher(
|
||||
if poll_watcher.is_some() {
|
||||
info!("config watcher: poll watcher active (Docker/NFS safe)");
|
||||
}
|
||||
if initial_loaded.is_some() && !initial_matches_disk {
|
||||
let _ = notify_tx.try_send(());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
let mut sighup = {
|
||||
@@ -364,7 +394,7 @@ pub fn spawn_config_watcher(
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
(config_rx, log_rx)
|
||||
(config_rx, log_rx, task)
|
||||
}
|
||||
|
||||
@@ -339,6 +339,9 @@ const WEB_TIMEOUTS_CONFIG_KEYS: &[&str] = &[
|
||||
"stream_handshake_secs",
|
||||
"stream_first_byte_secs",
|
||||
"long_poll_secs",
|
||||
"bridge_request_secs",
|
||||
"bridge_retry_secs",
|
||||
"carrier_probe_coalesce_ms",
|
||||
"lane_open_wait_secs",
|
||||
"carrier_health_secs",
|
||||
"websocket_upgrade_secs",
|
||||
|
||||
@@ -381,6 +381,11 @@ fn validate_vhosts(config: &mut ProxyConfig) -> Result<()> {
|
||||
validate_decoy(vhost_idx, &vhost.decoy)?;
|
||||
let mut profiles = HashSet::with_capacity(vhost.profiles.len());
|
||||
for (profile_idx, profile) in vhost.profiles.iter().enumerate() {
|
||||
if profile.user.is_empty() || profile.user.len() > 64 {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user must contain 1..64 bytes"
|
||||
));
|
||||
}
|
||||
if !config.access.users.contains_key(&profile.user) {
|
||||
return config_error(&format!(
|
||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user references unknown access user `{}`",
|
||||
|
||||
@@ -33,6 +33,18 @@ pub(super) fn validate(timeouts: &WebTimeoutsConfig) -> Result<()> {
|
||||
if !(2..=86_400).contains(&timeouts.carrier_learning_secs) {
|
||||
return config_error("web.timeouts.carrier_learning_secs must be within [2, 86400]");
|
||||
}
|
||||
if !(1..=60).contains(&timeouts.bridge_request_secs) {
|
||||
return config_error("web.timeouts.bridge_request_secs must be within [1, 60]");
|
||||
}
|
||||
if !(1..=300).contains(&timeouts.bridge_retry_secs) {
|
||||
return config_error("web.timeouts.bridge_retry_secs must be within [1, 300]");
|
||||
}
|
||||
if timeouts.bridge_request_secs > timeouts.bridge_retry_secs {
|
||||
return config_error("web.timeouts.bridge_request_secs must not exceed bridge_retry_secs");
|
||||
}
|
||||
if timeouts.carrier_probe_coalesce_ms > 10 {
|
||||
return config_error("web.timeouts.carrier_probe_coalesce_ms must be within [0, 10]");
|
||||
}
|
||||
if timeouts.stream_first_byte_secs > 300 {
|
||||
return config_error("web.timeouts.stream_first_byte_secs must be within [1, 300]");
|
||||
}
|
||||
|
||||
@@ -56,6 +56,17 @@ fn web_config_builds_canonical_runtime_snapshot() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_profile_user_labels_are_bounded_for_runtime_status() {
|
||||
let user = "a".repeat(65);
|
||||
let invalid = WEB_CONFIG.replace("alice", &user);
|
||||
|
||||
assert!(
|
||||
load_config_error_from_temp_toml(&invalid)
|
||||
.contains("web.vhosts[0].profiles[0].user must contain 1..64 bytes")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_carriers_missing_or_false_disable_negotiation() {
|
||||
let missing = load_config_from_temp_toml(WEB_CONFIG);
|
||||
@@ -108,10 +119,10 @@ fn web_carriers_reject_true_empty_and_duplicates() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_carrier_deadlines_and_learning_window_are_configurable() {
|
||||
fn web_carrier_and_bridge_deadlines_are_configurable() {
|
||||
let configured = WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
"[web.timeouts]\ncarrier_negotiation_deadlines_secs = [1, 2, 4, 9]\ncarrier_learning_secs = 30\n\n[[web.vhosts]]",
|
||||
"[web.timeouts]\ncarrier_negotiation_deadlines_secs = [1, 2, 4, 9]\ncarrier_learning_secs = 30\nbridge_request_secs = 7\nbridge_retry_secs = 41\ncarrier_probe_coalesce_ms = 4\n\n[[web.vhosts]]",
|
||||
);
|
||||
let config = load_config_from_temp_toml(&configured);
|
||||
assert_eq!(
|
||||
@@ -119,6 +130,51 @@ fn web_carrier_deadlines_and_learning_window_are_configurable() {
|
||||
[1, 2, 4, 9]
|
||||
);
|
||||
assert_eq!(config.web.timeouts.carrier_learning_secs, 30);
|
||||
assert_eq!(config.web.timeouts.bridge_request_secs, 7);
|
||||
assert_eq!(config.web.timeouts.bridge_retry_secs, 41);
|
||||
assert_eq!(config.web.timeouts.carrier_probe_coalesce_ms, 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_bridge_deadlines_are_known_in_strict_mode() {
|
||||
let configured = WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
"[web.timeouts]\nbridge_request_secs = 7\nbridge_retry_secs = 41\ncarrier_probe_coalesce_ms = 4\n\n[[web.vhosts]]",
|
||||
);
|
||||
let configured = format!("[general]\nconfig_strict = true\n{configured}");
|
||||
let config = load_config_from_temp_toml(&configured);
|
||||
|
||||
assert_eq!(config.web.timeouts.bridge_request_secs, 7);
|
||||
assert_eq!(config.web.timeouts.bridge_retry_secs, 41);
|
||||
assert_eq!(config.web.timeouts.carrier_probe_coalesce_ms, 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn web_bridge_deadlines_are_bounded_and_ordered() {
|
||||
for (field, value) in [
|
||||
("bridge_request_secs", "0"),
|
||||
("bridge_request_secs", "61"),
|
||||
("bridge_retry_secs", "0"),
|
||||
("bridge_retry_secs", "301"),
|
||||
("carrier_probe_coalesce_ms", "11"),
|
||||
] {
|
||||
let invalid = WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
&format!("[web.timeouts]\n{field} = {value}\n\n[[web.vhosts]]"),
|
||||
);
|
||||
assert!(
|
||||
load_config_error_from_temp_toml(&invalid).contains(&format!("web.timeouts.{field}"))
|
||||
);
|
||||
}
|
||||
|
||||
let reversed = WEB_CONFIG.replace(
|
||||
"[[web.vhosts]]",
|
||||
"[web.timeouts]\nbridge_request_secs = 20\nbridge_retry_secs = 10\n\n[[web.vhosts]]",
|
||||
);
|
||||
assert!(
|
||||
load_config_error_from_temp_toml(&reversed)
|
||||
.contains("bridge_request_secs must not exceed bridge_retry_secs")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -288,6 +288,15 @@ pub struct WebTimeoutsConfig {
|
||||
/// Maximum wait for one empty downlink long poll.
|
||||
#[serde(default = "default_web_long_poll_timeout_secs")]
|
||||
pub long_poll_secs: u64,
|
||||
/// Deadline for one generated-bridge HTTP attempt and response body.
|
||||
#[serde(default = "default_web_bridge_request_secs")]
|
||||
pub bridge_request_secs: u64,
|
||||
/// Absolute generated-bridge budget for one retryable HTTP operation.
|
||||
#[serde(default = "default_web_bridge_retry_secs")]
|
||||
pub bridge_retry_secs: u64,
|
||||
/// Optional delay for coalescing the first OPEN with immediate DATA.
|
||||
#[serde(default = "default_web_carrier_probe_coalesce_ms")]
|
||||
pub carrier_probe_coalesce_ms: u64,
|
||||
/// Grace for a canonical downlink poll that races its lane OPEN.
|
||||
#[serde(default = "default_web_lane_open_wait_secs")]
|
||||
pub lane_open_wait_secs: u64,
|
||||
@@ -340,6 +349,9 @@ impl Default for WebTimeoutsConfig {
|
||||
stream_handshake_secs: default_web_stream_handshake_timeout_secs(),
|
||||
stream_first_byte_secs: default_web_stream_first_byte_secs(),
|
||||
long_poll_secs: default_web_long_poll_timeout_secs(),
|
||||
bridge_request_secs: default_web_bridge_request_secs(),
|
||||
bridge_retry_secs: default_web_bridge_retry_secs(),
|
||||
carrier_probe_coalesce_ms: default_web_carrier_probe_coalesce_ms(),
|
||||
lane_open_wait_secs: default_web_lane_open_wait_secs(),
|
||||
carrier_health_secs: default_web_carrier_health_secs(),
|
||||
websocket_upgrade_secs: default_web_websocket_upgrade_secs(),
|
||||
|
||||
@@ -85,6 +85,9 @@ u64_default!(default_web_body_timeout_secs, 30);
|
||||
u64_default!(default_web_stream_handshake_timeout_secs, 10);
|
||||
u64_default!(default_web_stream_first_byte_secs, 30);
|
||||
u64_default!(default_web_long_poll_timeout_secs, 25);
|
||||
u64_default!(default_web_bridge_request_secs, 10);
|
||||
u64_default!(default_web_bridge_retry_secs, 90);
|
||||
u64_default!(default_web_carrier_probe_coalesce_ms, 0);
|
||||
u64_default!(default_web_lane_open_wait_secs, 2);
|
||||
u64_default!(default_web_carrier_health_secs, 30);
|
||||
u64_default!(default_web_websocket_upgrade_secs, 5);
|
||||
|
||||
@@ -284,7 +284,7 @@ impl ListenerSlot {
|
||||
}
|
||||
|
||||
pub(super) async fn stop(&mut self) -> Result<(), String> {
|
||||
self.cancellation.cancel();
|
||||
self.request_stop();
|
||||
if let Some(task) = self.task.take() {
|
||||
task.await.map_err(|error_value| {
|
||||
format!("listener {} task failed: {error_value}", self.spec.addr)
|
||||
@@ -305,6 +305,61 @@ impl ListenerSlot {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Cancels admission synchronously before the shared shutdown deadline starts draining.
|
||||
pub(super) fn request_stop(&self) {
|
||||
self.cancellation.cancel();
|
||||
}
|
||||
|
||||
/// Joins this acceptor and its WEB connections by one process shutdown deadline.
|
||||
pub(super) async fn stop_until(
|
||||
&mut self,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<(), String> {
|
||||
self.request_stop();
|
||||
let mut errors = Vec::new();
|
||||
if let Some(mut task) = self.task.take() {
|
||||
let joined = if task.is_finished() {
|
||||
Some(task.await)
|
||||
} else {
|
||||
match tokio::time::timeout_at(deadline, &mut task).await {
|
||||
Ok(result) => Some(result),
|
||||
Err(_) => {
|
||||
task.abort();
|
||||
let _ = task.await;
|
||||
None
|
||||
}
|
||||
}
|
||||
};
|
||||
match joined {
|
||||
Some(Ok(())) => {}
|
||||
Some(Err(error_value)) => errors.push(format!(
|
||||
"listener {} task failed: {error_value}",
|
||||
self.spec.addr
|
||||
)),
|
||||
None => errors.push(format!(
|
||||
"listener {} accept shutdown timed out",
|
||||
self.spec.addr
|
||||
)),
|
||||
}
|
||||
}
|
||||
self.connections.close();
|
||||
if !self.connections.is_empty()
|
||||
&& tokio::time::timeout_at(deadline, self.connections.wait())
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
errors.push(format!(
|
||||
"listener {} connection shutdown timed out",
|
||||
self.spec.addr
|
||||
));
|
||||
}
|
||||
if errors.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(errors.join("; "))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn restart(&mut self, active_runtime: Arc<ArcSwap<RuntimeGeneration>>) {
|
||||
self.active_runtime = active_runtime.clone();
|
||||
self.cancellation = CancellationToken::new();
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
|
||||
@@ -13,7 +14,8 @@ use super::bind::{BoundListeners, BoundTcpListener, PreparedTcpListener, prepare
|
||||
use super::plan::{ListenerBindSpec, listener_bind_plan};
|
||||
#[cfg(unix)]
|
||||
use super::unix::UnixAcceptHandle;
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
use crate::web::control::{WebRuntimeControl, WebRuntimeLifecycle};
|
||||
use crate::web::manager::{WebProcessRuntime, WebShutdownOutcome};
|
||||
use crate::web::trace::WebTraceStore;
|
||||
|
||||
/// Process-owned listener inventory and accept-task lifecycle controller.
|
||||
@@ -21,6 +23,8 @@ pub(crate) struct ListenerManager {
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
slots: BTreeMap<SocketAddr, ListenerSlot>,
|
||||
web_runtime: Option<Arc<WebProcessRuntime>>,
|
||||
web_control: WebRuntimeControl,
|
||||
web_listeners: Arc<[SocketAddr]>,
|
||||
#[cfg(unix)]
|
||||
unix: Option<UnixAcceptHandle>,
|
||||
}
|
||||
@@ -45,11 +49,15 @@ impl ListenerManager {
|
||||
bound: BoundListeners,
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
trace: Arc<WebTraceStore>,
|
||||
web_control: WebRuntimeControl,
|
||||
) -> Self {
|
||||
let has_web = bound
|
||||
let web_listeners: Arc<[SocketAddr]> = bound
|
||||
.listeners
|
||||
.iter()
|
||||
.any(|listener| listener.spec.transport == ListenerTransport::Web);
|
||||
.filter(|listener| listener.spec.transport == ListenerTransport::Web)
|
||||
.map(|listener| listener.spec.addr)
|
||||
.collect();
|
||||
let has_web = !web_listeners.is_empty();
|
||||
let web_runtime =
|
||||
has_web.then(|| WebProcessRuntime::start_with_trace(active_runtime.clone(), trace));
|
||||
let mut slots = BTreeMap::new();
|
||||
@@ -64,10 +72,23 @@ impl ListenerManager {
|
||||
let unix = bound
|
||||
.unix_listener
|
||||
.map(|listener| UnixAcceptHandle::start(listener, active_runtime.clone()));
|
||||
web_control.publish(
|
||||
if has_web {
|
||||
WebRuntimeLifecycle::Running
|
||||
} else {
|
||||
WebRuntimeLifecycle::NoWebListener
|
||||
},
|
||||
Arc::clone(&web_listeners),
|
||||
web_runtime
|
||||
.as_ref()
|
||||
.map_or_else(std::sync::Weak::new, Arc::downgrade),
|
||||
);
|
||||
Self {
|
||||
active_runtime,
|
||||
slots,
|
||||
web_runtime,
|
||||
web_control,
|
||||
web_listeners,
|
||||
#[cfg(unix)]
|
||||
unix,
|
||||
}
|
||||
@@ -75,10 +96,18 @@ impl ListenerManager {
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn empty(active_runtime: Arc<ArcSwap<RuntimeGeneration>>) -> Self {
|
||||
let web_control = WebRuntimeControl::new();
|
||||
web_control.publish(
|
||||
WebRuntimeLifecycle::NoWebListener,
|
||||
Arc::from([]),
|
||||
std::sync::Weak::new(),
|
||||
);
|
||||
Self {
|
||||
active_runtime,
|
||||
slots: BTreeMap::new(),
|
||||
web_runtime: None,
|
||||
web_control,
|
||||
web_listeners: Arc::from([]),
|
||||
#[cfg(unix)]
|
||||
unix: None,
|
||||
}
|
||||
@@ -214,28 +243,101 @@ impl ListenerManager {
|
||||
);
|
||||
}
|
||||
|
||||
/// Stops and joins every accept task before sockets are released.
|
||||
/// Stops every accept task and applies one deadline to the complete WEB ingress.
|
||||
pub(crate) async fn shutdown(&mut self) -> Result<(), String> {
|
||||
self.web_control.publish(
|
||||
WebRuntimeLifecycle::Draining,
|
||||
Arc::clone(&self.web_listeners),
|
||||
self.web_runtime
|
||||
.as_ref()
|
||||
.map_or_else(std::sync::Weak::new, Arc::downgrade),
|
||||
);
|
||||
if self.web_runtime.is_none() {
|
||||
let mut errors = Vec::new();
|
||||
for slot in self.slots.values_mut() {
|
||||
if let Err(error_value) = slot.stop().await {
|
||||
errors.push(error_value);
|
||||
}
|
||||
}
|
||||
#[cfg(unix)]
|
||||
if let Some(unix) = &mut self.unix
|
||||
&& let Err(error_value) = unix.stop().await
|
||||
{
|
||||
errors.push(error_value);
|
||||
}
|
||||
self.slots.clear();
|
||||
#[cfg(unix)]
|
||||
{
|
||||
self.unix = None;
|
||||
}
|
||||
self.web_control.publish(
|
||||
WebRuntimeLifecycle::Drained,
|
||||
Arc::clone(&self.web_listeners),
|
||||
std::sync::Weak::new(),
|
||||
);
|
||||
return if errors.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(errors.join("; "))
|
||||
};
|
||||
}
|
||||
let timeout_secs = self
|
||||
.active_runtime
|
||||
.load()
|
||||
.config()
|
||||
.web
|
||||
.timeouts
|
||||
.shutdown_secs;
|
||||
let now = tokio::time::Instant::now();
|
||||
let deadline = now
|
||||
.checked_add(Duration::from_secs(timeout_secs))
|
||||
.unwrap_or(now);
|
||||
for slot in self.slots.values() {
|
||||
slot.request_stop();
|
||||
}
|
||||
#[cfg(unix)]
|
||||
if let Some(unix) = &self.unix {
|
||||
unix.request_stop();
|
||||
}
|
||||
let Some(web_runtime) = self.web_runtime.take() else {
|
||||
return Err("WEB runtime disappeared during shutdown orchestration".to_string());
|
||||
};
|
||||
let drain = web_runtime.begin_shutdown();
|
||||
let mut errors = Vec::new();
|
||||
for slot in self.slots.values_mut() {
|
||||
if let Err(error_value) = slot.stop().await {
|
||||
let slot_waits = futures_util::future::join_all(
|
||||
self.slots
|
||||
.values_mut()
|
||||
.map(|slot| slot.stop_until(deadline)),
|
||||
);
|
||||
let (slot_results, web_outcome) = tokio::join!(slot_waits, drain.wait_until(deadline));
|
||||
for result in slot_results {
|
||||
if let Err(error_value) = result {
|
||||
errors.push(error_value);
|
||||
}
|
||||
}
|
||||
#[cfg(unix)]
|
||||
if let Some(unix) = &mut self.unix
|
||||
&& let Err(error_value) = unix.stop().await
|
||||
&& let Err(error_value) = unix.stop_until(deadline).await
|
||||
{
|
||||
errors.push(error_value);
|
||||
}
|
||||
self.slots.clear();
|
||||
if let Some(web_runtime) = self.web_runtime.take() {
|
||||
web_runtime.shutdown().await;
|
||||
if web_outcome == WebShutdownOutcome::DeadlineExceeded {
|
||||
errors.push("WEB ingress shutdown deadline exceeded".to_string());
|
||||
}
|
||||
self.slots.clear();
|
||||
#[cfg(unix)]
|
||||
{
|
||||
self.unix = None;
|
||||
}
|
||||
self.web_control.publish(
|
||||
if web_outcome == WebShutdownOutcome::DeadlineExceeded {
|
||||
WebRuntimeLifecycle::DeadlineExceeded
|
||||
} else {
|
||||
WebRuntimeLifecycle::Drained
|
||||
},
|
||||
Arc::clone(&self.web_listeners),
|
||||
std::sync::Weak::new(),
|
||||
);
|
||||
if errors.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
@@ -314,7 +416,8 @@ mod tests {
|
||||
runtime.config().web.debug.clone(),
|
||||
&runtime.config().web.limits,
|
||||
);
|
||||
let mut manager = ListenerManager::start(bound, active_runtime, trace);
|
||||
let mut manager =
|
||||
ListenerManager::start(bound, active_runtime, trace, WebRuntimeControl::new());
|
||||
let blocker = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let blocked_addr = blocker.local_addr().unwrap();
|
||||
let mut desired = ProxyConfig::default();
|
||||
@@ -341,7 +444,8 @@ mod tests {
|
||||
runtime.config().web.debug.clone(),
|
||||
&runtime.config().web.limits,
|
||||
);
|
||||
let mut manager = ListenerManager::start(bound, active_runtime, trace);
|
||||
let mut manager =
|
||||
ListenerManager::start(bound, active_runtime, trace, WebRuntimeControl::new());
|
||||
let reservation = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let new_addr = reservation.local_addr().unwrap();
|
||||
drop(reservation);
|
||||
|
||||
@@ -148,11 +148,42 @@ impl UnixAcceptHandle {
|
||||
}
|
||||
|
||||
pub(super) async fn stop(&mut self) -> Result<(), String> {
|
||||
self.cancellation.cancel();
|
||||
self.request_stop();
|
||||
if let Some(task) = self.task.take() {
|
||||
task.await
|
||||
.map_err(|error_value| format!("Unix listener task failed: {error_value}"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Cancels Unix admission before process-owned listener waits begin.
|
||||
pub(super) fn request_stop(&self) {
|
||||
self.cancellation.cancel();
|
||||
}
|
||||
|
||||
/// Joins the Unix acceptor by the process listener deadline.
|
||||
pub(super) async fn stop_until(
|
||||
&mut self,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> Result<(), String> {
|
||||
self.request_stop();
|
||||
let Some(mut task) = self.task.take() else {
|
||||
return Ok(());
|
||||
};
|
||||
if task.is_finished() {
|
||||
return task
|
||||
.await
|
||||
.map_err(|error_value| format!("Unix listener task failed: {error_value}"));
|
||||
}
|
||||
match tokio::time::timeout_at(deadline, &mut task).await {
|
||||
Ok(result) => {
|
||||
result.map_err(|error_value| format!("Unix listener task failed: {error_value}"))
|
||||
}
|
||||
Err(_) => {
|
||||
task.abort();
|
||||
let _ = task.await;
|
||||
Err("Unix listener shutdown timed out".to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ use crate::stats::{QuotaStore, Stats};
|
||||
use crate::synlimit_control;
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
use crate::web::control::WebRuntimeControl;
|
||||
use crate::web::trace::WebTraceStore;
|
||||
|
||||
use super::{
|
||||
@@ -106,6 +107,7 @@ pub(super) async fn run_telemt_core(
|
||||
config.access.cidr_rate_limits.clone(),
|
||||
);
|
||||
let web_trace = WebTraceStore::new(config.web.debug.clone(), &config.web.limits);
|
||||
let web_runtime_control = WebRuntimeControl::new();
|
||||
|
||||
let (detected_ips_tx, detected_ips_rx) = watch::channel((None::<IpAddr>, None::<IpAddr>));
|
||||
let initial_direct_first = config.general.use_middle_proxy && config.general.me2dc_fallback;
|
||||
@@ -157,6 +159,7 @@ pub(super) async fn run_telemt_core(
|
||||
let active_runtime_rx_api = active_runtime_rx.clone();
|
||||
let runtime_watch_rx_api = runtime_watch_rx.clone();
|
||||
let web_trace_api = web_trace.clone();
|
||||
let web_runtime_rx_api = web_runtime_control.subscribe();
|
||||
tokio::spawn(async move {
|
||||
api::serve(
|
||||
listen,
|
||||
@@ -175,6 +178,7 @@ pub(super) async fn run_telemt_core(
|
||||
active_runtime_rx_api,
|
||||
runtime_watch_rx_api,
|
||||
web_trace_api,
|
||||
web_runtime_rx_api,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
@@ -318,8 +322,12 @@ pub(super) async fn run_telemt_core(
|
||||
active_runtime_tx.send_replace(Some(active_runtime.clone()));
|
||||
runtime_tasks::mark_runtime_ready(&startup_tracker).await;
|
||||
|
||||
let listener_manager =
|
||||
listeners::ListenerManager::start(bound, active_runtime.clone(), web_trace.clone());
|
||||
let listener_manager = listeners::ListenerManager::start(
|
||||
bound,
|
||||
active_runtime.clone(),
|
||||
web_trace.clone(),
|
||||
web_runtime_control,
|
||||
);
|
||||
let reload_supervisor = reload_supervisor::ReloadSupervisor::spawn(
|
||||
active_runtime.clone(),
|
||||
reload_control,
|
||||
|
||||
@@ -278,7 +278,11 @@ impl ReloadSupervisor {
|
||||
self.control
|
||||
.mark_phase(command.reload_id, ReloadPhase::Activating)
|
||||
.await;
|
||||
let new_runtime = prepared.generation;
|
||||
let PreparedRuntime {
|
||||
generation: new_runtime,
|
||||
detected_ips,
|
||||
config_watcher_activation,
|
||||
} = prepared;
|
||||
if let Err(error) = install_dns(&new_runtime.config().network.dns_overrides) {
|
||||
let message = format!("runtime DNS activation failed: {}", error);
|
||||
if command.request.failure_policy == ReloadFailurePolicy::Rollback {
|
||||
@@ -313,14 +317,16 @@ impl ReloadSupervisor {
|
||||
};
|
||||
old_runtime.stop_accepting_sessions();
|
||||
let replaced = self.active_runtime.swap(new_runtime.clone());
|
||||
self.web_trace.apply_policy(&new_runtime.config().web.debug);
|
||||
self.web_trace
|
||||
.apply_policy(new_runtime.id, &new_runtime.config().web.debug);
|
||||
config_watcher_activation.send_replace(true);
|
||||
if let Some(pending) = pending_listener_transition {
|
||||
self.listener_manager
|
||||
.lock()
|
||||
.await
|
||||
.finish_transition(pending);
|
||||
}
|
||||
self.detected_ips_tx.send_replace(prepared.detected_ips);
|
||||
self.detected_ips_tx.send_replace(detected_ips);
|
||||
self.runtime_log_filter
|
||||
.apply_reload(&new_runtime.config().general.log_level);
|
||||
self.runtime_watch_tx
|
||||
|
||||
@@ -21,6 +21,15 @@ fn runtime_log_filter() -> RuntimeLogFilter {
|
||||
RuntimeLogFilter::new(handle)
|
||||
}
|
||||
|
||||
fn prepared_runtime(generation: Arc<RuntimeGeneration>) -> PreparedRuntime {
|
||||
let (config_watcher_activation, _activation_rx) = watch::channel(false);
|
||||
PreparedRuntime {
|
||||
generation,
|
||||
detected_ips: (None, None),
|
||||
config_watcher_activation,
|
||||
}
|
||||
}
|
||||
|
||||
async fn fixture(request: ReloadRequest) -> ReloadFixture {
|
||||
let old_runtime = test_runtime_generation(1, ProxyConfig::default());
|
||||
let new_config = Arc::new(ProxyConfig::default());
|
||||
@@ -120,10 +129,7 @@ async fn revision_rollback_keeps_old_generation_and_cleans_candidate() {
|
||||
.activate_prepared(
|
||||
fixture.command,
|
||||
fixture.old_runtime.clone(),
|
||||
PreparedRuntime {
|
||||
generation: fixture.new_runtime,
|
||||
detected_ips: (None, None),
|
||||
},
|
||||
prepared_runtime(fixture.new_runtime),
|
||||
RevisionGateAction::Rollback("revision changed".to_string()),
|
||||
|_| -> Result<(), String> { panic!("DNS activation must not run on rollback") },
|
||||
)
|
||||
@@ -161,10 +167,7 @@ async fn dns_failure_policy_controls_rollback_or_keep_new() {
|
||||
.activate_prepared(
|
||||
fixture.command,
|
||||
fixture.old_runtime.clone(),
|
||||
PreparedRuntime {
|
||||
generation: fixture.new_runtime.clone(),
|
||||
detected_ips: (None, None),
|
||||
},
|
||||
prepared_runtime(fixture.new_runtime.clone()),
|
||||
RevisionGateAction::Proceed,
|
||||
|_| Err("invalid DNS entry".to_string()),
|
||||
)
|
||||
@@ -215,10 +218,7 @@ async fn drain_publishes_new_generation_before_old_sessions_finish() {
|
||||
.activate_prepared(
|
||||
fixture.command,
|
||||
old_runtime,
|
||||
PreparedRuntime {
|
||||
generation: new_runtime,
|
||||
detected_ips: (None, None),
|
||||
},
|
||||
prepared_runtime(new_runtime),
|
||||
RevisionGateAction::Proceed,
|
||||
|_| Ok(()),
|
||||
)
|
||||
@@ -271,10 +271,7 @@ async fn drain_timeout_cancels_old_sessions_and_records_one_warning() {
|
||||
.activate_prepared(
|
||||
fixture.command,
|
||||
old_runtime,
|
||||
PreparedRuntime {
|
||||
generation: new_runtime,
|
||||
detected_ips: (None, None),
|
||||
},
|
||||
prepared_runtime(new_runtime),
|
||||
RevisionGateAction::Proceed,
|
||||
|_| Ok(()),
|
||||
)
|
||||
|
||||
@@ -28,9 +28,14 @@ use super::listeners::listener_rebind_supported;
|
||||
use super::runtime_tasks::RuntimeLogFilter;
|
||||
use super::{me_startup, runtime_tasks, tls_bootstrap};
|
||||
|
||||
/// Fully prepared candidate runtime and its activation-gated config watcher.
|
||||
pub(crate) struct PreparedRuntime {
|
||||
/// Candidate generation ready for publication.
|
||||
pub(crate) generation: Arc<RuntimeGeneration>,
|
||||
/// Detected public addresses associated with the candidate.
|
||||
pub(crate) detected_ips: (Option<IpAddr>, Option<IpAddr>),
|
||||
/// Gate opened only after the candidate becomes the active generation.
|
||||
pub(crate) config_watcher_activation: watch::Sender<bool>,
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_runtime(
|
||||
@@ -171,6 +176,7 @@ pub(crate) async fn prepare_runtime(
|
||||
config.server.max_connections as usize
|
||||
};
|
||||
let max_connections = Arc::new(Semaphore::new(max_connections_limit));
|
||||
let (config_watcher_activation, config_watcher_activation_rx) = watch::channel(false);
|
||||
let watches = runtime_tasks::spawn_runtime_tasks(
|
||||
&config,
|
||||
config_path,
|
||||
@@ -190,6 +196,7 @@ pub(crate) async fn prepare_runtime(
|
||||
proxy_shared.clone(),
|
||||
me_ready_tx.clone(),
|
||||
task_scope.clone(),
|
||||
Some(config_watcher_activation_rx),
|
||||
)
|
||||
.await;
|
||||
let config_rx = watches.config_rx;
|
||||
@@ -295,6 +302,7 @@ pub(crate) async fn prepare_runtime(
|
||||
|
||||
Ok(PreparedRuntime {
|
||||
generation,
|
||||
config_watcher_activation,
|
||||
detected_ips: (
|
||||
probe.detected_ipv4.map(IpAddr::V4),
|
||||
probe.detected_ipv6.map(IpAddr::V6),
|
||||
|
||||
@@ -247,6 +247,7 @@ pub(super) async fn prepare_runtime(
|
||||
shared_state.clone(),
|
||||
me_ready_tx.clone(),
|
||||
runtime_task_scope.clone(),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
let config_rx = runtime_watches.config_rx;
|
||||
|
||||
@@ -107,6 +107,7 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
shared_state: Arc<ProxySharedState>,
|
||||
me_ready_tx: watch::Sender<u64>,
|
||||
task_scope: RuntimeTaskScope,
|
||||
config_watcher_activation: Option<watch::Receiver<bool>>,
|
||||
) -> RuntimeWatches {
|
||||
let um_clone = upstream_manager.clone();
|
||||
let dc_overrides_for_health = config.dc_overrides.clone();
|
||||
@@ -151,14 +152,15 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
Some("spawn config hot-reload watcher".to_string()),
|
||||
)
|
||||
.await;
|
||||
let (config_rx, log_level_rx): (watch::Receiver<Arc<ProxyConfig>>, watch::Receiver<LogLevel>) =
|
||||
spawn_config_watcher(
|
||||
config_path.to_path_buf(),
|
||||
config.clone(),
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
task_scope.cancellation_token(),
|
||||
);
|
||||
let (config_rx, log_level_rx, config_watcher_task) = spawn_config_watcher(
|
||||
config_path.to_path_buf(),
|
||||
config.clone(),
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
task_scope.cancellation_token(),
|
||||
config_watcher_activation,
|
||||
);
|
||||
task_scope.spawn(config_watcher_task);
|
||||
startup_tracker
|
||||
.complete_component(
|
||||
COMPONENT_CONFIG_WATCHER_START,
|
||||
|
||||
+14
-455
@@ -24,12 +24,17 @@ pub(crate) fn render(
|
||||
negotiation_enabled: bool,
|
||||
candidate_count: usize,
|
||||
carrier_deadlines: [u64; 4],
|
||||
long_poll_secs: u64,
|
||||
bridge_request_secs: u64,
|
||||
bridge_retry_secs: u64,
|
||||
carrier_probe_coalesce_ms: u64,
|
||||
rng: &SecureRandom,
|
||||
) -> BridgePage {
|
||||
let mut nonce = [0u8; 18];
|
||||
rng.fill(&mut nonce);
|
||||
let nonce = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(nonce);
|
||||
let body = DOCUMENT
|
||||
.replace("__RUNTIME__", RUNTIME)
|
||||
.replace("__NONCE__", &nonce)
|
||||
.replace("__HOST__", host)
|
||||
.replace("__BOOTSTRAP__", bootstrap)
|
||||
@@ -41,6 +46,13 @@ pub(crate) fn render(
|
||||
if negotiation_enabled { "true" } else { "false" },
|
||||
)
|
||||
.replace("__CANDIDATE_COUNT__", &candidate_count.to_string())
|
||||
.replace("__LONG_POLL_SECS__", &long_poll_secs.to_string())
|
||||
.replace("__BRIDGE_REQUEST_SECS__", &bridge_request_secs.to_string())
|
||||
.replace("__BRIDGE_RETRY_SECS__", &bridge_retry_secs.to_string())
|
||||
.replace(
|
||||
"__CARRIER_PROBE_COALESCE_MS__",
|
||||
&carrier_probe_coalesce_ms.to_string(),
|
||||
)
|
||||
.replace(
|
||||
"__CARRIER_DEADLINES__",
|
||||
&carrier_deadlines
|
||||
@@ -57,461 +69,8 @@ pub(crate) fn render(
|
||||
}
|
||||
}
|
||||
|
||||
const DOCUMENT: &str = r##"<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Connection</title>
|
||||
</head>
|
||||
<body>
|
||||
<script nonce="__NONCE__">
|
||||
(()=>{
|
||||
'use strict';
|
||||
const bootstrap="__BOOTSTRAP__";
|
||||
const relayOrigin='https://__HOST__',carrierCapabilities='https,https-lanes,websocket,websocket-lanes';
|
||||
const negotiationEnabled=__NEGOTIATION_ENABLED__,candidateCount=__CANDIDATE_COUNT__,candidateDeadlines=[__CARRIER_DEADLINES__];
|
||||
let negotiatedCandidateCount=candidateCount,negotiatedFinalDeadline=candidateDeadlines[3],negotiatedFrozen=false;
|
||||
const batchLimit=__BATCH_LIMIT__,queueLimit=__QUEUE_LIMIT__,queueItemLimit=__QUEUE_ITEMS__;
|
||||
const laneQueueLimit=Math.min(queueLimit,8388608),laneItemLimit=Math.min(queueItemLimit,1024),closedLaneLimit=4096;
|
||||
const fragment=location.hash,androidNonce=/^#android=([A-Za-z0-9_-]{43})$/.exec(fragment)?.[1]||'';
|
||||
history.replaceState(null,'',location.pathname);
|
||||
let initialized=false,closed=false,port=null,sessionToken='',cleanupToken='',createStarted=false,socket=null,socketReady=false,carrier='';
|
||||
let queuedBytes=0,queuedItems=0,upSequence=1,downCursor='0',upRunning=false,pollController=null;
|
||||
let helloFrame=null,welcomeSent=false,carrierAttempt=1,carrierFailure='',carrierCommitted=false;
|
||||
let negotiationStartedAt=0,carrierTimer=null,attemptController=null,attemptEpoch=1,candidateRunning=false,switching=false,currentAttempt=null;
|
||||
const pending=[],upPending=[],lanes=new Map(),closedLanes=new Set(),closedLaneOrder=[];
|
||||
const status=state=>{if(port&&!closed)port.postMessage({t:'status',state})};
|
||||
const pause=(milliseconds,signal)=>new Promise((resolve,reject)=>{
|
||||
if(signal&&signal.aborted){reject(new Error('request aborted'));return}
|
||||
const timer=setTimeout(done,milliseconds);function done(){if(signal)signal.removeEventListener('abort',abort);resolve()}
|
||||
function abort(){clearTimeout(timer);signal.removeEventListener('abort',abort);reject(new Error('request aborted'))}
|
||||
if(signal)signal.addEventListener('abort',abort,{once:true});
|
||||
});
|
||||
const socketURL=()=>relayOrigin.replace(/^https:/,'wss:')+'/api/v1/ws';
|
||||
const options=(method,token,body,headers,signal,keepalive)=>({
|
||||
method,body,signal,keepalive:!!keepalive,mode:'same-origin',credentials:'omit',cache:'no-store',redirect:'error',referrerPolicy:'no-referrer',
|
||||
headers:Object.assign(token?{Authorization:'Bearer '+token}:{},body?{'Content-Type':'application/octet-stream'}:{},headers||{})
|
||||
});
|
||||
const attemptHeaders=(attempt,failure)=>negotiationEnabled?Object.assign({'X-Carrier-Capabilities':carrierCapabilities,'X-Carrier-Attempt':String(attempt)},failure?{'X-Carrier-Failure':failure}:{}):{};
|
||||
function reserve(data,lane){
|
||||
let buffered=socket?socket.bufferedAmount:0;for(const value of lanes.values())if(value.socket)buffered+=value.socket.bufferedAmount;
|
||||
if(!data.byteLength||data.byteLength>queueLimit-queuedBytes-buffered||queuedItems>=queueItemLimit)return false;
|
||||
if(lane&&(data.byteLength>laneQueueLimit-lane.bytes-(lane.socket?lane.socket.bufferedAmount:0)||lane.items>=laneItemLimit))return false;
|
||||
queuedBytes+=data.byteLength;queuedItems++;if(lane){lane.bytes+=data.byteLength;lane.items++}return true;
|
||||
}
|
||||
function release(bytes,items,lane){queuedBytes-=bytes;queuedItems-=items;if(lane){lane.bytes-=bytes;lane.items-=items}}
|
||||
function frameBound(value,maxFrames,maxBytes){
|
||||
const view=new DataView(value);let offset=0,frames=0;
|
||||
while(offset<value.byteLength){
|
||||
if(value.byteLength-offset<8)throw new Error('invalid frame batch');
|
||||
const size=view.getUint32(offset+4),end=offset+8+size;
|
||||
if(size>1048576||end>value.byteLength)throw new Error('invalid frame');
|
||||
if(frames>0&&(frames>=maxFrames||end>maxBytes))break;
|
||||
frames++;offset=end;
|
||||
}
|
||||
if(!frames)throw new Error('empty frame batch');
|
||||
return {frames,bytes:offset};
|
||||
}
|
||||
function splitFrames(value){
|
||||
const view=new DataView(value),result=[];let offset=0;
|
||||
while(offset<value.byteLength){
|
||||
if(value.byteLength-offset<8||result.length>=4096)throw new Error('invalid frame batch');
|
||||
const type=view.getUint8(offset),id=(view.getUint8(offset+1)<<16)|(view.getUint8(offset+2)<<8)|view.getUint8(offset+3);
|
||||
const size=view.getUint32(offset+4),end=offset+8+size;
|
||||
if((type===2&&!size)||size>1048576||end>value.byteLength)throw new Error('invalid frame');
|
||||
result.push({type,id,data:offset===0&&end===value.byteLength?value:value.slice(offset,end)});offset=end;
|
||||
}
|
||||
if(!result.length)throw new Error('empty frame batch');return result;
|
||||
}
|
||||
function findProbe(){
|
||||
for(let index=0;index<pending.length;index++){
|
||||
const value=pending[index],view=new DataView(value);let offset=0,frames=0;
|
||||
while(offset<value.byteLength){
|
||||
if(value.byteLength-offset<8||frames++>=4096)throw new Error('invalid frame batch');
|
||||
const type=view.getUint8(offset),id=(view.getUint8(offset+1)<<16)|(view.getUint8(offset+2)<<8)|view.getUint8(offset+3);
|
||||
const size=view.getUint32(offset+4),end=offset+8+size;
|
||||
if((type===2&&!size)||size>1048576||end>value.byteLength)throw new Error('invalid frame');
|
||||
if(type===1||type===2)return {source:value,index,start:offset,end,id,data:value.slice(offset,end)};
|
||||
offset=end;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function consumeProbe(probe){
|
||||
if(pending[probe.index]!==probe.source)throw new Error('stale carrier probe');
|
||||
const before=probe.source.slice(0,probe.start),after=probe.source.slice(probe.end),remaining=before.byteLength+after.byteLength;
|
||||
if(!remaining){pending.splice(probe.index,1);release(probe.end-probe.start,1,null);return}
|
||||
const merged=new Uint8Array(remaining);merged.set(new Uint8Array(before),0);merged.set(new Uint8Array(after),before.byteLength);
|
||||
pending[probe.index]=merged.buffer;release(probe.end-probe.start,0,null);
|
||||
}
|
||||
function joinPending(values,lane){
|
||||
let total=0,count=0,frames=0;
|
||||
while(count<values.length){
|
||||
const bound=frameBound(values[count],4096,batchLimit),whole=bound.bytes===values[count].byteLength;
|
||||
if(count===0&&!whole){
|
||||
const head=new Uint8Array(values[0],0,bound.bytes).slice();
|
||||
values[0]=values[0].slice(bound.bytes);queuedItems++;if(lane)lane.items++;
|
||||
return {body:head.buffer,total:bound.bytes,count:1};
|
||||
}
|
||||
if(count&&(total+values[count].byteLength>batchLimit||frames+bound.frames>4096))break;
|
||||
total+=values[count].byteLength;frames+=bound.frames;count++;
|
||||
}
|
||||
const joined=new Uint8Array(total);let offset=0;
|
||||
for(const data of values.splice(0,count)){joined.set(new Uint8Array(data),offset);offset+=data.byteLength}
|
||||
return {body:joined.buffer,total,count};
|
||||
}
|
||||
function retryAfterMs(response){
|
||||
const header=response.headers.get('Retry-After');
|
||||
if(!header)return 0;
|
||||
const seconds=Number(header);
|
||||
if(Number.isFinite(seconds)&&seconds>=0)return Math.min(seconds*1000,30000);
|
||||
const when=Date.parse(header);
|
||||
if(Number.isFinite(when)){const delta=when-Date.now();return delta>0?Math.min(delta,30000):0}
|
||||
return 0;
|
||||
}
|
||||
async function request(path,frozenOptions){
|
||||
let delay=250,attempt=0;const deadline=Date.now()+90000;
|
||||
while(true){
|
||||
const controller=new AbortController(),external=frozenOptions.signal;
|
||||
if(closed||(external&&external.aborted))throw new Error('request aborted');
|
||||
const abort=()=>controller.abort();if(external)external.addEventListener('abort',abort,{once:true});
|
||||
const requestOptions=Object.assign({},frozenOptions,{signal:controller.signal});const timer=setTimeout(abort,90000);
|
||||
let serviceUnavailable=false,wait=0;
|
||||
try{
|
||||
const response=await fetch(relayOrigin+path,requestOptions);
|
||||
if(response.status!==503)return response;
|
||||
serviceUnavailable=true;wait=retryAfterMs(response);await response.arrayBuffer();
|
||||
}catch(error){
|
||||
if(closed||(external&&external.aborted))throw error;
|
||||
if(++attempt===9)throw new Error('carrier retry limit reached');
|
||||
}finally{clearTimeout(timer);if(external)external.removeEventListener('abort',abort)}
|
||||
if(serviceUnavailable&&Date.now()>=deadline)throw new Error('carrier retry limit reached');
|
||||
status('reconnecting');await pause(wait||(delay+Math.floor(Math.random()*Math.max(1,delay/4))),external);
|
||||
if(closed||(external&&external.aborted))throw new Error('request aborted');
|
||||
if(!serviceUnavailable)delay=Math.min(delay*2,5000);
|
||||
}
|
||||
}
|
||||
function fail(){if(closed)return;status('failed');if(port)port.postMessage({t:'close'});close(true)}
|
||||
function knownCarrier(value){return value==='https'||value==='https-lanes'||value==='websocket'||value==='websocket-lanes'}
|
||||
function sessionEcho(response,expectedAttempt,states,exactAttempt){
|
||||
const selected=response.headers.get('X-Carrier-Mode')||'',echo=response.headers.get('X-Carrier-Attempt')||'';
|
||||
if(!knownCarrier(selected))throw new Error('invalid carrier mode');
|
||||
if(!negotiationEnabled){if(echo!=='')throw new Error('unexpected carrier attempt');return {selected,state:''}}
|
||||
const count=response.headers.get('X-Carrier-Candidate-Count')||'',deadline=response.headers.get('X-Carrier-Deadline')||'',state=response.headers.get('X-Carrier-State')||'';
|
||||
if(!/^[1-4]$/.test(count)||!/^[1-9]\d*$/.test(deadline)||!states.includes(state))throw new Error('invalid carrier state');
|
||||
const echoedAttempt=Number(echo),parsedCount=Number(count),parsedDeadline=Number(deadline);
|
||||
if(!Number.isInteger(echoedAttempt)||echoedAttempt<1||(exactAttempt?echoedAttempt!==expectedAttempt:echoedAttempt>expectedAttempt))throw new Error('invalid carrier attempt');
|
||||
if(parsedCount>candidateCount||parsedDeadline>candidateDeadlines[3])throw new Error('invalid carrier bounds');
|
||||
if(!negotiatedFrozen){negotiatedCandidateCount=parsedCount;negotiatedFinalDeadline=parsedDeadline;negotiatedFrozen=true}
|
||||
else if(parsedCount!==negotiatedCandidateCount||parsedDeadline!==negotiatedFinalDeadline)throw new Error('changed carrier bounds');
|
||||
if(echoedAttempt>negotiatedCandidateCount)throw new Error('carrier attempt exceeds candidates');
|
||||
return {selected,state};
|
||||
}
|
||||
function armCarrierDeadline(epoch){
|
||||
if(!negotiationStartedAt||epoch!==attemptEpoch)return;
|
||||
if(carrierTimer)clearTimeout(carrierTimer);
|
||||
const deadline=carrierAttempt>=negotiatedCandidateCount?negotiatedFinalDeadline:candidateDeadlines[carrierAttempt-1];
|
||||
const remaining=negotiationStartedAt+deadline*1000-Date.now();
|
||||
carrierTimer=setTimeout(()=>advanceCarrier('timeout',epoch),Math.max(0,remaining));
|
||||
}
|
||||
function resetCandidate(){
|
||||
if(pollController)pollController.abort();pollController=null;
|
||||
if(socket){const previous=socket;socket=null;previous.close()}socketReady=false;
|
||||
for(const lane of lanes.values()){if(lane.controller)lane.controller.abort();if(lane.socket)lane.socket.close()}
|
||||
lanes.clear();closedLanes.clear();closedLaneOrder.length=0;upPending.length=0;upSequence=1;downCursor='0';upRunning=false;
|
||||
sessionToken='';carrier='';candidateRunning=false;currentAttempt=null;
|
||||
}
|
||||
function advanceConfirmed(reason,epoch){
|
||||
if(closed||carrierCommitted||epoch!==attemptEpoch)return;
|
||||
resetCandidate();
|
||||
if(carrierAttempt>=negotiatedCandidateCount||Date.now()>=negotiationStartedAt+negotiatedFinalDeadline*1000){switching=false;fail();return}
|
||||
carrierAttempt++;carrierFailure=reason;attemptEpoch++;const nextEpoch=attemptEpoch;switching=false;
|
||||
status('reconnecting');armCarrierDeadline(nextEpoch);createSession(nextEpoch);
|
||||
}
|
||||
function advanceCarrier(reason,epoch){
|
||||
if(closed||carrierCommitted||epoch!==attemptEpoch||switching)return;
|
||||
if(!negotiationEnabled){fail();return}
|
||||
switching=true;if(carrierTimer)clearTimeout(carrierTimer);carrierTimer=null;
|
||||
const snapshot=currentAttempt;if(attemptController)attemptController.abort();attemptController=null;
|
||||
if(!snapshot||snapshot.epoch!==epoch){switching=false;fail();return}
|
||||
resolveAttempt(reason,epoch,snapshot);
|
||||
}
|
||||
async function resolveAttempt(reason,epoch,snapshot){
|
||||
const controller=new AbortController();attemptController=controller;
|
||||
const remaining=negotiationStartedAt+negotiatedFinalDeadline*1000-Date.now();
|
||||
if(remaining<=0){switching=false;fail();return}
|
||||
const timer=setTimeout(()=>controller.abort(),remaining);
|
||||
try{
|
||||
const frozen=options('POST',bootstrap,snapshot.hello,attemptHeaders(snapshot.attempt,snapshot.failure),controller.signal);
|
||||
const response=await request('/api/v1/session',frozen);
|
||||
if(closed||epoch!==attemptEpoch){await response.arrayBuffer();return}
|
||||
if(response.status===409){sessionEcho(response,snapshot.attempt,['committed','healthy'],false);await response.arrayBuffer();switching=false;fail();return}
|
||||
if(response.status!==200){await response.arrayBuffer();switching=false;fail();return}
|
||||
const echo=sessionEcho(response,snapshot.attempt,['provisional','committed','healthy'],true);
|
||||
const token=response.headers.get('X-Session-Token')||'',cursor=response.headers.get('X-Down-Cursor')||'';
|
||||
if(!token||cursor!=='0'||(snapshot.selected&&echo.selected!==snapshot.selected))throw new Error('changed carrier replay');
|
||||
const welcome=await response.arrayBuffer();if(closed||epoch!==attemptEpoch)return;
|
||||
cleanupToken=token;
|
||||
if(!welcomeSent){welcomeSent=true;port.postMessage(welcome,[welcome])}
|
||||
if(echo.state!=='provisional'){switching=false;fail();return}
|
||||
advanceConfirmed(reason,epoch);
|
||||
}catch(error){if(!closed&&epoch===attemptEpoch){switching=false;fail()}}
|
||||
finally{clearTimeout(timer);if(attemptController===controller)attemptController=null}
|
||||
}
|
||||
function maybeStartCandidate(){
|
||||
let probe;try{probe=findProbe()}catch(error){fail();return}
|
||||
if(!probe||closed||carrierCommitted)return;
|
||||
if(!sessionToken||candidateRunning)return;
|
||||
candidateRunning=true;const epoch=attemptEpoch;
|
||||
if(carrier==='https')probeHttp(probe,null,epoch);
|
||||
else if(carrier==='https-lanes')probeHttp(probe,probe.id,epoch);
|
||||
else if(carrier==='websocket')openCandidateSocket(probe,null,epoch);
|
||||
else if(carrier==='websocket-lanes')openCandidateSocket(probe,probe.id,epoch);
|
||||
else advanceCarrier('protocol',epoch);
|
||||
}
|
||||
async function createSession(epoch){
|
||||
const controller=new AbortController(),attempt=carrierAttempt,failure=carrierFailure;
|
||||
const snapshot={epoch,attempt,failure,hello:helloFrame,selected:''};currentAttempt=snapshot;attemptController=controller;
|
||||
try{
|
||||
status('connecting');
|
||||
const frozen=options('POST',bootstrap,snapshot.hello,attemptHeaders(attempt,failure),controller.signal);
|
||||
const response=await request('/api/v1/session',frozen);
|
||||
if(closed||epoch!==attemptEpoch){await response.arrayBuffer();return}
|
||||
if(response.status===409){sessionEcho(response,attempt,['committed','healthy'],false);await response.arrayBuffer();fail();return}
|
||||
if(response.status!==200){await response.arrayBuffer();advanceCarrier('http',epoch);return}
|
||||
const echo=sessionEcho(response,attempt,['provisional'],true),selected=echo.selected;snapshot.selected=selected;
|
||||
const token=response.headers.get('X-Session-Token')||'',cursor=response.headers.get('X-Down-Cursor')||'';
|
||||
if(!token||cursor!=='0'){await response.arrayBuffer();advanceCarrier('protocol',epoch);return}
|
||||
const welcome=await response.arrayBuffer();if(closed||epoch!==attemptEpoch)return;
|
||||
carrier=selected;sessionToken=token;cleanupToken=token;downCursor=cursor;
|
||||
if(!welcomeSent){welcomeSent=true;port.postMessage(welcome,[welcome])}
|
||||
maybeStartCandidate();
|
||||
}catch(error){if(closed||epoch!==attemptEpoch)return;advanceCarrier('network',epoch)}
|
||||
}
|
||||
async function probeHttp(probe,laneID,epoch){
|
||||
try{
|
||||
const headers={'X-Up-Seq':'1'},token=sessionToken,controller=attemptController,body=probe.data;if(laneID!==null)headers['X-Lane-ID']=String(laneID);
|
||||
const response=await request('/api/v1/up',options('POST',token,body,headers,controller.signal));
|
||||
if(closed||epoch!==attemptEpoch){await response.arrayBuffer();return}
|
||||
if(response.status!==204){await response.arrayBuffer();advanceCarrier('http',epoch);return}
|
||||
if(response.headers.get('X-Up-Ack')!=='1'){advanceCarrier('protocol',epoch);return}
|
||||
if(laneID===null)upSequence=2;else ensureLane(laneID).sequence=2;
|
||||
commitCarrier(probe,epoch);
|
||||
}catch(error){if(!closed&&epoch===attemptEpoch)advanceCarrier('network',epoch)}
|
||||
}
|
||||
function commitCarrier(probe,epoch){
|
||||
if(closed||carrierCommitted||epoch!==attemptEpoch)return;
|
||||
if(switching){fail();return}
|
||||
try{consumeProbe(probe)}catch(error){fail();return}
|
||||
carrierCommitted=true;candidateRunning=false;if(carrierTimer)clearTimeout(carrierTimer);carrierTimer=null;
|
||||
attemptController=null;currentAttempt=null;
|
||||
status('connected');
|
||||
if(carrier==='https')poll();
|
||||
else if(carrier==='https-lanes'){const lane=lanes.get(probe.id);if(lane&&!lane.polling)pollLane(lane)}
|
||||
for(const data of pending.splice(0)){release(data.byteLength,1,null);queueCarrier(data)}
|
||||
}
|
||||
function queueCarrier(data){
|
||||
try{
|
||||
if(carrier==='https')queueUp(data);
|
||||
else if(carrier==='websocket')queueSocket(data);
|
||||
else for(const value of splitFrames(data))queueLane(value);
|
||||
}catch(error){fail()}
|
||||
}
|
||||
function queueUp(data){if(!reserve(data,null)){fail();return}upPending.push(data);runUp()}
|
||||
async function runUp(){
|
||||
if(upRunning)return;upRunning=true;
|
||||
try{
|
||||
while(!closed&&sessionToken&&upPending.length){
|
||||
const batch=joinPending(upPending,null),sequence=String(upSequence);
|
||||
const response=await request('/api/v1/up',options('POST',sessionToken,batch.body,{'X-Up-Seq':sequence}));
|
||||
if(response.status!==204||response.headers.get('X-Up-Ack')!==sequence)throw new Error('uplink rejected');
|
||||
release(batch.total,batch.count,null);port.postMessage({t:'traffic',up:batch.total,down:0});upSequence++;
|
||||
}
|
||||
}catch(error){fail()}
|
||||
finally{upRunning=false;if(!closed&&sessionToken&&upPending.length)runUp()}
|
||||
}
|
||||
function openCandidateSocket(probe,laneID,epoch){
|
||||
const token=sessionToken,protocol=laneID===null?(negotiationEnabled?'tproxy-auto-v1.':'tproxy-v1.')+token:(negotiationEnabled?'tproxy-auto-lane-v1.':'tproxy-lane-v1.')+token+'.'+String(laneID);
|
||||
const next=new WebSocket(socketURL(),protocol);next.binaryType='arraybuffer';let opened=false,lane=null;
|
||||
if(laneID===null)socket=next;else{lane=ensureLane(laneID);lane.socket=next}
|
||||
next.onopen=()=>{
|
||||
if(closed||epoch!==attemptEpoch)return;opened=true;
|
||||
if(lane){lane.ready=true}else socketReady=true;
|
||||
try{next.send(probe.data);if(!negotiationEnabled)commitCarrier(probe,epoch)}catch(error){advanceCarrier('upgrade',epoch)}
|
||||
};
|
||||
next.onmessage=event=>{
|
||||
if(closed||epoch!==attemptEpoch||!(event.data instanceof ArrayBuffer))return;
|
||||
if(!carrierCommitted){if(event.data.byteLength!==0){advanceCarrier('protocol',epoch);return}commitCarrier(probe,epoch);return}
|
||||
try{
|
||||
if(lane){const values=splitFrames(event.data);for(const value of values)if(value.id!==lane.id)throw new Error('cross-lane frame');if(values.some(value=>value.type===3))lane.remoteClosed=true}
|
||||
else{const bound=frameBound(event.data,4096,batchLimit);if(bound.bytes!==event.data.byteLength)throw new Error('invalid frame batch')}
|
||||
}catch(error){if(lane)finishLane(lane,true);else fail();return}
|
||||
port.postMessage({t:'traffic',up:0,down:event.data.byteLength});port.postMessage(event.data,[event.data]);status('connected');
|
||||
};
|
||||
next.onerror=()=>{};
|
||||
next.onclose=()=>{
|
||||
if(epoch!==attemptEpoch||closed)return;
|
||||
if(!carrierCommitted){advanceCarrier(opened?'network':'upgrade',epoch);return}
|
||||
if(lane){lane.ready=false;lane.socket=null;finishLane(lane,true)}else{socketReady=false;fail()}
|
||||
};
|
||||
}
|
||||
function queueSocket(data){if(!reserve(data,null)){fail();return}upPending.push(data);runSocketUp()}
|
||||
async function waitSocket(next,size,limit){
|
||||
while(!closed&&next.readyState===WebSocket.OPEN&&next.bufferedAmount>limit-size)await pause(10);
|
||||
if(closed||next.readyState!==WebSocket.OPEN)throw new Error('websocket closed');
|
||||
}
|
||||
async function runSocketUp(){
|
||||
if(upRunning||!socketReady)return;upRunning=true;
|
||||
try{
|
||||
while(!closed&&socketReady&&upPending.length){
|
||||
const batch=joinPending(upPending,null);await waitSocket(socket,batch.total,queueLimit);socket.send(batch.body);
|
||||
release(batch.total,batch.count,null);port.postMessage({t:'traffic',up:batch.total,down:0});
|
||||
}
|
||||
}catch(error){if(!closed)fail()}
|
||||
finally{upRunning=false;if(!closed&&socketReady&&upPending.length)runSocketUp()}
|
||||
}
|
||||
async function poll(){
|
||||
while(!closed&&sessionToken){
|
||||
try{
|
||||
pollController=new AbortController();
|
||||
const response=await request('/api/v1/down',options('POST',sessionToken,null,{'X-Down-Cursor':downCursor},pollController.signal));
|
||||
if(response.status===204){status('connected');continue}
|
||||
if(response.status!==200)throw new Error('downlink rejected');
|
||||
const next=response.headers.get('X-Down-Cursor')||'',data=await response.arrayBuffer();
|
||||
if(!next||!data.byteLength)throw new Error('invalid downlink response');
|
||||
if(closed)return;
|
||||
port.postMessage({t:'traffic',up:0,down:data.byteLength});port.postMessage(data,[data]);downCursor=next;status('connected');
|
||||
}catch(error){if(!closed)fail();return}
|
||||
}
|
||||
}
|
||||
function ensureLane(id){
|
||||
let lane=lanes.get(id);
|
||||
if(!lane){lane={id,sequence:1,cursor:'0',pending:[],bytes:0,items:0,running:false,polling:false,controller:null,socket:null,ready:false,remoteClosed:false};lanes.set(id,lane)}
|
||||
return lane;
|
||||
}
|
||||
function rememberLaneClosed(id){
|
||||
if(!id||closedLanes.has(id))return;
|
||||
if(closedLaneOrder.length===closedLaneLimit)closedLanes.delete(closedLaneOrder.shift());
|
||||
closedLanes.add(id);closedLaneOrder.push(id);
|
||||
}
|
||||
function closeFrame(id){const value=new Uint8Array(8);value[0]=3;value[1]=(id>>>16)&255;value[2]=(id>>>8)&255;value[3]=id&255;return value.buffer}
|
||||
function finishLane(lane,notifyClient){
|
||||
if(lanes.get(lane.id)!==lane)return;
|
||||
if(lane.socket&&lane.socket.readyState<WebSocket.CLOSING)lane.socket.close();
|
||||
if(lane.bytes||lane.items)release(lane.bytes,lane.items,lane);
|
||||
lane.pending.length=0;lanes.delete(lane.id);rememberLaneClosed(lane.id);
|
||||
if(notifyClient&&!lane.remoteClosed&&port){const frame=closeFrame(lane.id);port.postMessage(frame,[frame])}
|
||||
}
|
||||
function queueLane(value){
|
||||
let lane=lanes.get(value.id);
|
||||
if(!lane&&(value.type===2||value.type===3||value.type===4))return;
|
||||
if(!lane&&closedLanes.has(value.id))throw new Error('closed lane was reused');
|
||||
if(!lane&&value.type!==1)throw new Error('lane did not begin with OPEN');
|
||||
lane=lane||ensureLane(value.id);
|
||||
if(!reserve(value.data,lane)){fail();return}
|
||||
lane.pending.push(value.data);
|
||||
if(carrier==='websocket-lanes'){openLaneSocket(lane);runLaneSocketUp(lane)}else runLaneUp(lane);
|
||||
}
|
||||
function openLaneSocket(lane){
|
||||
if(lane.socket||closed)return;lane.socket=new WebSocket(socketURL(),'tproxy-lane-v1.'+sessionToken+'.'+String(lane.id));lane.socket.binaryType='arraybuffer';
|
||||
lane.socket.onopen=()=>{if(closed||lanes.get(lane.id)!==lane)return;lane.ready=true;status('connected');runLaneSocketUp(lane)};
|
||||
lane.socket.onmessage=event=>{
|
||||
if(closed||lanes.get(lane.id)!==lane||!(event.data instanceof ArrayBuffer)){finishLane(lane,true);return}
|
||||
let values;try{values=splitFrames(event.data);for(const value of values)if(value.id!==lane.id)throw new Error('cross-lane frame')}catch(error){finishLane(lane,true);return}
|
||||
if(values.some(value=>value.type===3))lane.remoteClosed=true;
|
||||
port.postMessage({t:'traffic',up:0,down:event.data.byteLength});port.postMessage(event.data,[event.data]);status('connected');
|
||||
};
|
||||
lane.socket.onerror=()=>{};lane.socket.onclose=()=>{lane.ready=false;lane.socket=null;if(!closed)finishLane(lane,true)};
|
||||
}
|
||||
async function runLaneSocketUp(lane){
|
||||
if(lane.running||!lane.ready)return;lane.running=true;
|
||||
try{
|
||||
while(!closed&&lane.ready&&lanes.get(lane.id)===lane&&lane.pending.length){
|
||||
const batch=joinPending(lane.pending,lane);await waitSocket(lane.socket,batch.total,laneQueueLimit);lane.socket.send(batch.body);
|
||||
release(batch.total,batch.count,lane);port.postMessage({t:'traffic',up:batch.total,down:0});
|
||||
}
|
||||
}catch(error){if(!closed)finishLane(lane,true)}
|
||||
finally{lane.running=false;if(!closed&&lane.ready&&lane.pending.length)runLaneSocketUp(lane)}
|
||||
}
|
||||
async function runLaneUp(lane){
|
||||
if(lane.running)return;lane.running=true;
|
||||
try{
|
||||
while(!closed&&sessionToken&&lane.pending.length){
|
||||
const batch=joinPending(lane.pending,lane),sequence=String(lane.sequence),laneID=String(lane.id);
|
||||
const response=await request('/api/v1/up',options('POST',sessionToken,batch.body,{'X-Up-Seq':sequence,'X-Lane-ID':laneID}));
|
||||
if(response.status!==204||response.headers.get('X-Up-Ack')!==sequence)throw new Error('lane uplink rejected');
|
||||
release(batch.total,batch.count,lane);port.postMessage({t:'traffic',up:batch.total,down:0});lane.sequence++;
|
||||
if(!lane.polling)pollLane(lane);
|
||||
}
|
||||
}catch(error){fail()}
|
||||
finally{lane.running=false;if(!closed&&sessionToken&&lane.pending.length)runLaneUp(lane)}
|
||||
}
|
||||
async function pollLane(lane){
|
||||
if(!lane||lane.polling)return;lane.polling=true;
|
||||
try{
|
||||
while(!closed&&sessionToken&&lanes.get(lane.id)===lane){
|
||||
const controller=new AbortController(),laneID=String(lane.id);lane.controller=controller;
|
||||
const response=await request('/api/v1/down',options('POST',sessionToken,null,{'X-Down-Cursor':lane.cursor,'X-Lane-ID':laneID},controller.signal));
|
||||
if(response.status===204){
|
||||
if(response.headers.get('X-Lane-Closed')==='1'){finishLane(lane,false);return}
|
||||
status('connected');continue;
|
||||
}
|
||||
if(response.status!==200)throw new Error('lane downlink rejected');
|
||||
const next=response.headers.get('X-Down-Cursor')||'',data=await response.arrayBuffer();
|
||||
if(!next||!data.byteLength)throw new Error('invalid lane downlink response');
|
||||
for(const value of splitFrames(data))if(value.id!==lane.id)throw new Error('cross-lane frame');
|
||||
if(closed)return;
|
||||
port.postMessage({t:'traffic',up:0,down:data.byteLength});port.postMessage(data,[data]);lane.cursor=next;status('connected');
|
||||
}
|
||||
}catch(error){if(!closed)fail()}
|
||||
finally{lane.polling=false;lane.controller=null}
|
||||
}
|
||||
function deleteSession(){
|
||||
const token=cleanupToken||sessionToken;if(token)fetch(relayOrigin+'/api/v1/session',options('DELETE',token,null,null,undefined,true)).catch(()=>{});
|
||||
}
|
||||
function close(notifyServer){
|
||||
if(closed)return;closed=true;if(carrierTimer)clearTimeout(carrierTimer);if(attemptController)attemptController.abort();if(pollController)pollController.abort();
|
||||
if(socket)socket.close();for(const lane of lanes.values()){if(lane.controller)lane.controller.abort();if(lane.socket)lane.socket.close()}
|
||||
if(notifyServer)deleteSession();pending.length=0;upPending.length=0;
|
||||
for(const lane of lanes.values())lane.pending.length=0;lanes.clear();queuedBytes=0;queuedItems=0;if(port)port.close();
|
||||
}
|
||||
function activatePort(nextPort){
|
||||
initialized=true;port=nextPort;
|
||||
port.onmessage=message=>{
|
||||
if(message.data instanceof ArrayBuffer){
|
||||
if(!createStarted){createStarted=true;helloFrame=message.data;if(negotiationEnabled){negotiationStartedAt=Date.now();armCarrierDeadline(attemptEpoch)}createSession(attemptEpoch)}
|
||||
else if(!carrierCommitted){if(!reserve(message.data,null)){fail();return}pending.push(message.data);maybeStartCandidate()}
|
||||
else queueCarrier(message.data);
|
||||
}else if(message.data&&message.data.t==='close')close(true);
|
||||
};
|
||||
port.start();status('connecting');
|
||||
}
|
||||
addEventListener('message',event=>{
|
||||
if(initialized||event.source!==parent||event.data===null||typeof event.data!=='object')return;
|
||||
const keys=Object.keys(event.data).sort();
|
||||
if(keys.length!==2||keys[0]!=='t'||keys[1]!=='v'||event.data.t!=='tproxy-init'||event.data.v!==1||event.ports.length!==1)return;
|
||||
let source;try{source=new URL(event.origin)}catch(error){return}
|
||||
if(source.protocol!=='http:'||source.hostname!=='127.0.0.1'||!source.port||source.origin!==event.origin)return;
|
||||
activatePort(event.ports[0]);
|
||||
},{once:false});
|
||||
const androidBridge=globalThis.TelegramWebProxy;
|
||||
if(!initialized&&androidNonce&&androidBridge&&typeof androidBridge.postMessage==='function'){
|
||||
const androidPort={onmessage:null,start(){},close(){androidBridge.onmessage=null},postMessage(value){
|
||||
if(value instanceof ArrayBuffer){
|
||||
let frames;try{frames=splitFrames(value)}catch(error){fail();return}
|
||||
for(const frame of frames)androidBridge.postMessage(frame.data);
|
||||
}else androidBridge.postMessage(JSON.stringify(value));
|
||||
}};
|
||||
androidBridge.onmessage=event=>{let data=event.data;if(typeof data==='string'){try{data=JSON.parse(data)}catch(error){return}}if(androidPort.onmessage)androidPort.onmessage({data})};
|
||||
activatePort(androidPort);androidBridge.postMessage(JSON.stringify({t:'tproxy-android-init',v:1,nonce:androidNonce}));
|
||||
}
|
||||
addEventListener('pagehide',()=>close(true),{once:true});
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"##;
|
||||
const DOCUMENT: &str = include_str!("bridge/document.html");
|
||||
const RUNTIME: &str = include_str!("bridge/runtime.js");
|
||||
|
||||
// Rendered wire-contract tests remain separate from the embedded document.
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Connection</title>
|
||||
</head>
|
||||
<body>
|
||||
<script nonce="__NONCE__">
|
||||
__RUNTIME__
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,522 @@
|
||||
(()=>{
|
||||
'use strict';
|
||||
const bootstrap="__BOOTSTRAP__";
|
||||
const relayOrigin='https://__HOST__',carrierCapabilities='https,https-lanes,websocket,websocket-lanes';
|
||||
const negotiationEnabled=__NEGOTIATION_ENABLED__,candidateCount=__CANDIDATE_COUNT__,candidateDeadlines=[__CARRIER_DEADLINES__];
|
||||
const longPollMs=__LONG_POLL_SECS__*1000,bridgeRequestMs=__BRIDGE_REQUEST_SECS__*1000,bridgeRetryMs=__BRIDGE_RETRY_SECS__*1000;
|
||||
const probeCoalesceMs=__CARRIER_PROBE_COALESCE_MS__;
|
||||
let negotiatedCandidateCount=candidateCount,negotiatedFinalDeadline=candidateDeadlines[3],negotiatedFrozen=false;
|
||||
const batchLimit=__BATCH_LIMIT__,queueLimit=__QUEUE_LIMIT__,queueItemLimit=__QUEUE_ITEMS__;
|
||||
const laneQueueLimit=Math.min(queueLimit,8388608),laneItemLimit=Math.min(queueItemLimit,1024),closedLaneLimit=4096;
|
||||
const fragment=location.hash,androidNonce=/^#android=([A-Za-z0-9_-]{43})$/.exec(fragment)?.[1]||'';
|
||||
history.replaceState(null,'',location.pathname);
|
||||
let initialized=false,closed=false,port=null,sessionToken='',cleanupToken='',createStarted=false,socket=null,socketReady=false,carrier='';
|
||||
let queuedBytes=0,queuedItems=0,upSequence=1,downCursor='0',upRunning=false,upLease=null,pollController=null;
|
||||
let helloFrame=null,welcomeSent=false,carrierAttempt=1,carrierFailure='',carrierCommitted=false;
|
||||
let negotiationStartedAt=0,carrierTimer=null,probeTimer=null,attemptController=null,attemptEpoch=1,candidateRunning=false,switching=false,currentAttempt=null;
|
||||
const pending=[],upPending=[],lanes=new Map(),closedLanes=new Set(),closedLaneOrder=[];
|
||||
const status=state=>{if(port&&!closed)port.postMessage({t:'status',state})};
|
||||
const pause=(milliseconds,signal)=>new Promise((resolve,reject)=>{
|
||||
if(signal&&signal.aborted){reject(new Error('request aborted'));return}
|
||||
const timer=setTimeout(done,milliseconds);function done(){if(signal)signal.removeEventListener('abort',abort);resolve()}
|
||||
function abort(){clearTimeout(timer);signal.removeEventListener('abort',abort);reject(new Error('request aborted'))}
|
||||
if(signal)signal.addEventListener('abort',abort,{once:true});
|
||||
});
|
||||
const socketURL=()=>relayOrigin.replace(/^https:/,'wss:')+'/api/v1/ws';
|
||||
const options=(method,token,body,headers,signal,keepalive)=>({
|
||||
method,body,signal,keepalive:!!keepalive,mode:'same-origin',credentials:'omit',cache:'no-store',redirect:'error',referrerPolicy:'no-referrer',
|
||||
headers:Object.assign(token?{Authorization:'Bearer '+token}:{},body?{'Content-Type':'application/octet-stream'}:{},headers||{})
|
||||
});
|
||||
const attemptHeaders=(attempt,failure)=>negotiationEnabled?Object.assign({'X-Carrier-Capabilities':carrierCapabilities,'X-Carrier-Attempt':String(attempt)},failure?{'X-Carrier-Failure':failure}:{}):{};
|
||||
function reserve(data,lane){
|
||||
let buffered=socket?socket.bufferedAmount:0;for(const value of lanes.values())if(value.socket)buffered+=value.socket.bufferedAmount;
|
||||
if(!data.byteLength||data.byteLength>queueLimit-queuedBytes-buffered||queuedItems>=queueItemLimit)return false;
|
||||
if(lane&&(data.byteLength>laneQueueLimit-lane.bytes-(lane.socket?lane.socket.bufferedAmount:0)||lane.items>=laneItemLimit))return false;
|
||||
queuedBytes+=data.byteLength;queuedItems++;if(lane){lane.bytes+=data.byteLength;lane.items++}return true;
|
||||
}
|
||||
function release(bytes,items,lane){
|
||||
if(bytes>queuedBytes||items>queuedItems||(lane&&(bytes>lane.bytes||items>lane.items)))throw new Error('queue accounting invariant');
|
||||
queuedBytes-=bytes;queuedItems-=items;if(lane){lane.bytes-=bytes;lane.items-=items}
|
||||
}
|
||||
function releasePending(values,lane){
|
||||
if(!values.length)return;let bytes=0;for(const value of values)bytes+=value.byteLength;
|
||||
const items=values.length;values.length=0;release(bytes,items,lane);
|
||||
}
|
||||
function frameBound(value,maxFrames,maxBytes){
|
||||
const view=new DataView(value);let offset=0,frames=0;
|
||||
while(offset<value.byteLength){
|
||||
if(value.byteLength-offset<8)throw new Error('invalid frame batch');
|
||||
const size=view.getUint32(offset+4),end=offset+8+size;
|
||||
if(size>1048576||end>value.byteLength)throw new Error('invalid frame');
|
||||
if(frames>0&&(frames>=maxFrames||end>maxBytes))break;
|
||||
frames++;offset=end;
|
||||
}
|
||||
if(!frames)throw new Error('empty frame batch');
|
||||
return {frames,bytes:offset};
|
||||
}
|
||||
function splitFrames(value){
|
||||
const view=new DataView(value),result=[];let offset=0;
|
||||
while(offset<value.byteLength){
|
||||
if(value.byteLength-offset<8||result.length>=4096)throw new Error('invalid frame batch');
|
||||
const type=view.getUint8(offset),id=(view.getUint8(offset+1)<<16)|(view.getUint8(offset+2)<<8)|view.getUint8(offset+3);
|
||||
const size=view.getUint32(offset+4),end=offset+8+size;
|
||||
if((type===2&&!size)||size>1048576||end>value.byteLength)throw new Error('invalid frame');
|
||||
result.push({type,id,data:offset===0&&end===value.byteLength?value:value.slice(offset,end)});offset=end;
|
||||
}
|
||||
if(!result.length)throw new Error('empty frame batch');return result;
|
||||
}
|
||||
function probeFrames(){
|
||||
const result=[];let scanned=0;
|
||||
for(let index=0;index<pending.length;index++){
|
||||
const source=pending[index],view=new DataView(source);let start=0;
|
||||
while(start<source.byteLength){
|
||||
if(source.byteLength-start<8||result.length>=4096)throw new Error('invalid frame batch');
|
||||
const type=view.getUint8(start),id=(view.getUint8(start+1)<<16)|(view.getUint8(start+2)<<8)|view.getUint8(start+3);
|
||||
const size=view.getUint32(start+4),end=start+8+size,bytes=end-start;
|
||||
if((type===2&&!size)||size>1048576||end>source.byteLength)throw new Error('invalid frame');
|
||||
if(scanned+bytes>batchLimit)return result;
|
||||
result.push({source,index,start,end,type,id});scanned+=bytes;start=end;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
function findProbe(includeData){
|
||||
const frames=probeFrames(),first=frames.findIndex(frame=>frame.type===1||frame.type===2);if(first<0)return null;
|
||||
const laneMode=carrier==='https-lanes'||carrier==='websocket-lanes',selected=[];let hasData=frames[first].type===2;
|
||||
if(laneMode){
|
||||
selected.push(frames[first]);
|
||||
if(includeData&&!hasData)for(let index=first+1;index<frames.length;index++)if(frames[index].id===frames[first].id){selected.push(frames[index]);if(frames[index].type===2){hasData=true;break}}
|
||||
}else{
|
||||
let last=first;
|
||||
if(includeData&&!hasData)for(let index=first+1;index<frames.length;index++){if(frames[index].id===frames[first].id&&frames[index].type===2){last=index;hasData=true;break}}
|
||||
for(let index=0;index<=(hasData?last:first);index++)selected.push(frames[index]);
|
||||
}
|
||||
const spans=[];let total=0;
|
||||
for(const frame of selected){
|
||||
const previous=spans[spans.length-1];total+=frame.end-frame.start;
|
||||
if(previous&&previous.source===frame.source&&previous.end===frame.start)previous.end=frame.end;
|
||||
else spans.push({source:frame.source,index:frame.index,start:frame.start,end:frame.end});
|
||||
}
|
||||
const joined=new Uint8Array(total);let offset=0;
|
||||
for(const span of spans){const part=new Uint8Array(span.source,span.start,span.end-span.start);joined.set(part,offset);offset+=part.byteLength}
|
||||
return {spans,id:frames[first].id,data:joined.buffer,hasData};
|
||||
}
|
||||
function consumeProbe(probe){
|
||||
const groups=new Map();
|
||||
for(const span of probe.spans){if(pending[span.index]!==span.source)throw new Error('stale carrier probe');const values=groups.get(span.index)||[];values.push(span);groups.set(span.index,values)}
|
||||
const indexes=Array.from(groups.keys()).sort((left,right)=>right-left);
|
||||
for(const index of indexes){
|
||||
const source=pending[index],spans=groups.get(index).sort((left,right)=>left.start-right.start);let removed=0,offset=0;
|
||||
for(const span of spans){if(span.start<offset)throw new Error('overlapping carrier probe');removed+=span.end-span.start;offset=span.end}
|
||||
if(removed===source.byteLength){pending.splice(index,1);release(removed,1,null);continue}
|
||||
const merged=new Uint8Array(source.byteLength-removed);let write=0;offset=0;
|
||||
for(const span of spans){merged.set(new Uint8Array(source,offset,span.start-offset),write);write+=span.start-offset;offset=span.end}
|
||||
merged.set(new Uint8Array(source,offset),write);pending[index]=merged.buffer;release(removed,0,null);
|
||||
}
|
||||
}
|
||||
function joinPending(values,lane){
|
||||
let total=0,count=0,frames=0;
|
||||
while(count<values.length){
|
||||
const bound=frameBound(values[count],4096,batchLimit),whole=bound.bytes===values[count].byteLength;
|
||||
if(count===0&&!whole){
|
||||
const head=new Uint8Array(values[0],0,bound.bytes).slice();
|
||||
values[0]=values[0].slice(bound.bytes);queuedItems++;if(lane)lane.items++;
|
||||
return {body:head.buffer,total:bound.bytes,count:1};
|
||||
}
|
||||
if(count&&(total+values[count].byteLength>batchLimit||frames+bound.frames>4096))break;
|
||||
total+=values[count].byteLength;frames+=bound.frames;count++;
|
||||
}
|
||||
const joined=new Uint8Array(total);let offset=0;
|
||||
for(const data of values.splice(0,count)){joined.set(new Uint8Array(data),offset);offset+=data.byteLength}
|
||||
return {body:joined.buffer,total,count};
|
||||
}
|
||||
function takeBatch(values,lane){
|
||||
const batch=joinPending(values,lane);
|
||||
return Object.assign(batch,{lane,controller:null,cancelled:false,settled:false});
|
||||
}
|
||||
function settleBatch(lease){
|
||||
if(!lease||lease.settled)return false;lease.settled=true;
|
||||
if(lease.lane){if(lease.lane.upLease===lease)lease.lane.upLease=null}else if(upLease===lease)upLease=null;
|
||||
release(lease.total,lease.count,lease.lane);return true;
|
||||
}
|
||||
function cancelBatch(lease){
|
||||
if(!lease||lease.settled)return;lease.cancelled=true;if(lease.controller)lease.controller.abort();settleBatch(lease);
|
||||
}
|
||||
function retryAfterMs(response){
|
||||
const header=response.headers.get('Retry-After');
|
||||
if(!header)return 0;
|
||||
const seconds=Number(header);
|
||||
if(Number.isFinite(seconds)&&seconds>=0)return Math.min(seconds*1000,30000);
|
||||
const when=Date.parse(header);
|
||||
if(Number.isFinite(when)){const delta=when-Date.now();return delta>0?Math.min(delta,30000):0}
|
||||
return 0;
|
||||
}
|
||||
function retryableStatus(status){return status===408||status===429||status===502||status===503||status===504}
|
||||
async function request(path,frozenOptions){
|
||||
let delay=250,attempt=0;const deadline=Date.now()+bridgeRetryMs,external=frozenOptions.signal;
|
||||
const attemptLimit=path==='/api/v1/down'?longPollMs+bridgeRequestMs:bridgeRequestMs;
|
||||
while(attempt<9){
|
||||
if(closed||(external&&external.aborted))throw new Error('request aborted');
|
||||
const remaining=deadline-Date.now();if(remaining<=0)break;attempt++;
|
||||
const controller=new AbortController(),abort=()=>controller.abort();
|
||||
if(external)external.addEventListener('abort',abort,{once:true});
|
||||
const requestOptions=Object.assign({},frozenOptions,{signal:controller.signal});
|
||||
const timer=setTimeout(abort,Math.max(1,Math.min(attemptLimit,remaining)));
|
||||
let response=null,wait=0;
|
||||
try{
|
||||
const fetched=await fetch(relayOrigin+path,requestOptions),body=await fetched.arrayBuffer();
|
||||
response={status:fetched.status,headers:fetched.headers,body};
|
||||
if(!retryableStatus(response.status))return response;
|
||||
wait=retryAfterMs(response);
|
||||
}catch(error){
|
||||
if(closed||(external&&external.aborted))throw error;
|
||||
}finally{clearTimeout(timer);if(external)external.removeEventListener('abort',abort)}
|
||||
const after=deadline-Date.now();if(attempt>=9||after<=0)break;
|
||||
status('reconnecting');
|
||||
const backoff=wait||delay+Math.floor(Math.random()*Math.max(1,delay/4));
|
||||
await pause(Math.min(backoff,after),external);delay=Math.min(delay*2,5000);
|
||||
}
|
||||
throw new Error('carrier retry limit reached');
|
||||
}
|
||||
function fail(){if(closed)return;status('failed');if(port)port.postMessage({t:'close'});close(true)}
|
||||
function knownCarrier(value){return value==='https'||value==='https-lanes'||value==='websocket'||value==='websocket-lanes'}
|
||||
function sessionEcho(response,expectedAttempt,states,exactAttempt){
|
||||
const selected=response.headers.get('X-Carrier-Mode')||'',echo=response.headers.get('X-Carrier-Attempt')||'';
|
||||
if(!knownCarrier(selected))throw new Error('invalid carrier mode');
|
||||
if(!negotiationEnabled){if(echo!=='')throw new Error('unexpected carrier attempt');return {selected,state:''}}
|
||||
const count=response.headers.get('X-Carrier-Candidate-Count')||'',deadline=response.headers.get('X-Carrier-Deadline')||'',state=response.headers.get('X-Carrier-State')||'';
|
||||
if(!/^[1-4]$/.test(count)||!/^[1-9]\d*$/.test(deadline)||!states.includes(state))throw new Error('invalid carrier state');
|
||||
const echoedAttempt=Number(echo),parsedCount=Number(count),parsedDeadline=Number(deadline);
|
||||
if(!Number.isInteger(echoedAttempt)||echoedAttempt<1||(exactAttempt?echoedAttempt!==expectedAttempt:echoedAttempt>expectedAttempt))throw new Error('invalid carrier attempt');
|
||||
if(parsedCount>candidateCount||parsedDeadline>candidateDeadlines[3])throw new Error('invalid carrier bounds');
|
||||
if(!negotiatedFrozen){negotiatedCandidateCount=parsedCount;negotiatedFinalDeadline=parsedDeadline;negotiatedFrozen=true}
|
||||
else if(parsedCount!==negotiatedCandidateCount||parsedDeadline!==negotiatedFinalDeadline)throw new Error('changed carrier bounds');
|
||||
if(echoedAttempt>negotiatedCandidateCount)throw new Error('carrier attempt exceeds candidates');
|
||||
return {selected,state};
|
||||
}
|
||||
function armCarrierDeadline(epoch){
|
||||
if(!negotiationStartedAt||epoch!==attemptEpoch)return;
|
||||
if(carrierTimer)clearTimeout(carrierTimer);
|
||||
const deadline=carrierAttempt>=negotiatedCandidateCount?negotiatedFinalDeadline:candidateDeadlines[carrierAttempt-1];
|
||||
const remaining=negotiationStartedAt+deadline*1000-Date.now();
|
||||
carrierTimer=setTimeout(()=>advanceCarrier('timeout',epoch),Math.max(0,remaining));
|
||||
}
|
||||
function clearProbeTimer(){if(probeTimer){clearTimeout(probeTimer.timer);probeTimer=null}}
|
||||
function resetCandidate(){
|
||||
clearProbeTimer();
|
||||
if(pollController)pollController.abort();pollController=null;
|
||||
if(socket){const previous=socket;socket=null;previous.close()}socketReady=false;
|
||||
cancelBatch(upLease);releasePending(upPending,null);
|
||||
for(const lane of lanes.values()){
|
||||
if(lane.controller)lane.controller.abort();cancelBatch(lane.upLease);releasePending(lane.pending,lane);if(lane.socket)lane.socket.close();
|
||||
}
|
||||
lanes.clear();closedLanes.clear();closedLaneOrder.length=0;upSequence=1;downCursor='0';upRunning=false;
|
||||
sessionToken='';carrier='';candidateRunning=false;currentAttempt=null;
|
||||
}
|
||||
function advanceConfirmed(reason,epoch){
|
||||
if(closed||carrierCommitted||epoch!==attemptEpoch)return;
|
||||
resetCandidate();
|
||||
if(carrierAttempt>=negotiatedCandidateCount||Date.now()>=negotiationStartedAt+negotiatedFinalDeadline*1000){switching=false;fail();return}
|
||||
carrierAttempt++;carrierFailure=reason;attemptEpoch++;const nextEpoch=attemptEpoch;switching=false;
|
||||
status('reconnecting');armCarrierDeadline(nextEpoch);createSession(nextEpoch);
|
||||
}
|
||||
function advanceCarrier(reason,epoch){
|
||||
if(closed||carrierCommitted||epoch!==attemptEpoch||switching)return;
|
||||
if(!negotiationEnabled){fail();return}
|
||||
switching=true;if(carrierTimer)clearTimeout(carrierTimer);carrierTimer=null;clearProbeTimer();
|
||||
const snapshot=currentAttempt;if(attemptController)attemptController.abort();attemptController=null;
|
||||
if(!snapshot||snapshot.epoch!==epoch){switching=false;fail();return}
|
||||
if(snapshot.selected){advanceConfirmed(reason,epoch);return}
|
||||
resolveAttempt(reason,epoch,snapshot);
|
||||
}
|
||||
async function resolveAttempt(reason,epoch,snapshot){
|
||||
const controller=new AbortController();attemptController=controller;
|
||||
const remaining=negotiationStartedAt+negotiatedFinalDeadline*1000-Date.now();
|
||||
if(remaining<=0){switching=false;fail();return}
|
||||
const timer=setTimeout(()=>controller.abort(),remaining);
|
||||
try{
|
||||
const frozen=options('POST',bootstrap,snapshot.hello,attemptHeaders(snapshot.attempt,snapshot.failure),controller.signal);
|
||||
const response=await request('/api/v1/session',frozen);
|
||||
if(closed||epoch!==attemptEpoch)return
|
||||
if(response.status===409){sessionEcho(response,snapshot.attempt,['committed','healthy'],false);switching=false;fail();return}
|
||||
if(response.status!==200){switching=false;fail();return}
|
||||
const echo=sessionEcho(response,snapshot.attempt,['provisional','committed','healthy'],true);
|
||||
const token=response.headers.get('X-Session-Token')||'',cursor=response.headers.get('X-Down-Cursor')||'';
|
||||
if(!token||cursor!=='0'||(snapshot.selected&&echo.selected!==snapshot.selected))throw new Error('changed carrier replay');
|
||||
const welcome=response.body;if(closed||epoch!==attemptEpoch)return;
|
||||
cleanupToken=token;
|
||||
if(!welcomeSent){welcomeSent=true;port.postMessage(welcome,[welcome])}
|
||||
if(echo.state!=='provisional'){switching=false;fail();return}
|
||||
advanceConfirmed(reason,epoch);
|
||||
}catch(error){if(!closed&&epoch===attemptEpoch){switching=false;fail()}}
|
||||
finally{clearTimeout(timer);if(attemptController===controller)attemptController=null}
|
||||
}
|
||||
function startCandidate(probe,epoch){
|
||||
if(!probe||closed||carrierCommitted||!sessionToken||candidateRunning||epoch!==attemptEpoch)return;
|
||||
clearProbeTimer();candidateRunning=true;
|
||||
if(carrier==='https')probeHttp(probe,null,epoch);
|
||||
else if(carrier==='https-lanes')probeHttp(probe,probe.id,epoch);
|
||||
else if(carrier==='websocket')openCandidateSocket(probe,null,epoch);
|
||||
else if(carrier==='websocket-lanes')openCandidateSocket(probe,probe.id,epoch);
|
||||
else advanceCarrier('protocol',epoch);
|
||||
}
|
||||
function maybeStartCandidate(){
|
||||
if(closed||carrierCommitted||!sessionToken||candidateRunning)return;const epoch=attemptEpoch;
|
||||
let probe;try{probe=findProbe(probeCoalesceMs>0)}catch(error){fail();return}if(!probe)return;
|
||||
if(!probeCoalesceMs||probe.hasData){startCandidate(probe,epoch);return}
|
||||
if(probeTimer)return;const owner={epoch,timer:null};
|
||||
owner.timer=setTimeout(()=>{if(probeTimer!==owner||closed||owner.epoch!==attemptEpoch)return;probeTimer=null;let current;try{current=findProbe(false)}catch(error){fail();return}startCandidate(current,owner.epoch)},probeCoalesceMs);
|
||||
probeTimer=owner;
|
||||
}
|
||||
async function createSession(epoch){
|
||||
const controller=new AbortController(),attempt=carrierAttempt,failure=carrierFailure;
|
||||
const snapshot={epoch,attempt,failure,hello:helloFrame,selected:''};currentAttempt=snapshot;attemptController=controller;
|
||||
try{
|
||||
status('connecting');
|
||||
const frozen=options('POST',bootstrap,snapshot.hello,attemptHeaders(attempt,failure),controller.signal);
|
||||
const response=await request('/api/v1/session',frozen);
|
||||
if(closed||epoch!==attemptEpoch)return
|
||||
if(response.status===409){sessionEcho(response,attempt,['committed','healthy'],false);fail();return}
|
||||
if(response.status!==200){advanceCarrier('http',epoch);return}
|
||||
const echo=sessionEcho(response,attempt,['provisional'],true),selected=echo.selected;snapshot.selected=selected;
|
||||
const token=response.headers.get('X-Session-Token')||'',cursor=response.headers.get('X-Down-Cursor')||'';
|
||||
if(!token||cursor!=='0'){advanceCarrier('protocol',epoch);return}
|
||||
const welcome=response.body;if(closed||epoch!==attemptEpoch)return;
|
||||
carrier=selected;sessionToken=token;cleanupToken=token;downCursor=cursor;
|
||||
if(!welcomeSent){welcomeSent=true;port.postMessage(welcome,[welcome])}
|
||||
if(carrier==='websocket')openCandidateSocket(null,null,epoch);
|
||||
maybeStartCandidate();
|
||||
}catch(error){if(closed||epoch!==attemptEpoch)return;advanceCarrier('network',epoch)}
|
||||
}
|
||||
async function probeHttp(probe,laneID,epoch){
|
||||
try{
|
||||
const headers={'X-Up-Seq':'1'},token=sessionToken,controller=attemptController,body=probe.data;if(laneID!==null)headers['X-Lane-ID']=String(laneID);
|
||||
const response=await request('/api/v1/up',options('POST',token,body,headers,controller.signal));
|
||||
if(closed||epoch!==attemptEpoch)return
|
||||
if(response.status!==204){advanceCarrier('http',epoch);return}
|
||||
if(response.headers.get('X-Up-Ack')!=='1'){advanceCarrier('protocol',epoch);return}
|
||||
if(laneID===null)upSequence=2;else ensureLane(laneID).sequence=2;
|
||||
commitCarrier(probe,epoch);
|
||||
}catch(error){if(!closed&&epoch===attemptEpoch)advanceCarrier('network',epoch)}
|
||||
}
|
||||
function commitCarrier(probe,epoch){
|
||||
if(closed||carrierCommitted||epoch!==attemptEpoch)return;
|
||||
if(switching){fail();return}
|
||||
clearProbeTimer();try{consumeProbe(probe)}catch(error){fail();return}
|
||||
carrierCommitted=true;candidateRunning=false;if(carrierTimer)clearTimeout(carrierTimer);carrierTimer=null;
|
||||
attemptController=null;currentAttempt=null;
|
||||
status('connected');
|
||||
if(carrier==='https')poll();
|
||||
else if(carrier==='https-lanes'){const lane=lanes.get(probe.id);if(lane&&!lane.polling)pollLane(lane)}
|
||||
for(const data of pending.splice(0)){release(data.byteLength,1,null);queueCarrier(data)}
|
||||
}
|
||||
function queueCarrier(data){
|
||||
try{
|
||||
if(carrier==='https')queueUp(data);
|
||||
else if(carrier==='websocket')queueSocket(data);
|
||||
else for(const value of splitFrames(data))queueLane(value);
|
||||
}catch(error){fail()}
|
||||
}
|
||||
function queueUp(data){if(!reserve(data,null)){fail();return}upPending.push(data);runUp()}
|
||||
async function runUp(){
|
||||
if(upRunning)return;upRunning=true;let lease=null;
|
||||
try{
|
||||
while(!closed&&sessionToken&&upPending.length){
|
||||
lease=takeBatch(upPending,null);upLease=lease;lease.controller=new AbortController();const sequence=String(upSequence);
|
||||
const response=await request('/api/v1/up',options('POST',sessionToken,lease.body,{'X-Up-Seq':sequence},lease.controller.signal));
|
||||
if(response.status!==204||response.headers.get('X-Up-Ack')!==sequence)throw new Error('uplink rejected');
|
||||
if(!settleBatch(lease))return;port.postMessage({t:'traffic',up:lease.total,down:0});upSequence++;lease=null;
|
||||
}
|
||||
}catch(error){if(!closed&&!(lease&&lease.cancelled))fail()}
|
||||
finally{upRunning=false;if(!closed&&sessionToken&&upPending.length)runUp()}
|
||||
}
|
||||
function sendCandidateSocket(next){
|
||||
const state=next.telemt;if(!state||state.sent||next.readyState!==WebSocket.OPEN||!state.probe)return;
|
||||
let probe=state.probe;
|
||||
try{const fresh=findProbe(true);if(fresh&&fresh.id===probe.id)probe=fresh;next.send(probe.data)}catch(error){advanceCarrier('upgrade',state.epoch);return}
|
||||
state.probe=probe;state.sent=true;if(!negotiationEnabled)commitCarrier(probe,state.epoch);
|
||||
}
|
||||
function openCandidateSocket(probe,laneID,epoch){
|
||||
let lane=laneID===null?null:ensureLane(laneID),next=lane?lane.socket:socket;
|
||||
if(next){if(!next.telemt||next.telemt.epoch!==epoch){advanceCarrier('protocol',epoch);return}if(probe)next.telemt.probe=probe;sendCandidateSocket(next);return}
|
||||
const token=sessionToken,protocol=laneID===null?(negotiationEnabled?'tproxy-auto-v1.':'tproxy-v1.')+token:(negotiationEnabled?'tproxy-auto-lane-v1.':'tproxy-lane-v1.')+token+'.'+String(laneID);
|
||||
next=new WebSocket(socketURL(),protocol);next.binaryType='arraybuffer';next.telemt={epoch,lane,probe,opened:false,sent:false};
|
||||
if(lane)lane.socket=next;else socket=next;
|
||||
next.onopen=()=>{
|
||||
const state=next.telemt;if(closed||state.epoch!==attemptEpoch){next.close();return}state.opened=true;
|
||||
if(state.lane){state.lane.ready=true}else socketReady=true;sendCandidateSocket(next);
|
||||
};
|
||||
next.onmessage=event=>{
|
||||
const state=next.telemt;if(closed||state.epoch!==attemptEpoch||!(event.data instanceof ArrayBuffer))return;
|
||||
if(!carrierCommitted){if(!state.sent||event.data.byteLength!==0){advanceCarrier('protocol',state.epoch);return}commitCarrier(state.probe,state.epoch);return}
|
||||
try{
|
||||
if(state.lane){const values=splitFrames(event.data);for(const value of values)if(value.id!==state.lane.id)throw new Error('cross-lane frame');if(values.some(value=>value.type===3))state.lane.remoteClosed=true}
|
||||
else{const bound=frameBound(event.data,4096,batchLimit);if(bound.bytes!==event.data.byteLength)throw new Error('invalid frame batch')}
|
||||
}catch(error){if(state.lane)finishLane(state.lane,true);else fail();return}
|
||||
port.postMessage({t:'traffic',up:0,down:event.data.byteLength});port.postMessage(event.data,[event.data]);status('connected');
|
||||
};
|
||||
next.onerror=()=>{};
|
||||
next.onclose=()=>{
|
||||
const state=next.telemt;if(state.epoch!==attemptEpoch||closed)return;
|
||||
if(!carrierCommitted){advanceCarrier(state.opened?'network':'upgrade',state.epoch);return}
|
||||
if(state.lane){state.lane.ready=false;state.lane.socket=null;finishLane(state.lane,true)}else{socketReady=false;fail()}
|
||||
};
|
||||
}
|
||||
function queueSocket(data){if(!reserve(data,null)){fail();return}upPending.push(data);runSocketUp()}
|
||||
async function waitSocket(next,size,limit,signal){
|
||||
while(!closed&&next.readyState===WebSocket.OPEN&&next.bufferedAmount>limit-size)await pause(10,signal);
|
||||
if(closed||(signal&&signal.aborted)||next.readyState!==WebSocket.OPEN)throw new Error('websocket closed');
|
||||
}
|
||||
async function runSocketUp(){
|
||||
if(upRunning||!socketReady)return;upRunning=true;let lease=null;
|
||||
try{
|
||||
while(!closed&&socketReady&&upPending.length){
|
||||
lease=takeBatch(upPending,null);upLease=lease;lease.controller=new AbortController();
|
||||
await waitSocket(socket,lease.total,queueLimit,lease.controller.signal);socket.send(lease.body);
|
||||
if(!settleBatch(lease))return;port.postMessage({t:'traffic',up:lease.total,down:0});lease=null;
|
||||
}
|
||||
}catch(error){if(!closed&&!(lease&&lease.cancelled))fail()}
|
||||
finally{upRunning=false;if(!closed&&socketReady&&upPending.length)runSocketUp()}
|
||||
}
|
||||
async function poll(){
|
||||
while(!closed&&sessionToken){
|
||||
try{
|
||||
pollController=new AbortController();
|
||||
const response=await request('/api/v1/down',options('POST',sessionToken,null,{'X-Down-Cursor':downCursor},pollController.signal));
|
||||
if(response.status===204){status('connected');continue}
|
||||
if(response.status!==200)throw new Error('downlink rejected');
|
||||
const next=response.headers.get('X-Down-Cursor')||'',data=response.body;
|
||||
if(!next||!data.byteLength)throw new Error('invalid downlink response');
|
||||
if(closed)return;
|
||||
port.postMessage({t:'traffic',up:0,down:data.byteLength});port.postMessage(data,[data]);downCursor=next;status('connected');
|
||||
}catch(error){if(!closed)fail();return}
|
||||
}
|
||||
}
|
||||
function ensureLane(id){
|
||||
let lane=lanes.get(id);
|
||||
if(!lane){lane={id,sequence:1,cursor:'0',pending:[],bytes:0,items:0,running:false,upLease:null,polling:false,controller:null,socket:null,ready:false,remoteClosed:false};lanes.set(id,lane)}
|
||||
return lane;
|
||||
}
|
||||
function rememberLaneClosed(id){
|
||||
if(!id||closedLanes.has(id))return;
|
||||
if(closedLaneOrder.length===closedLaneLimit)closedLanes.delete(closedLaneOrder.shift());
|
||||
closedLanes.add(id);closedLaneOrder.push(id);
|
||||
}
|
||||
function closeFrame(id){const value=new Uint8Array(8);value[0]=3;value[1]=(id>>>16)&255;value[2]=(id>>>8)&255;value[3]=id&255;return value.buffer}
|
||||
function finishLane(lane,notifyClient){
|
||||
if(lanes.get(lane.id)!==lane)return;
|
||||
if(lane.controller)lane.controller.abort();lane.controller=null;cancelBatch(lane.upLease);
|
||||
if(lane.socket&&lane.socket.readyState<WebSocket.CLOSING)lane.socket.close();
|
||||
releasePending(lane.pending,lane);lanes.delete(lane.id);rememberLaneClosed(lane.id);
|
||||
if(notifyClient&&!lane.remoteClosed&&port){const frame=closeFrame(lane.id);port.postMessage(frame,[frame])}
|
||||
}
|
||||
function queueLane(value){
|
||||
let lane=lanes.get(value.id);
|
||||
if(!lane&&(value.type===2||value.type===3||value.type===4))return;
|
||||
if(!lane&&closedLanes.has(value.id))throw new Error('closed lane was reused');
|
||||
if(!lane&&value.type!==1)throw new Error('lane did not begin with OPEN');
|
||||
lane=lane||ensureLane(value.id);
|
||||
if(!reserve(value.data,lane)){fail();return}
|
||||
lane.pending.push(value.data);
|
||||
if(carrier==='websocket-lanes'){openLaneSocket(lane);runLaneSocketUp(lane)}else runLaneUp(lane);
|
||||
}
|
||||
function openLaneSocket(lane){
|
||||
if(lane.socket||closed)return;lane.socket=new WebSocket(socketURL(),'tproxy-lane-v1.'+sessionToken+'.'+String(lane.id));lane.socket.binaryType='arraybuffer';
|
||||
lane.socket.onopen=()=>{if(closed||lanes.get(lane.id)!==lane)return;lane.ready=true;status('connected');runLaneSocketUp(lane)};
|
||||
lane.socket.onmessage=event=>{
|
||||
if(closed||lanes.get(lane.id)!==lane||!(event.data instanceof ArrayBuffer)){finishLane(lane,true);return}
|
||||
let values;try{values=splitFrames(event.data);for(const value of values)if(value.id!==lane.id)throw new Error('cross-lane frame')}catch(error){finishLane(lane,true);return}
|
||||
if(values.some(value=>value.type===3))lane.remoteClosed=true;
|
||||
port.postMessage({t:'traffic',up:0,down:event.data.byteLength});port.postMessage(event.data,[event.data]);status('connected');
|
||||
};
|
||||
lane.socket.onerror=()=>{};lane.socket.onclose=()=>{lane.ready=false;lane.socket=null;if(!closed)finishLane(lane,true)};
|
||||
}
|
||||
async function runLaneSocketUp(lane){
|
||||
if(lane.running||!lane.ready)return;lane.running=true;let lease=null;
|
||||
try{
|
||||
while(!closed&&lane.ready&&lanes.get(lane.id)===lane&&lane.pending.length){
|
||||
lease=takeBatch(lane.pending,lane);lane.upLease=lease;lease.controller=new AbortController();
|
||||
await waitSocket(lane.socket,lease.total,laneQueueLimit,lease.controller.signal);lane.socket.send(lease.body);
|
||||
if(!settleBatch(lease))return;port.postMessage({t:'traffic',up:lease.total,down:0});lease=null;
|
||||
}
|
||||
}catch(error){if(!closed&&lanes.get(lane.id)===lane&&!(lease&&lease.cancelled))finishLane(lane,true)}
|
||||
finally{lane.running=false;if(!closed&&lanes.get(lane.id)===lane&&lane.ready&&lane.pending.length)runLaneSocketUp(lane)}
|
||||
}
|
||||
async function runLaneUp(lane){
|
||||
if(lane.running)return;lane.running=true;let lease=null;
|
||||
try{
|
||||
while(!closed&&sessionToken&&lane.pending.length){
|
||||
lease=takeBatch(lane.pending,lane);lane.upLease=lease;lease.controller=new AbortController();
|
||||
const sequence=String(lane.sequence),laneID=String(lane.id);
|
||||
const response=await request('/api/v1/up',options('POST',sessionToken,lease.body,{'X-Up-Seq':sequence,'X-Lane-ID':laneID},lease.controller.signal));
|
||||
if(response.status!==204||response.headers.get('X-Up-Ack')!==sequence)throw new Error('lane uplink rejected');
|
||||
if(!settleBatch(lease))return;port.postMessage({t:'traffic',up:lease.total,down:0});lane.sequence++;lease=null;
|
||||
if(!lane.polling)pollLane(lane);
|
||||
}
|
||||
}catch(error){if(!closed&&lanes.get(lane.id)===lane&&!(lease&&lease.cancelled))fail()}
|
||||
finally{lane.running=false;if(!closed&&lanes.get(lane.id)===lane&&sessionToken&&lane.pending.length)runLaneUp(lane)}
|
||||
}
|
||||
async function pollLane(lane){
|
||||
if(!lane||lane.polling)return;lane.polling=true;
|
||||
try{
|
||||
while(!closed&&sessionToken&&lanes.get(lane.id)===lane){
|
||||
const controller=new AbortController(),laneID=String(lane.id);lane.controller=controller;
|
||||
const response=await request('/api/v1/down',options('POST',sessionToken,null,{'X-Down-Cursor':lane.cursor,'X-Lane-ID':laneID},controller.signal));
|
||||
if(response.status===204){
|
||||
if(response.headers.get('X-Lane-Closed')==='1'){finishLane(lane,false);return}
|
||||
status('connected');continue;
|
||||
}
|
||||
if(response.status!==200)throw new Error('lane downlink rejected');
|
||||
const next=response.headers.get('X-Down-Cursor')||'',data=response.body;
|
||||
if(!next||!data.byteLength)throw new Error('invalid lane downlink response');
|
||||
for(const value of splitFrames(data))if(value.id!==lane.id)throw new Error('cross-lane frame');
|
||||
if(closed)return;
|
||||
port.postMessage({t:'traffic',up:0,down:data.byteLength});port.postMessage(data,[data]);lane.cursor=next;status('connected');
|
||||
}
|
||||
}catch(error){if(!closed)fail()}
|
||||
finally{lane.polling=false;lane.controller=null}
|
||||
}
|
||||
function deleteSession(){
|
||||
const token=cleanupToken||sessionToken;if(token)fetch(relayOrigin+'/api/v1/session',options('DELETE',token,null,null,undefined,true)).catch(()=>{});
|
||||
}
|
||||
function close(notifyServer){
|
||||
if(closed)return;closed=true;if(carrierTimer)clearTimeout(carrierTimer);clearProbeTimer();if(attemptController)attemptController.abort();if(pollController)pollController.abort();
|
||||
if(socket)socket.close();cancelBatch(upLease);releasePending(upPending,null);
|
||||
for(const lane of lanes.values()){
|
||||
if(lane.controller)lane.controller.abort();cancelBatch(lane.upLease);releasePending(lane.pending,lane);if(lane.socket)lane.socket.close();
|
||||
}
|
||||
if(notifyServer)deleteSession();releasePending(pending,null);lanes.clear();if(port)port.close();
|
||||
if(queuedBytes!==0||queuedItems!==0)throw new Error('queue accounting leak');
|
||||
}
|
||||
function activatePort(nextPort){
|
||||
initialized=true;port=nextPort;
|
||||
port.onmessage=message=>{
|
||||
if(message.data instanceof ArrayBuffer){
|
||||
if(!createStarted){createStarted=true;helloFrame=message.data;if(negotiationEnabled){negotiationStartedAt=Date.now();armCarrierDeadline(attemptEpoch)}createSession(attemptEpoch)}
|
||||
else if(!carrierCommitted){if(!reserve(message.data,null)){fail();return}pending.push(message.data);maybeStartCandidate()}
|
||||
else queueCarrier(message.data);
|
||||
}else if(message.data&&message.data.t==='close')close(true);
|
||||
};
|
||||
port.start();status('connecting');
|
||||
}
|
||||
addEventListener('message',event=>{
|
||||
if(initialized||event.source!==parent||event.data===null||typeof event.data!=='object')return;
|
||||
const keys=Object.keys(event.data).sort();
|
||||
if(keys.length!==2||keys[0]!=='t'||keys[1]!=='v'||event.data.t!=='tproxy-init'||event.data.v!==1||event.ports.length!==1)return;
|
||||
let source;try{source=new URL(event.origin)}catch(error){return}
|
||||
if(source.protocol!=='http:'||source.hostname!=='127.0.0.1'||!source.port||source.origin!==event.origin)return;
|
||||
activatePort(event.ports[0]);
|
||||
},{once:false});
|
||||
const androidBridge=globalThis.TelegramWebProxy;
|
||||
if(!initialized&&androidNonce&&androidBridge&&typeof androidBridge.postMessage==='function'){
|
||||
const androidPort={onmessage:null,start(){},close(){androidBridge.onmessage=null},postMessage(value){
|
||||
if(value instanceof ArrayBuffer){
|
||||
let frames;try{frames=splitFrames(value)}catch(error){fail();return}
|
||||
for(const frame of frames)androidBridge.postMessage(frame.data);
|
||||
}else androidBridge.postMessage(JSON.stringify(value));
|
||||
}};
|
||||
androidBridge.onmessage=event=>{let data=event.data;if(typeof data==='string'){try{data=JSON.parse(data)}catch(error){return}}if(androidPort.onmessage)androidPort.onmessage({data})};
|
||||
activatePort(androidPort);androidBridge.postMessage(JSON.stringify({t:'tproxy-android-init',v:1,nonce:androidNonce}));
|
||||
}
|
||||
addEventListener('pagehide',()=>close(true),{once:true});
|
||||
})();
|
||||
@@ -10,6 +10,10 @@ fn render_page(bootstrap: &str, candidate_count: usize) -> BridgePage {
|
||||
true,
|
||||
candidate_count,
|
||||
[3, 5, 8, 12],
|
||||
25,
|
||||
10,
|
||||
90,
|
||||
0,
|
||||
&SecureRandom::new(),
|
||||
)
|
||||
}
|
||||
@@ -43,6 +47,30 @@ fn rendered_page_preserves_the_ios_bootstrap_literal() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rendered_page_embeds_the_configured_bridge_timing_policy() {
|
||||
let page = render(
|
||||
"proxy.example.com",
|
||||
"GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG",
|
||||
2 * 1024 * 1024,
|
||||
32 * 1024 * 1024,
|
||||
16 * 1024,
|
||||
true,
|
||||
4,
|
||||
[3, 5, 8, 12],
|
||||
17,
|
||||
7,
|
||||
41,
|
||||
4,
|
||||
&SecureRandom::new(),
|
||||
);
|
||||
|
||||
assert!(page.body.contains("const longPollMs=17*1000"));
|
||||
assert!(page.body.contains("bridgeRequestMs=7*1000"));
|
||||
assert!(page.body.contains("bridgeRetryMs=41*1000"));
|
||||
assert!(page.body.contains("const probeCoalesceMs=4"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn effective_deadline_formula_uses_the_final_checkpoint() {
|
||||
let page = render_page("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC", 3);
|
||||
@@ -67,6 +95,10 @@ fn disabled_negotiation_does_not_arm_a_carrier_deadline() {
|
||||
false,
|
||||
1,
|
||||
[3, 5, 8, 12],
|
||||
25,
|
||||
10,
|
||||
90,
|
||||
0,
|
||||
&SecureRandom::new(),
|
||||
);
|
||||
assert!(page.body.contains(
|
||||
@@ -98,6 +130,10 @@ fn retry_and_attempt_state_are_frozen_before_fetch() {
|
||||
#[test]
|
||||
fn ambiguous_commit_is_resolved_before_carrier_advance() {
|
||||
let page = render_page("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", 4);
|
||||
assert!(
|
||||
page.body
|
||||
.contains("if(snapshot.selected){advanceConfirmed(reason,epoch);return}")
|
||||
);
|
||||
assert!(page.body.contains("resolveAttempt(reason,epoch,snapshot)"));
|
||||
assert!(page.body.contains(
|
||||
"sessionEcho(response,snapshot.attempt,['provisional','committed','healthy'],true)"
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Weak;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
use tokio::sync::watch;
|
||||
|
||||
use super::manager::WebProcessRuntime;
|
||||
|
||||
/// Process-owned WEB ingress lifecycle state.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum WebRuntimeLifecycle {
|
||||
/// Listener orchestration has not completed.
|
||||
Starting,
|
||||
/// This process has no WEB listener.
|
||||
NoWebListener,
|
||||
/// WEB admission and request handling are active.
|
||||
Running,
|
||||
/// WEB admission is closed while owned work drains.
|
||||
Draining,
|
||||
/// All WEB ingress work drained within the deadline.
|
||||
Drained,
|
||||
/// The bounded shutdown deadline expired.
|
||||
DeadlineExceeded,
|
||||
}
|
||||
|
||||
impl WebRuntimeLifecycle {
|
||||
/// Returns the stable API token for this lifecycle state.
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Starting => "starting",
|
||||
Self::NoWebListener => "no_web_listener",
|
||||
Self::Running => "running",
|
||||
Self::Draining => "draining",
|
||||
Self::Drained => "drained",
|
||||
Self::DeadlineExceeded => "deadline_exceeded",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One immutable lifecycle publication consumed by the control plane.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct WebRuntimePublication {
|
||||
/// Monotonic process-local lifecycle transition number.
|
||||
pub(crate) epoch: u64,
|
||||
/// Current lifecycle state.
|
||||
pub(crate) lifecycle: WebRuntimeLifecycle,
|
||||
/// Monotonic transition time used only for relative age.
|
||||
pub(crate) since: Instant,
|
||||
/// Actual WEB listener addresses frozen for this process.
|
||||
pub(crate) listeners: Arc<[SocketAddr]>,
|
||||
/// Weak runtime access that never extends data-plane ownership.
|
||||
pub(crate) runtime: Weak<WebProcessRuntime>,
|
||||
}
|
||||
|
||||
/// Single-writer process lifecycle publisher for WEB ingress.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct WebRuntimeControl {
|
||||
epoch: Arc<AtomicU64>,
|
||||
tx: watch::Sender<WebRuntimePublication>,
|
||||
}
|
||||
|
||||
impl WebRuntimeControl {
|
||||
/// Creates the process channel in the pre-listener `starting` state.
|
||||
pub(crate) fn new() -> Self {
|
||||
let publication = WebRuntimePublication {
|
||||
epoch: 1,
|
||||
lifecycle: WebRuntimeLifecycle::Starting,
|
||||
since: Instant::now(),
|
||||
listeners: Arc::from([]),
|
||||
runtime: Weak::new(),
|
||||
};
|
||||
let (tx, _rx) = watch::channel(publication);
|
||||
Self {
|
||||
epoch: Arc::new(AtomicU64::new(1)),
|
||||
tx,
|
||||
}
|
||||
}
|
||||
|
||||
/// Subscribes without transferring runtime ownership to the receiver.
|
||||
pub(crate) fn subscribe(&self) -> watch::Receiver<WebRuntimePublication> {
|
||||
self.tx.subscribe()
|
||||
}
|
||||
|
||||
/// Publishes one lifecycle transition and optional weak runtime reference.
|
||||
pub(crate) fn publish(
|
||||
&self,
|
||||
lifecycle: WebRuntimeLifecycle,
|
||||
listeners: Arc<[SocketAddr]>,
|
||||
runtime: Weak<WebProcessRuntime>,
|
||||
) {
|
||||
let epoch = self.epoch.fetch_add(1, Ordering::AcqRel).saturating_add(1);
|
||||
self.tx.send_replace(WebRuntimePublication {
|
||||
epoch,
|
||||
lifecycle,
|
||||
since: Instant::now(),
|
||||
listeners,
|
||||
runtime,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for WebRuntimeControl {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn publication_is_monotonic_and_does_not_require_a_runtime_owner() {
|
||||
let control = WebRuntimeControl::new();
|
||||
let receiver = control.subscribe();
|
||||
control.publish(
|
||||
WebRuntimeLifecycle::NoWebListener,
|
||||
Arc::from([]),
|
||||
Weak::new(),
|
||||
);
|
||||
|
||||
let publication = receiver.borrow().clone();
|
||||
assert_eq!(publication.epoch, 2);
|
||||
assert_eq!(publication.lifecycle, WebRuntimeLifecycle::NoWebListener);
|
||||
assert!(publication.runtime.upgrade().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn publication_keeps_only_weak_runtime_ownership() {
|
||||
let generation = crate::maestro::generation::test_runtime_generation(
|
||||
1,
|
||||
crate::config::ProxyConfig::default(),
|
||||
);
|
||||
let runtime =
|
||||
WebProcessRuntime::start(Arc::new(arc_swap::ArcSwap::from(generation.clone())));
|
||||
let strong_before = Arc::strong_count(&runtime);
|
||||
let control = WebRuntimeControl::new();
|
||||
control.publish(
|
||||
WebRuntimeLifecycle::Running,
|
||||
Arc::from([]),
|
||||
Arc::downgrade(&runtime),
|
||||
);
|
||||
|
||||
assert_eq!(Arc::strong_count(&runtime), strong_before);
|
||||
runtime.shutdown().await;
|
||||
drop(runtime);
|
||||
assert!(control.subscribe().borrow().runtime.upgrade().is_none());
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
+48
-16
@@ -2,7 +2,7 @@ use std::convert::Infallible;
|
||||
use std::error::Error;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use http_body_util::BodyExt;
|
||||
@@ -13,11 +13,11 @@ use hyper::service::service_fn;
|
||||
use hyper::{Method, Request, Response, StatusCode};
|
||||
use hyper_util::rt::{TokioIo, TokioTimer};
|
||||
use ipnetwork::IpNetwork;
|
||||
use parking_lot::Mutex;
|
||||
use tokio::net::TcpStream;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::config::{WebClientIpSource, WebRuntimeVhost};
|
||||
use crate::maestro::generation::RuntimeGeneration;
|
||||
use crate::web::bridge;
|
||||
use crate::web::manager::{ManagerError, WebProcessRuntime};
|
||||
|
||||
@@ -45,7 +45,7 @@ mod websocket;
|
||||
mod trace_tests;
|
||||
|
||||
use crate::web::trace::{HttpTraceExchange, TraceDirection, TraceLifecycleEvent, TraceRoute};
|
||||
use activity::{ActivityBody, RequestActivity};
|
||||
use activity::{ActivityBody, ConnectionActivity, RequestActivity, RequestDeadlineHandle};
|
||||
use body::{CollectBodyError, CollectedBody, RequestBody, collect_body};
|
||||
use decoy::serve_decoy;
|
||||
use down::handle_down;
|
||||
@@ -80,15 +80,22 @@ pub(crate) async fn serve_connection(
|
||||
let max_header_bytes = config.web.limits.max_header_bytes;
|
||||
let header_timeout = Duration::from_secs(config.web.timeouts.header_secs);
|
||||
let idle_timeout = Duration::from_secs(config.web.timeouts.http_idle_secs);
|
||||
let last_activity = Arc::new(Mutex::new(Instant::now()));
|
||||
let service_last_activity = Arc::clone(&last_activity);
|
||||
let connection_activity = ConnectionActivity::new();
|
||||
let service_activity = connection_activity.clone();
|
||||
let service = service_fn(move |mut request| {
|
||||
let runtime = Arc::clone(&runtime);
|
||||
let trusted_proxy_cidrs = Arc::clone(&trusted_proxy_cidrs);
|
||||
let last_activity = Arc::clone(&service_last_activity);
|
||||
let connection_activity = service_activity.clone();
|
||||
let client_ip_source = client_ip_source;
|
||||
async move {
|
||||
let activity = RequestActivity::begin(last_activity);
|
||||
let Some(activity) = RequestActivity::begin(connection_activity) else {
|
||||
let mut response = service_unavailable();
|
||||
response
|
||||
.headers_mut()
|
||||
.insert(header::CONNECTION, HeaderValue::from_static("close"));
|
||||
return Ok::<_, Infallible>(response);
|
||||
};
|
||||
request.extensions_mut().insert(activity.deadline_handle());
|
||||
let trace = runtime.trace().begin_http(&request, peer.ip());
|
||||
if let Some(trace) = &trace {
|
||||
request.extensions_mut().insert(Arc::clone(trace));
|
||||
@@ -133,9 +140,7 @@ pub(crate) async fn serve_connection(
|
||||
_ = cancellation.cancelled() => break,
|
||||
_ = &mut connection => break,
|
||||
_ = idle_check.tick() => {
|
||||
if Instant::now().saturating_duration_since(*last_activity.lock())
|
||||
>= idle_timeout
|
||||
{
|
||||
if connection_activity.should_close(tokio::time::Instant::now(), idle_timeout) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -197,6 +202,7 @@ async fn handle_request(
|
||||
client_ip_source,
|
||||
trusted_proxy_cidrs,
|
||||
runtime,
|
||||
generation,
|
||||
vhost,
|
||||
)
|
||||
.await;
|
||||
@@ -210,6 +216,7 @@ async fn handle_root(
|
||||
client_ip_source: WebClientIpSource,
|
||||
trusted_proxy_cidrs: &[IpNetwork],
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
generation: Arc<RuntimeGeneration>,
|
||||
vhost: Arc<WebRuntimeVhost>,
|
||||
) -> HttpResponse {
|
||||
let (candidate, canonical) = bridge_candidate(request.uri().query());
|
||||
@@ -225,7 +232,16 @@ async fn handle_root(
|
||||
trace.set_route(TraceRoute::Bridge);
|
||||
trace.set_effective_ip(client_ip);
|
||||
}
|
||||
let bootstrap = match runtime.issue_bootstrap(Arc::clone(&profile), client_ip) {
|
||||
let user_agent = request
|
||||
.headers()
|
||||
.get(header::USER_AGENT)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let bootstrap = match runtime.issue_bootstrap_for_request(
|
||||
&generation,
|
||||
Arc::clone(&profile),
|
||||
client_ip,
|
||||
user_agent,
|
||||
) {
|
||||
Ok(bootstrap) => bootstrap,
|
||||
Err(error) => {
|
||||
runtime.trace().record_profile_lifecycle(
|
||||
@@ -244,16 +260,20 @@ async fn handle_root(
|
||||
trace.bind_profile(&profile, bootstrap.trace_session_id);
|
||||
trace.register_redaction(bootstrap.token.as_bytes());
|
||||
}
|
||||
let generation = runtime.active_generation();
|
||||
let config = generation.config();
|
||||
let page = bridge::render(
|
||||
&vhost.host,
|
||||
&bootstrap.token,
|
||||
generation.config().web.limits.carrier_batch_bytes,
|
||||
generation.config().web.limits.pending_bytes_per_session,
|
||||
generation.config().web.limits.pending_items_per_session,
|
||||
config.web.limits.carrier_batch_bytes,
|
||||
config.web.limits.pending_bytes_per_session,
|
||||
config.web.limits.pending_items_per_session,
|
||||
profile.carrier_negotiation_enabled,
|
||||
profile.carriers.len(),
|
||||
profile.carrier_negotiation_deadlines_secs,
|
||||
config.web.timeouts.long_poll_secs,
|
||||
config.web.timeouts.bridge_request_secs,
|
||||
config.web.timeouts.bridge_retry_secs,
|
||||
config.web.timeouts.carrier_probe_coalesce_ms,
|
||||
&generation.rng,
|
||||
);
|
||||
let mut response = full_response(StatusCode::OK, Bytes::from(page.body));
|
||||
@@ -348,7 +368,15 @@ async fn handle_up(
|
||||
request,
|
||||
body,
|
||||
_body_budget,
|
||||
} = match collect_body(request, &runtime, limit, false).await {
|
||||
} = match collect_body(
|
||||
request,
|
||||
&runtime,
|
||||
Duration::from_secs(session.timeouts().body_secs),
|
||||
limit,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(CollectBodyError::Limit) => return service_unavailable(),
|
||||
Err(CollectBodyError::Invalid(request)) => {
|
||||
@@ -391,6 +419,10 @@ fn request_trace<B>(request: &Request<B>) -> Option<&Arc<HttpTraceExchange>> {
|
||||
request.extensions().get::<Arc<HttpTraceExchange>>()
|
||||
}
|
||||
|
||||
fn request_deadline<B>(request: &Request<B>) -> Option<RequestDeadlineHandle> {
|
||||
request.extensions().get::<RequestDeadlineHandle>().cloned()
|
||||
}
|
||||
|
||||
fn set_trace_route<B>(request: &Request<B>, route: TraceRoute) {
|
||||
if let Some(trace) = request_trace(request) {
|
||||
trace.set_route(route);
|
||||
|
||||
+342
-8
@@ -1,31 +1,293 @@
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Instant;
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use hyper::body::{Body, Frame, SizeHint};
|
||||
use parking_lot::Mutex;
|
||||
use tokio::time::Instant;
|
||||
|
||||
use super::{BoxError, HttpBody};
|
||||
use crate::web::trace::{HttpTraceExchange, TraceBodyState, TraceDirection};
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct DeadlineSlot {
|
||||
id: u64,
|
||||
deadline: Instant,
|
||||
}
|
||||
|
||||
struct RequestSlot {
|
||||
id: u64,
|
||||
deadline: Option<DeadlineSlot>,
|
||||
}
|
||||
|
||||
struct UpgradeSlot {
|
||||
request_id: u64,
|
||||
deadline: DeadlineSlot,
|
||||
}
|
||||
|
||||
struct ActivityState {
|
||||
last_progress: Instant,
|
||||
next_request_id: u64,
|
||||
next_deadline_id: u64,
|
||||
request: Option<RequestSlot>,
|
||||
upgrade: Option<UpgradeSlot>,
|
||||
failed: bool,
|
||||
}
|
||||
|
||||
/// Shared liveness state for one accepted HTTP connection.
|
||||
#[derive(Clone)]
|
||||
pub(super) struct ConnectionActivity {
|
||||
state: Arc<Mutex<ActivityState>>,
|
||||
}
|
||||
|
||||
impl ConnectionActivity {
|
||||
/// Creates activity state at the connection acceptance boundary.
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
state: Arc::new(Mutex::new(ActivityState {
|
||||
last_progress: Instant::now(),
|
||||
next_request_id: 1,
|
||||
next_deadline_id: 1,
|
||||
request: None,
|
||||
upgrade: None,
|
||||
failed: false,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether the connection has no protected operation or recent progress.
|
||||
pub(super) fn should_close(&self, now: Instant, idle: Duration) -> bool {
|
||||
let state = self.state.lock();
|
||||
if state.failed {
|
||||
return true;
|
||||
}
|
||||
let request_deadline = state
|
||||
.request
|
||||
.as_ref()
|
||||
.and_then(|request| request.deadline)
|
||||
.map(|deadline| deadline.deadline);
|
||||
let upgrade_deadline = state
|
||||
.upgrade
|
||||
.as_ref()
|
||||
.map(|upgrade| upgrade.deadline.deadline);
|
||||
let protected_until = request_deadline.into_iter().chain(upgrade_deadline).max();
|
||||
let idle_since = protected_until
|
||||
.filter(|deadline| *deadline > state.last_progress)
|
||||
.unwrap_or(state.last_progress);
|
||||
now.saturating_duration_since(idle_since) >= idle
|
||||
}
|
||||
|
||||
fn fail(&self) {
|
||||
self.state.lock().failed = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Cloneable authority for protecting one request's explicitly bounded awaits.
|
||||
#[derive(Clone)]
|
||||
pub(super) struct RequestDeadlineHandle {
|
||||
activity: ConnectionActivity,
|
||||
request_id: u64,
|
||||
}
|
||||
|
||||
impl RequestDeadlineHandle {
|
||||
/// Protects the current bounded request operation until its absolute deadline.
|
||||
pub(super) fn lease_until(&self, deadline: Instant) -> Option<RequestDeadlineLease> {
|
||||
let now = Instant::now();
|
||||
let mut state = self.activity.state.lock();
|
||||
if state.failed {
|
||||
return None;
|
||||
}
|
||||
let Some(current) = state.request.as_ref() else {
|
||||
state.failed = true;
|
||||
return None;
|
||||
};
|
||||
if current.id != self.request_id
|
||||
|| current
|
||||
.deadline
|
||||
.is_some_and(|active| now <= active.deadline)
|
||||
{
|
||||
state.failed = true;
|
||||
return None;
|
||||
}
|
||||
let id = state.next_deadline_id;
|
||||
let Some(next) = id.checked_add(1) else {
|
||||
state.failed = true;
|
||||
return None;
|
||||
};
|
||||
state.next_deadline_id = next;
|
||||
let Some(current) = state.request.as_mut() else {
|
||||
state.failed = true;
|
||||
return None;
|
||||
};
|
||||
current.deadline = Some(DeadlineSlot { id, deadline });
|
||||
Some(RequestDeadlineLease {
|
||||
handle: self.clone(),
|
||||
deadline_id: id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Protects the current bounded request operation for one checked duration.
|
||||
pub(super) fn lease_for(&self, duration: Duration) -> Option<RequestDeadlineLease> {
|
||||
let Some(deadline) = Instant::now().checked_add(duration) else {
|
||||
self.activity.fail();
|
||||
return None;
|
||||
};
|
||||
self.lease_until(deadline)
|
||||
}
|
||||
|
||||
/// Transfers idle protection to a pending Hyper upgrade operation.
|
||||
pub(super) fn upgrade_until(&self, deadline: Instant) -> Option<UpgradeDeadlineLease> {
|
||||
let now = Instant::now();
|
||||
let mut state = self.activity.state.lock();
|
||||
if state.failed
|
||||
|| state
|
||||
.request
|
||||
.as_ref()
|
||||
.is_none_or(|request| request.id != self.request_id)
|
||||
|| state
|
||||
.upgrade
|
||||
.as_ref()
|
||||
.is_some_and(|upgrade| now <= upgrade.deadline.deadline)
|
||||
{
|
||||
state.failed = true;
|
||||
return None;
|
||||
}
|
||||
let id = state.next_deadline_id;
|
||||
let Some(next) = id.checked_add(1) else {
|
||||
state.failed = true;
|
||||
return None;
|
||||
};
|
||||
state.next_deadline_id = next;
|
||||
state.upgrade = Some(UpgradeSlot {
|
||||
request_id: self.request_id,
|
||||
deadline: DeadlineSlot { id, deadline },
|
||||
});
|
||||
Some(UpgradeDeadlineLease {
|
||||
activity: self.activity.clone(),
|
||||
request_id: self.request_id,
|
||||
deadline_id: id,
|
||||
deadline,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Exact request-operation lease that cannot clear a newer deadline.
|
||||
pub(super) struct RequestDeadlineLease {
|
||||
handle: RequestDeadlineHandle,
|
||||
deadline_id: u64,
|
||||
}
|
||||
|
||||
impl Drop for RequestDeadlineLease {
|
||||
fn drop(&mut self) {
|
||||
let mut state = self.handle.activity.state.lock();
|
||||
let matches = state.request.as_ref().is_some_and(|request| {
|
||||
request.id == self.handle.request_id
|
||||
&& request
|
||||
.deadline
|
||||
.is_some_and(|deadline| deadline.id == self.deadline_id)
|
||||
});
|
||||
if matches {
|
||||
if let Some(request) = state.request.as_mut() {
|
||||
request.deadline = None;
|
||||
}
|
||||
state.last_progress = Instant::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Exact pending-upgrade lease retained by the spawned upgrade future.
|
||||
pub(super) struct UpgradeDeadlineLease {
|
||||
activity: ConnectionActivity,
|
||||
request_id: u64,
|
||||
deadline_id: u64,
|
||||
deadline: Instant,
|
||||
}
|
||||
|
||||
impl UpgradeDeadlineLease {
|
||||
/// Returns the absolute deadline shared with the upgrade timeout.
|
||||
pub(super) fn deadline(&self) -> Instant {
|
||||
self.deadline
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for UpgradeDeadlineLease {
|
||||
fn drop(&mut self) {
|
||||
let mut state = self.activity.state.lock();
|
||||
let matches = state.upgrade.as_ref().is_some_and(|upgrade| {
|
||||
upgrade.request_id == self.request_id && upgrade.deadline.id == self.deadline_id
|
||||
});
|
||||
if matches {
|
||||
state.upgrade = None;
|
||||
state.last_progress = Instant::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Request lifecycle guard that refreshes HTTP connection activity on completion.
|
||||
pub(super) struct RequestActivity {
|
||||
last_activity: Arc<Mutex<Instant>>,
|
||||
handle: RequestDeadlineHandle,
|
||||
}
|
||||
|
||||
impl RequestActivity {
|
||||
/// Starts activity accounting for one HTTP request.
|
||||
pub(super) fn begin(last_activity: Arc<Mutex<Instant>>) -> Self {
|
||||
*last_activity.lock() = Instant::now();
|
||||
Self { last_activity }
|
||||
pub(super) fn begin(activity: ConnectionActivity) -> Option<Self> {
|
||||
let mut state = activity.state.lock();
|
||||
if state.failed || state.request.is_some() {
|
||||
state.failed = true;
|
||||
return None;
|
||||
}
|
||||
let id = state.next_request_id;
|
||||
let Some(next) = id.checked_add(1) else {
|
||||
state.failed = true;
|
||||
return None;
|
||||
};
|
||||
state.next_request_id = next;
|
||||
state.last_progress = Instant::now();
|
||||
state.request = Some(RequestSlot { id, deadline: None });
|
||||
drop(state);
|
||||
Some(Self {
|
||||
handle: RequestDeadlineHandle {
|
||||
activity,
|
||||
request_id: id,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the authority copied into request extensions for bounded awaits.
|
||||
pub(super) fn deadline_handle(&self) -> RequestDeadlineHandle {
|
||||
self.handle.clone()
|
||||
}
|
||||
|
||||
fn progress(&self) {
|
||||
self.handle.activity.state.lock().last_progress = Instant::now();
|
||||
}
|
||||
|
||||
fn enter_response(&mut self) {
|
||||
let mut state = self.handle.activity.state.lock();
|
||||
if let Some(request) = state
|
||||
.request
|
||||
.as_mut()
|
||||
.filter(|request| request.id == self.handle.request_id)
|
||||
{
|
||||
request.deadline = None;
|
||||
state.last_progress = Instant::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RequestActivity {
|
||||
fn drop(&mut self) {
|
||||
*self.last_activity.lock() = Instant::now();
|
||||
let mut state = self.handle.activity.state.lock();
|
||||
if state
|
||||
.request
|
||||
.as_ref()
|
||||
.is_some_and(|request| request.id == self.handle.request_id)
|
||||
{
|
||||
state.request = None;
|
||||
state.last_progress = Instant::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,9 +303,10 @@ impl ActivityBody {
|
||||
/// Binds one response body to its request activity guard.
|
||||
pub(super) fn new(
|
||||
inner: HttpBody,
|
||||
activity: RequestActivity,
|
||||
mut activity: RequestActivity,
|
||||
trace: Option<Arc<HttpTraceExchange>>,
|
||||
) -> Self {
|
||||
activity.enter_response();
|
||||
Self {
|
||||
inner,
|
||||
activity,
|
||||
@@ -88,7 +351,7 @@ impl Body for ActivityBody {
|
||||
Poll::Pending => {}
|
||||
}
|
||||
if result.is_ready() {
|
||||
*self.activity.last_activity.lock() = Instant::now();
|
||||
self.activity.progress();
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -107,3 +370,74 @@ impl Drop for ActivityBody {
|
||||
self.finish(TraceBodyState::Aborted);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn bounded_request_deadline_suspends_only_idle_expiry() {
|
||||
let activity = ConnectionActivity::new();
|
||||
let request = RequestActivity::begin(activity.clone()).unwrap();
|
||||
let now = Instant::now();
|
||||
let lease = request
|
||||
.deadline_handle()
|
||||
.lease_until(now + Duration::from_secs(5))
|
||||
.unwrap();
|
||||
|
||||
assert!(!activity.should_close(now + Duration::from_secs(4), Duration::from_secs(1)));
|
||||
assert!(activity.should_close(now + Duration::from_secs(6), Duration::from_secs(1)));
|
||||
|
||||
drop(lease);
|
||||
assert!(!activity.should_close(Instant::now(), Duration::from_secs(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expired_operation_lease_gets_one_idle_interval_to_publish_its_result() {
|
||||
let activity = ConnectionActivity::new();
|
||||
let request = RequestActivity::begin(activity.clone()).unwrap();
|
||||
let now = Instant::now();
|
||||
let _lease = request
|
||||
.deadline_handle()
|
||||
.lease_until(now + Duration::from_secs(1))
|
||||
.unwrap();
|
||||
|
||||
assert!(!activity.should_close(now + Duration::from_millis(1500), Duration::from_secs(1)));
|
||||
assert!(activity.should_close(now + Duration::from_secs(2), Duration::from_secs(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_request_lease_cannot_clear_a_new_request_deadline() {
|
||||
let activity = ConnectionActivity::new();
|
||||
let request_a = RequestActivity::begin(activity.clone()).unwrap();
|
||||
let now = Instant::now();
|
||||
let lease_a = request_a
|
||||
.deadline_handle()
|
||||
.lease_until(now - Duration::from_secs(1))
|
||||
.unwrap();
|
||||
drop(request_a);
|
||||
let request_b = RequestActivity::begin(activity.clone()).unwrap();
|
||||
let _lease_b = request_b
|
||||
.deadline_handle()
|
||||
.lease_until(now + Duration::from_secs(5))
|
||||
.unwrap();
|
||||
|
||||
drop(lease_a);
|
||||
|
||||
assert!(!activity.should_close(now + Duration::from_secs(4), Duration::from_secs(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_upgrade_lease_cannot_clear_its_replacement() {
|
||||
let activity = ConnectionActivity::new();
|
||||
let request = RequestActivity::begin(activity.clone()).unwrap();
|
||||
let handle = request.deadline_handle();
|
||||
let now = Instant::now();
|
||||
let lease_a = handle.upgrade_until(now - Duration::from_secs(1)).unwrap();
|
||||
let _lease_b = handle.upgrade_until(now + Duration::from_secs(5)).unwrap();
|
||||
|
||||
drop(lease_a);
|
||||
|
||||
assert!(!activity.should_close(now + Duration::from_secs(4), Duration::from_secs(1)));
|
||||
}
|
||||
}
|
||||
|
||||
+10
-2
@@ -114,9 +114,11 @@ pub(super) enum CollectBodyError {
|
||||
pub(super) async fn collect_body(
|
||||
request: Request<RequestBody>,
|
||||
runtime: &WebProcessRuntime,
|
||||
body_timeout: Duration,
|
||||
limit: usize,
|
||||
allow_empty: bool,
|
||||
) -> Result<CollectedBody, CollectBodyError> {
|
||||
let request_deadline = super::request_deadline(&request);
|
||||
let exceeds_limit = request.body().size_hint().lower() > limit as u64
|
||||
|| request
|
||||
.body()
|
||||
@@ -133,8 +135,14 @@ pub(super) async fn collect_body(
|
||||
let Some((reader_budget, body_budget)) = runtime.try_body_budget(limit) else {
|
||||
return Err(CollectBodyError::Limit);
|
||||
};
|
||||
let body_timeout =
|
||||
Duration::from_secs(runtime.active_generation().config().web.timeouts.body_secs);
|
||||
let _deadline_lease = match request_deadline {
|
||||
Some(deadline) => Some(
|
||||
deadline
|
||||
.lease_for(body_timeout)
|
||||
.ok_or(CollectBodyError::Limit)?,
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
let body = match tokio::time::timeout(body_timeout, Limited::new(body, limit).collect()).await {
|
||||
Ok(Ok(body)) => body.to_bytes(),
|
||||
_ => {
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_index_lists_and_asynchronously_closes_by_opaque_reference() {
|
||||
let capability = [19u8; 32];
|
||||
let generation = test_runtime_generation(1, runtime_config(capability, WebCarrier::Https));
|
||||
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||
let runtime = WebProcessRuntime::start(active_runtime);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||
let root = format!(
|
||||
"GET /?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.19\r\nUser-Agent: Telemt-Control-Test/1\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
let root_response = request(&listener, &runtime, root).await;
|
||||
let (_, root_body) = split_response(&root_response);
|
||||
let bootstrap = std::str::from_utf8(root_body)
|
||||
.unwrap()
|
||||
.split_once("bootstrap=\"")
|
||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||
.map(|(token, _)| token)
|
||||
.unwrap();
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
let mut create = format!(
|
||||
"POST /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.19\r\nAuthorization: Bearer {bootstrap}\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
hello.len()
|
||||
)
|
||||
.into_bytes();
|
||||
create.extend_from_slice(&hello);
|
||||
let create_response = request(&listener, &runtime, create).await;
|
||||
assert!(create_response.starts_with(b"HTTP/1.1 200"));
|
||||
|
||||
let page = runtime.list_sessions(SessionListRequest {
|
||||
limit: 50,
|
||||
cursor: None,
|
||||
filter: SessionFilter::default(),
|
||||
});
|
||||
assert_eq!(page.sessions.len(), 1);
|
||||
assert_eq!(
|
||||
page.sessions[0].user_agent.as_deref(),
|
||||
Some("Telemt-Control-Test/1")
|
||||
);
|
||||
let session_ref = page.sessions[0].session_ref.clone();
|
||||
let trace_session_id = runtime.parse_session_ref(&session_ref).unwrap();
|
||||
let noncanonical_session_ref = format!("ws1.{}.000000000000000A", runtime.runtime_instance());
|
||||
assert_eq!(
|
||||
runtime.parse_session_ref(&noncanonical_session_ref),
|
||||
Err(SessionRefError::Invalid)
|
||||
);
|
||||
let noncanonical_operation_id = format!("wo1.{}.000000000000000A", runtime.runtime_instance());
|
||||
assert!(matches!(
|
||||
runtime.control_operation(&noncanonical_operation_id),
|
||||
Err(ControlError::InvalidOperation)
|
||||
));
|
||||
let nonmatching = runtime
|
||||
.start_close_operation(
|
||||
runtime.runtime_instance(),
|
||||
CloseOperationSelector::Filter(SessionFilter {
|
||||
state: Some("healthy".to_string()),
|
||||
..SessionFilter::default()
|
||||
}),
|
||||
)
|
||||
.unwrap();
|
||||
let mut nonmatching_status = None;
|
||||
for _ in 0..32 {
|
||||
let status = runtime
|
||||
.control_operation(&nonmatching.operation_id)
|
||||
.unwrap();
|
||||
if serde_json::to_value(&status).unwrap()["state"] == "completed" {
|
||||
nonmatching_status = Some(status);
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
let nonmatching_status = nonmatching_status.expect("filtered close operation completed");
|
||||
assert_eq!(nonmatching_status.matched, 0);
|
||||
assert_eq!(nonmatching_status.close_signalled, 0);
|
||||
assert!(matches!(
|
||||
runtime.session_detail(trace_session_id),
|
||||
SessionDetail::Active(_)
|
||||
));
|
||||
let operation = runtime
|
||||
.start_close_operation(
|
||||
runtime.runtime_instance(),
|
||||
CloseOperationSelector::Refs(vec![trace_session_id]),
|
||||
)
|
||||
.unwrap();
|
||||
for _ in 0..32 {
|
||||
let status = runtime.control_operation(&operation.operation_id).unwrap();
|
||||
if serde_json::to_value(&status).unwrap()["state"] == "completed" {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
assert!(matches!(
|
||||
runtime.session_detail(trace_session_id),
|
||||
SessionDetail::Gone { .. }
|
||||
));
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
@@ -184,6 +184,7 @@ async fn proxy_to_upstream(
|
||||
header_timeout: Duration,
|
||||
runtime: &WebProcessRuntime,
|
||||
) -> HttpResponse {
|
||||
let request_deadline = super::request_deadline(&request);
|
||||
remove_hop_by_hop(request.headers_mut());
|
||||
if let Ok(host) = HeaderValue::from_str(authority) {
|
||||
request.headers_mut().insert(header::HOST, host);
|
||||
@@ -197,10 +198,15 @@ async fn proxy_to_upstream(
|
||||
return bad_gateway();
|
||||
};
|
||||
*request.uri_mut() = uri;
|
||||
let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
|
||||
Ok(lease) => lease,
|
||||
Err(()) => return bad_gateway(),
|
||||
};
|
||||
let stream = match tokio::time::timeout(header_timeout, TcpStream::connect(addr)).await {
|
||||
Ok(Ok(stream)) => stream,
|
||||
_ => return bad_gateway(),
|
||||
};
|
||||
drop(_deadline_lease);
|
||||
let max_header_bytes = runtime
|
||||
.active_generation()
|
||||
.config()
|
||||
@@ -209,19 +215,29 @@ async fn proxy_to_upstream(
|
||||
.max_header_bytes;
|
||||
let mut builder = hyper::client::conn::http1::Builder::new();
|
||||
builder.max_buf_size(max_header_bytes);
|
||||
let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
|
||||
Ok(lease) => lease,
|
||||
Err(()) => return bad_gateway(),
|
||||
};
|
||||
let (mut sender, connection) =
|
||||
match tokio::time::timeout(header_timeout, builder.handshake(TokioIo::new(stream))).await {
|
||||
Ok(Ok(parts)) => parts,
|
||||
_ => return bad_gateway(),
|
||||
};
|
||||
drop(_deadline_lease);
|
||||
runtime.spawn_auxiliary(async move {
|
||||
let _ = connection.await;
|
||||
});
|
||||
let _deadline_lease = match lease_deadline(request_deadline.as_ref(), header_timeout) {
|
||||
Ok(lease) => lease,
|
||||
Err(()) => return bad_gateway(),
|
||||
};
|
||||
let mut response =
|
||||
match tokio::time::timeout(header_timeout, sender.send_request(request)).await {
|
||||
Ok(Ok(response)) => response,
|
||||
_ => return bad_gateway(),
|
||||
};
|
||||
drop(_deadline_lease);
|
||||
remove_hop_by_hop(response.headers_mut());
|
||||
response.map(|body| {
|
||||
body.map_err(|error| -> BoxError { Box::new(error) })
|
||||
@@ -229,6 +245,16 @@ async fn proxy_to_upstream(
|
||||
})
|
||||
}
|
||||
|
||||
fn lease_deadline(
|
||||
deadline: Option<&super::activity::RequestDeadlineHandle>,
|
||||
timeout: Duration,
|
||||
) -> Result<Option<super::activity::RequestDeadlineLease>, ()> {
|
||||
match deadline {
|
||||
Some(deadline) => deadline.lease_for(timeout).map(Some).ok_or(()),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn sanitize_transport_request<B>(request: &mut Request<B>) {
|
||||
for name in [
|
||||
header::AUTHORIZATION,
|
||||
|
||||
+26
-1
@@ -1,4 +1,5 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use hyper::header::{self, HeaderName, HeaderValue};
|
||||
use hyper::{Request, StatusCode};
|
||||
@@ -46,7 +47,15 @@ pub(super) async fn handle_down(
|
||||
request,
|
||||
body,
|
||||
_body_budget,
|
||||
} = match collect_body(request, &runtime, 1, true).await {
|
||||
} = match collect_body(
|
||||
request,
|
||||
&runtime,
|
||||
Duration::from_secs(session.timeouts().body_secs),
|
||||
1,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(CollectBodyError::Limit) => return service_unavailable(),
|
||||
Err(CollectBodyError::Invalid(request)) => {
|
||||
@@ -67,10 +76,26 @@ pub(super) async fn handle_down(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let poll_timeout = match lane_id {
|
||||
Some(_) => Duration::from_secs(session.timeouts().lane_open_wait_secs)
|
||||
.checked_add(Duration::from_secs(session.timeouts().long_poll_secs)),
|
||||
None => Some(Duration::from_secs(session.timeouts().long_poll_secs)),
|
||||
};
|
||||
let Some(poll_timeout) = poll_timeout else {
|
||||
return service_unavailable();
|
||||
};
|
||||
let _deadline_lease = match super::request_deadline(&request) {
|
||||
Some(deadline) => match deadline.lease_for(poll_timeout) {
|
||||
Some(lease) => Some(lease),
|
||||
None => return service_unavailable(),
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
let result = match lane_id {
|
||||
Some(lane_id) => session.poll_down_lane(lane_id, cursor).await,
|
||||
None => session.poll_down(cursor).await,
|
||||
};
|
||||
drop(_deadline_lease);
|
||||
match result {
|
||||
Ok(result) if result.body.is_empty() => {
|
||||
let mut response = carrier_empty(StatusCode::NO_CONTENT);
|
||||
|
||||
@@ -178,7 +178,7 @@ async fn metadata_free_native_client_uses_fallback_when_candidates_are_enabled()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_native_capabilities_participate_in_automatic_selection() {
|
||||
async fn explicit_native_capabilities_are_limited_to_https() {
|
||||
let capability = [56; 32];
|
||||
let generation = test_runtime_generation(
|
||||
1,
|
||||
@@ -202,12 +202,9 @@ async fn explicit_native_capabilities_participate_in_automatic_selection() {
|
||||
.await;
|
||||
let (headers, _) = split_response(&response);
|
||||
assert!(headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(
|
||||
response_header(headers, "x-carrier-mode"),
|
||||
"websocket-lanes"
|
||||
);
|
||||
assert_eq!(response_header(headers, "x-carrier-mode"), "https");
|
||||
assert_eq!(response_header(headers, "x-carrier-attempt"), "1");
|
||||
assert_eq!(response_header(headers, "x-carrier-candidate-count"), "2");
|
||||
assert_eq!(response_header(headers, "x-carrier-candidate-count"), "1");
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
|
||||
+12
-285
@@ -140,6 +140,11 @@ pub(super) fn carrier_request<B>(request: &Request<B>, host: &str) -> Option<Car
|
||||
}
|
||||
(Some(capabilities), Some(attempt)) => {
|
||||
let capabilities = parse_capabilities(capabilities)?;
|
||||
let capabilities = if native_ios {
|
||||
capabilities.intersection(CarrierCapabilities::ios())?
|
||||
} else {
|
||||
capabilities
|
||||
};
|
||||
if (attempt == 1) != failure.is_none() {
|
||||
return None;
|
||||
}
|
||||
@@ -156,15 +161,14 @@ pub(super) fn carrier_request<B>(request: &Request<B>, host: &str) -> Option<Car
|
||||
))
|
||||
}
|
||||
(None, Some(attempt)) if strict_browser_hint(request, host) => {
|
||||
if native_ios {
|
||||
return None;
|
||||
}
|
||||
if (attempt == 1) != failure.is_none() {
|
||||
return None;
|
||||
}
|
||||
Some(CarrierRequest::automatic(
|
||||
if native_ios {
|
||||
CarrierClientClass::Ios
|
||||
} else {
|
||||
CarrierClientClass::BrowserHint
|
||||
},
|
||||
CarrierClientClass::BrowserHint,
|
||||
CarrierCapabilities::all(),
|
||||
attempt,
|
||||
failure,
|
||||
@@ -259,284 +263,7 @@ fn single_header<B>(request: &Request<B>, name: impl header::AsHeaderName) -> Op
|
||||
values.next().is_none().then_some(value)
|
||||
}
|
||||
|
||||
// Canonical request parsing and client-classification tests.
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use ipnetwork::IpNetwork;
|
||||
|
||||
use crate::config::{WebCarrier, WebClientIpSource};
|
||||
|
||||
#[test]
|
||||
fn canonical_bridge_query_rejects_aliases() {
|
||||
let token = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode([7u8; 32]);
|
||||
assert!(bridge_candidate(Some(&format!("bridge={token}"))).1);
|
||||
assert!(!bridge_candidate(Some(&format!("x=1&bridge={token}"))).1);
|
||||
assert!(!bridge_candidate(Some(&format!("bridge={token}="))).1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_is_canonical_and_forwarded_identity_is_single_parseable_ip() {
|
||||
let request = Request::builder()
|
||||
.header(header::HOST, "proxy.example.com:443")
|
||||
.header("x-forwarded-for", "192.0.2.10")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert_eq!(canonical_request_host(&request), Some("proxy.example.com"));
|
||||
let trusted: [IpNetwork; 1] = ["127.0.0.1/32".parse().unwrap()];
|
||||
assert_eq!(
|
||||
client_ip(
|
||||
&request,
|
||||
"127.0.0.1:40000".parse().unwrap(),
|
||||
WebClientIpSource::XForwardedFor,
|
||||
&trusted,
|
||||
),
|
||||
Some("192.0.2.10".parse().unwrap())
|
||||
);
|
||||
|
||||
let expanded_ipv6 = Request::builder()
|
||||
.header("x-forwarded-for", "2001:0db8:0:0:0:0:0:10")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
client_ip(
|
||||
&expanded_ipv6,
|
||||
"127.0.0.1:40000".parse().unwrap(),
|
||||
WebClientIpSource::XForwardedFor,
|
||||
&trusted,
|
||||
),
|
||||
Some("2001:db8::10".parse().unwrap())
|
||||
);
|
||||
|
||||
let without_forwarded_address = Request::builder().body(()).unwrap();
|
||||
assert_eq!(
|
||||
client_ip(
|
||||
&without_forwarded_address,
|
||||
"127.0.0.1:40000".parse().unwrap(),
|
||||
WebClientIpSource::XForwardedFor,
|
||||
&trusted,
|
||||
),
|
||||
Some("127.0.0.1".parse().unwrap())
|
||||
);
|
||||
|
||||
let empty_forwarded_address = Request::builder()
|
||||
.header("x-forwarded-for", "")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
client_ip(
|
||||
&empty_forwarded_address,
|
||||
"127.0.0.1:40000".parse().unwrap(),
|
||||
WebClientIpSource::XForwardedFor,
|
||||
&trusted,
|
||||
),
|
||||
Some("127.0.0.1".parse().unwrap())
|
||||
);
|
||||
|
||||
let uppercase = Request::builder()
|
||||
.header(header::HOST, "Proxy.Example.com")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(canonical_request_host(&uppercase).is_none());
|
||||
let appended = Request::builder()
|
||||
.header("x-forwarded-for", "192.0.2.10, 198.51.100.4")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(
|
||||
client_ip(
|
||||
&appended,
|
||||
"127.0.0.1:40000".parse().unwrap(),
|
||||
WebClientIpSource::XForwardedFor,
|
||||
&trusted,
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bearer_and_sequence_headers_reject_noncanonical_aliases() {
|
||||
let token = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode([1u8; 32]);
|
||||
let request = Request::builder()
|
||||
.header(header::AUTHORIZATION, format!("Bearer {token}"))
|
||||
.header("x-up-seq", "17")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
bearer_token_hash(&request),
|
||||
Some(Sha256::digest([1u8; 32]).into())
|
||||
);
|
||||
assert_eq!(canonical_u64_header(&request, "x-up-seq"), Some(17));
|
||||
|
||||
let leading_zero = Request::builder()
|
||||
.header("x-up-seq", "017")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(canonical_u64_header(&leading_zero, "x-up-seq").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cookie_header_accepts_only_absent_or_one_empty_value() {
|
||||
let absent = Request::new(());
|
||||
assert!(compatible_cookie_header(&absent));
|
||||
|
||||
let empty = Request::builder()
|
||||
.header(header::COOKIE, "")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(compatible_cookie_header(&empty));
|
||||
|
||||
let nonempty = Request::builder()
|
||||
.header(header::COOKIE, "state=unexpected")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(!compatible_cookie_header(&nonempty));
|
||||
|
||||
let whitespace = Request::builder()
|
||||
.header(header::COOKIE, " ")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(!compatible_cookie_header(&whitespace));
|
||||
|
||||
let mut duplicate_empty = Request::new(());
|
||||
duplicate_empty
|
||||
.headers_mut()
|
||||
.append(header::COOKIE, "".parse().unwrap());
|
||||
duplicate_empty
|
||||
.headers_mut()
|
||||
.append(header::COOKIE, "".parse().unwrap());
|
||||
assert!(!compatible_cookie_header(&duplicate_empty));
|
||||
|
||||
let mut duplicate_mixed = Request::new(());
|
||||
duplicate_mixed
|
||||
.headers_mut()
|
||||
.append(header::COOKIE, "".parse().unwrap());
|
||||
duplicate_mixed
|
||||
.headers_mut()
|
||||
.append(header::COOKIE, "state=unexpected".parse().unwrap());
|
||||
assert!(!compatible_cookie_header(&duplicate_mixed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn carrier_metadata_is_canonical_and_legacy_safe() {
|
||||
let automatic = Request::builder()
|
||||
.header(
|
||||
"x-carrier-capabilities",
|
||||
"https,https-lanes,websocket,websocket-lanes",
|
||||
)
|
||||
.header("x-carrier-attempt", "2")
|
||||
.header("x-carrier-failure", "timeout")
|
||||
.header(header::USER_AGENT, "Example Browser")
|
||||
.body(())
|
||||
.unwrap();
|
||||
let parsed = carrier_request(&automatic, "proxy.example.com").unwrap();
|
||||
assert!(parsed.is_automatic());
|
||||
assert_eq!(parsed.attempt(), Some(2));
|
||||
assert_eq!(parsed.failure(), Some(CarrierFailure::Timeout));
|
||||
|
||||
let missing_failure = Request::builder()
|
||||
.header(
|
||||
"x-carrier-capabilities",
|
||||
"https,https-lanes,websocket,websocket-lanes",
|
||||
)
|
||||
.header("x-carrier-attempt", "2")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(carrier_request(&missing_failure, "proxy.example.com").is_none());
|
||||
|
||||
let legacy = Request::builder()
|
||||
.header(header::USER_AGENT, "Native")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(
|
||||
!carrier_request(&legacy, "proxy.example.com")
|
||||
.unwrap()
|
||||
.is_automatic()
|
||||
);
|
||||
|
||||
let reordered = Request::builder()
|
||||
.header("x-carrier-capabilities", "websocket,https")
|
||||
.header("x-carrier-attempt", "1")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(carrier_request(&reordered, "proxy.example.com").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_ios_user_agent_classifies_without_overriding_capabilities() {
|
||||
let metadata_free = Request::builder()
|
||||
.header(
|
||||
header::USER_AGENT,
|
||||
"Telemt/1 CFNetwork/1498.700.2 Darwin/23.6.0",
|
||||
)
|
||||
.body(())
|
||||
.unwrap();
|
||||
let parsed = carrier_request(&metadata_free, "proxy.example.com").unwrap();
|
||||
assert_eq!(parsed.class(), CarrierClientClass::Ios);
|
||||
assert!(!parsed.is_automatic());
|
||||
assert!(!parsed.uses_capabilities());
|
||||
|
||||
let automatic = Request::builder()
|
||||
.header("x-carrier-capabilities", "https,https-lanes")
|
||||
.header("x-carrier-attempt", "1")
|
||||
.header(
|
||||
header::USER_AGENT,
|
||||
"Telemt/1 CFNetwork/1498.700.2 Darwin/23.6.0",
|
||||
)
|
||||
.body(())
|
||||
.unwrap();
|
||||
let parsed = carrier_request(&automatic, "proxy.example.com").unwrap();
|
||||
assert_eq!(parsed.class(), CarrierClientClass::Ios);
|
||||
assert!(parsed.is_automatic());
|
||||
assert!(parsed.supports(WebCarrier::Https));
|
||||
assert!(parsed.supports(WebCarrier::HttpsLanes));
|
||||
assert!(!parsed.supports(WebCarrier::Websocket));
|
||||
assert!(!parsed.supports(WebCarrier::WebsocketLanes));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mapped_private_addresses_are_not_learning_evidence() {
|
||||
for address in ["::ffff:127.0.0.1", "::ffff:10.0.0.1"] {
|
||||
let effective_ip = address.parse().unwrap();
|
||||
let request = Request::builder()
|
||||
.header("x-forwarded-for", address)
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(!carrier_ip_learning_eligible(&request, effective_ip));
|
||||
}
|
||||
let effective_ip = "::ffff:8.8.8.8".parse().unwrap();
|
||||
let request = Request::builder()
|
||||
.header("x-forwarded-for", "::ffff:8.8.8.8")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(carrier_ip_learning_eligible(&request, effective_ip));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_browser_metadata_recovers_a_stripped_capability_marker() {
|
||||
let request = Request::builder()
|
||||
.header("x-carrier-attempt", "1")
|
||||
.header(header::ORIGIN, "https://proxy.example.com")
|
||||
.header("sec-fetch-site", "same-origin")
|
||||
.header("sec-fetch-mode", "cors")
|
||||
.header("sec-fetch-dest", "empty")
|
||||
.body(())
|
||||
.unwrap();
|
||||
let parsed = carrier_request(&request, "proxy.example.com").unwrap();
|
||||
assert_eq!(parsed.class(), CarrierClientClass::BrowserHint);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_agent_learning_key_is_case_and_whitespace_normalized() {
|
||||
let first = Request::builder()
|
||||
.header(header::USER_AGENT, " Example\t Browser ")
|
||||
.body(())
|
||||
.unwrap();
|
||||
let second = Request::builder()
|
||||
.header(header::USER_AGENT, "example browser")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
normalized_user_agent_hash(&first),
|
||||
normalized_user_agent_hash(&second)
|
||||
);
|
||||
}
|
||||
}
|
||||
#[path = "request/tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
use super::*;
|
||||
use ipnetwork::IpNetwork;
|
||||
|
||||
use crate::config::{WebCarrier, WebClientIpSource};
|
||||
|
||||
#[test]
|
||||
fn canonical_bridge_query_rejects_aliases() {
|
||||
let token = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode([7u8; 32]);
|
||||
assert!(bridge_candidate(Some(&format!("bridge={token}"))).1);
|
||||
assert!(!bridge_candidate(Some(&format!("x=1&bridge={token}"))).1);
|
||||
assert!(!bridge_candidate(Some(&format!("bridge={token}="))).1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_is_canonical_and_forwarded_identity_is_single_parseable_ip() {
|
||||
let request = Request::builder()
|
||||
.header(header::HOST, "proxy.example.com:443")
|
||||
.header("x-forwarded-for", "192.0.2.10")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert_eq!(canonical_request_host(&request), Some("proxy.example.com"));
|
||||
let trusted: [IpNetwork; 1] = ["127.0.0.1/32".parse().unwrap()];
|
||||
assert_eq!(
|
||||
client_ip(
|
||||
&request,
|
||||
"127.0.0.1:40000".parse().unwrap(),
|
||||
WebClientIpSource::XForwardedFor,
|
||||
&trusted,
|
||||
),
|
||||
Some("192.0.2.10".parse().unwrap())
|
||||
);
|
||||
|
||||
let expanded_ipv6 = Request::builder()
|
||||
.header("x-forwarded-for", "2001:0db8:0:0:0:0:0:10")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
client_ip(
|
||||
&expanded_ipv6,
|
||||
"127.0.0.1:40000".parse().unwrap(),
|
||||
WebClientIpSource::XForwardedFor,
|
||||
&trusted,
|
||||
),
|
||||
Some("2001:db8::10".parse().unwrap())
|
||||
);
|
||||
|
||||
let without_forwarded_address = Request::builder().body(()).unwrap();
|
||||
assert_eq!(
|
||||
client_ip(
|
||||
&without_forwarded_address,
|
||||
"127.0.0.1:40000".parse().unwrap(),
|
||||
WebClientIpSource::XForwardedFor,
|
||||
&trusted,
|
||||
),
|
||||
Some("127.0.0.1".parse().unwrap())
|
||||
);
|
||||
|
||||
let empty_forwarded_address = Request::builder()
|
||||
.header("x-forwarded-for", "")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
client_ip(
|
||||
&empty_forwarded_address,
|
||||
"127.0.0.1:40000".parse().unwrap(),
|
||||
WebClientIpSource::XForwardedFor,
|
||||
&trusted,
|
||||
),
|
||||
Some("127.0.0.1".parse().unwrap())
|
||||
);
|
||||
|
||||
let uppercase = Request::builder()
|
||||
.header(header::HOST, "Proxy.Example.com")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(canonical_request_host(&uppercase).is_none());
|
||||
let appended = Request::builder()
|
||||
.header("x-forwarded-for", "192.0.2.10, 198.51.100.4")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(
|
||||
client_ip(
|
||||
&appended,
|
||||
"127.0.0.1:40000".parse().unwrap(),
|
||||
WebClientIpSource::XForwardedFor,
|
||||
&trusted,
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bearer_and_sequence_headers_reject_noncanonical_aliases() {
|
||||
let token = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode([1u8; 32]);
|
||||
let request = Request::builder()
|
||||
.header(header::AUTHORIZATION, format!("Bearer {token}"))
|
||||
.header("x-up-seq", "17")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
bearer_token_hash(&request),
|
||||
Some(Sha256::digest([1u8; 32]).into())
|
||||
);
|
||||
assert_eq!(canonical_u64_header(&request, "x-up-seq"), Some(17));
|
||||
|
||||
let leading_zero = Request::builder()
|
||||
.header("x-up-seq", "017")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(canonical_u64_header(&leading_zero, "x-up-seq").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cookie_header_accepts_only_absent_or_one_empty_value() {
|
||||
let absent = Request::new(());
|
||||
assert!(compatible_cookie_header(&absent));
|
||||
|
||||
let empty = Request::builder()
|
||||
.header(header::COOKIE, "")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(compatible_cookie_header(&empty));
|
||||
|
||||
let nonempty = Request::builder()
|
||||
.header(header::COOKIE, "state=unexpected")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(!compatible_cookie_header(&nonempty));
|
||||
|
||||
let whitespace = Request::builder()
|
||||
.header(header::COOKIE, " ")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(!compatible_cookie_header(&whitespace));
|
||||
|
||||
let mut duplicate_empty = Request::new(());
|
||||
duplicate_empty
|
||||
.headers_mut()
|
||||
.append(header::COOKIE, "".parse().unwrap());
|
||||
duplicate_empty
|
||||
.headers_mut()
|
||||
.append(header::COOKIE, "".parse().unwrap());
|
||||
assert!(!compatible_cookie_header(&duplicate_empty));
|
||||
|
||||
let mut duplicate_mixed = Request::new(());
|
||||
duplicate_mixed
|
||||
.headers_mut()
|
||||
.append(header::COOKIE, "".parse().unwrap());
|
||||
duplicate_mixed
|
||||
.headers_mut()
|
||||
.append(header::COOKIE, "state=unexpected".parse().unwrap());
|
||||
assert!(!compatible_cookie_header(&duplicate_mixed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn carrier_metadata_is_canonical_and_legacy_safe() {
|
||||
let automatic = Request::builder()
|
||||
.header(
|
||||
"x-carrier-capabilities",
|
||||
"https,https-lanes,websocket,websocket-lanes",
|
||||
)
|
||||
.header("x-carrier-attempt", "2")
|
||||
.header("x-carrier-failure", "timeout")
|
||||
.header(header::USER_AGENT, "Example Browser")
|
||||
.body(())
|
||||
.unwrap();
|
||||
let parsed = carrier_request(&automatic, "proxy.example.com").unwrap();
|
||||
assert!(parsed.is_automatic());
|
||||
assert_eq!(parsed.attempt(), Some(2));
|
||||
assert_eq!(parsed.failure(), Some(CarrierFailure::Timeout));
|
||||
|
||||
let missing_failure = Request::builder()
|
||||
.header(
|
||||
"x-carrier-capabilities",
|
||||
"https,https-lanes,websocket,websocket-lanes",
|
||||
)
|
||||
.header("x-carrier-attempt", "2")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(carrier_request(&missing_failure, "proxy.example.com").is_none());
|
||||
|
||||
let legacy = Request::builder()
|
||||
.header(header::USER_AGENT, "Native")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(
|
||||
!carrier_request(&legacy, "proxy.example.com")
|
||||
.unwrap()
|
||||
.is_automatic()
|
||||
);
|
||||
|
||||
let reordered = Request::builder()
|
||||
.header("x-carrier-capabilities", "websocket,https")
|
||||
.header("x-carrier-attempt", "1")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(carrier_request(&reordered, "proxy.example.com").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_ios_user_agent_enforces_the_https_capability_ceiling() {
|
||||
let metadata_free = Request::builder()
|
||||
.header(
|
||||
header::USER_AGENT,
|
||||
"Telemt/1 CFNetwork/1498.700.2 Darwin/23.6.0",
|
||||
)
|
||||
.body(())
|
||||
.unwrap();
|
||||
let parsed = carrier_request(&metadata_free, "proxy.example.com").unwrap();
|
||||
assert_eq!(parsed.class(), CarrierClientClass::Ios);
|
||||
assert!(!parsed.is_automatic());
|
||||
assert!(!parsed.uses_capabilities());
|
||||
|
||||
let automatic = Request::builder()
|
||||
.header("x-carrier-capabilities", "https,https-lanes")
|
||||
.header("x-carrier-attempt", "1")
|
||||
.header(
|
||||
header::USER_AGENT,
|
||||
"Telemt/1 CFNetwork/1498.700.2 Darwin/23.6.0",
|
||||
)
|
||||
.body(())
|
||||
.unwrap();
|
||||
let parsed = carrier_request(&automatic, "proxy.example.com").unwrap();
|
||||
assert_eq!(parsed.class(), CarrierClientClass::Ios);
|
||||
assert!(parsed.is_automatic());
|
||||
assert!(parsed.supports(WebCarrier::Https));
|
||||
assert!(!parsed.supports(WebCarrier::HttpsLanes));
|
||||
assert!(!parsed.supports(WebCarrier::Websocket));
|
||||
assert!(!parsed.supports(WebCarrier::WebsocketLanes));
|
||||
|
||||
let incompatible = Request::builder()
|
||||
.header("x-carrier-capabilities", "websocket")
|
||||
.header("x-carrier-attempt", "1")
|
||||
.header(
|
||||
header::USER_AGENT,
|
||||
"Telemt/1 CFNetwork/1498.700.2 Darwin/23.6.0",
|
||||
)
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(carrier_request(&incompatible, "proxy.example.com").is_none());
|
||||
|
||||
let stripped = Request::builder()
|
||||
.header("x-carrier-attempt", "1")
|
||||
.header(header::ORIGIN, "https://proxy.example.com")
|
||||
.header("sec-fetch-site", "same-origin")
|
||||
.header("sec-fetch-mode", "cors")
|
||||
.header("sec-fetch-dest", "empty")
|
||||
.header(
|
||||
header::USER_AGENT,
|
||||
"Telemt/1 CFNetwork/1498.700.2 Darwin/23.6.0",
|
||||
)
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(carrier_request(&stripped, "proxy.example.com").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mapped_private_addresses_are_not_learning_evidence() {
|
||||
for address in ["::ffff:127.0.0.1", "::ffff:10.0.0.1"] {
|
||||
let effective_ip = address.parse().unwrap();
|
||||
let request = Request::builder()
|
||||
.header("x-forwarded-for", address)
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(!carrier_ip_learning_eligible(&request, effective_ip));
|
||||
}
|
||||
let effective_ip = "::ffff:8.8.8.8".parse().unwrap();
|
||||
let request = Request::builder()
|
||||
.header("x-forwarded-for", "::ffff:8.8.8.8")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert!(carrier_ip_learning_eligible(&request, effective_ip));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_browser_metadata_recovers_a_stripped_capability_marker() {
|
||||
let request = Request::builder()
|
||||
.header("x-carrier-attempt", "1")
|
||||
.header(header::ORIGIN, "https://proxy.example.com")
|
||||
.header("sec-fetch-site", "same-origin")
|
||||
.header("sec-fetch-mode", "cors")
|
||||
.header("sec-fetch-dest", "empty")
|
||||
.body(())
|
||||
.unwrap();
|
||||
let parsed = carrier_request(&request, "proxy.example.com").unwrap();
|
||||
assert_eq!(parsed.class(), CarrierClientClass::BrowserHint);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_agent_learning_key_is_case_and_whitespace_normalized() {
|
||||
let first = Request::builder()
|
||||
.header(header::USER_AGENT, " Example\t Browser ")
|
||||
.body(())
|
||||
.unwrap();
|
||||
let second = Request::builder()
|
||||
.header(header::USER_AGENT, "example browser")
|
||||
.body(())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
normalized_user_agent_hash(&first),
|
||||
normalized_user_agent_hash(&second)
|
||||
);
|
||||
}
|
||||
+10
-4
@@ -1,5 +1,6 @@
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use hyper::header::{self, HeaderName, HeaderValue};
|
||||
use hyper::{Method, Request, StatusCode};
|
||||
@@ -33,17 +34,22 @@ pub(super) async fn handle_session(
|
||||
if request.headers().contains_key(header::CONTENT_TYPE) {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
let session = runtime.get_session(token_hash, &vhost.host).ok();
|
||||
if let Some(trace) = request_trace(&request)
|
||||
&& let Ok(session) = runtime.get_session(token_hash, &vhost.host)
|
||||
&& let Some(session) = &session
|
||||
{
|
||||
trace.set_route(TraceRoute::Session);
|
||||
trace.bind_identity(session.trace_identity());
|
||||
}
|
||||
let body_timeout = session.as_ref().map_or_else(
|
||||
|| Duration::from_secs(runtime.active_generation().config().web.timeouts.body_secs),
|
||||
|session| Duration::from_secs(session.timeouts().body_secs),
|
||||
);
|
||||
let CollectedBody {
|
||||
request,
|
||||
body,
|
||||
_body_budget,
|
||||
} = match collect_body(request, &runtime, 1, true).await {
|
||||
} = match collect_body(request, &runtime, body_timeout, 1, true).await {
|
||||
Ok(result) => result,
|
||||
Err(CollectBodyError::Limit) => return service_unavailable(),
|
||||
Err(CollectBodyError::Invalid(request)) => {
|
||||
@@ -62,7 +68,7 @@ pub(super) async fn handle_session(
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
let ip_learning_eligible = carrier_ip_learning_eligible(&request, client_ip);
|
||||
let Some((trace_session_id, profile)) =
|
||||
let Some((trace_session_id, profile, body_timeout)) =
|
||||
runtime.bootstrap_trace_identity(token_hash, &vhost.host)
|
||||
else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
@@ -75,7 +81,7 @@ pub(super) async fn handle_session(
|
||||
request,
|
||||
body,
|
||||
_body_budget,
|
||||
} = match collect_body(request, &runtime, CREATE_BODY_LIMIT, false).await {
|
||||
} = match collect_body(request, &runtime, body_timeout, CREATE_BODY_LIMIT, false).await {
|
||||
Ok(result) => result,
|
||||
Err(CollectBodyError::Limit) => return service_unavailable(),
|
||||
Err(CollectBodyError::Invalid(request)) => {
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
use super::*;
|
||||
|
||||
async fn open_keepalive(
|
||||
listener: &TcpListener,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
) -> (TcpStream, CancellationToken, tokio::task::JoinHandle<()>) {
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (accepted, client) = tokio::join!(listener.accept(), TcpStream::connect(addr));
|
||||
let (server, peer) = accepted.unwrap();
|
||||
let cancellation = CancellationToken::new();
|
||||
let permit = runtime.try_http_connection().unwrap();
|
||||
let task = tokio::spawn(serve_connection(
|
||||
server,
|
||||
peer,
|
||||
WebClientIpSource::XForwardedFor,
|
||||
Arc::from(["127.0.0.1/32".parse().unwrap()]),
|
||||
Arc::clone(runtime),
|
||||
cancellation.clone(),
|
||||
permit,
|
||||
));
|
||||
(client.unwrap(), cancellation, task)
|
||||
}
|
||||
|
||||
async fn read_http_response(client: &mut TcpStream) -> Vec<u8> {
|
||||
let mut response = Vec::new();
|
||||
while !response.ends_with(b"\r\n\r\n") {
|
||||
assert!(response.len() < 16 * 1024);
|
||||
response.push(client.read_u8().await.unwrap());
|
||||
}
|
||||
let content_length = std::str::from_utf8(&response)
|
||||
.unwrap()
|
||||
.lines()
|
||||
.filter_map(|line| line.split_once(':'))
|
||||
.find_map(|(name, value)| {
|
||||
name.eq_ignore_ascii_case("content-length")
|
||||
.then(|| value.trim().parse::<usize>().unwrap())
|
||||
})
|
||||
.unwrap_or(0);
|
||||
let body_start = response.len();
|
||||
response.resize(body_start + content_length, 0);
|
||||
client
|
||||
.read_exact(&mut response[body_start..])
|
||||
.await
|
||||
.unwrap();
|
||||
response
|
||||
}
|
||||
|
||||
async fn request_with_body_delay(
|
||||
listener: &TcpListener,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
head: Vec<u8>,
|
||||
body: &[u8],
|
||||
delay: std::time::Duration,
|
||||
) -> Vec<u8> {
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let (accepted, client) = tokio::join!(listener.accept(), TcpStream::connect(addr));
|
||||
let (server, peer) = accepted.unwrap();
|
||||
let mut client = client.unwrap();
|
||||
let permit = runtime.try_http_connection().unwrap();
|
||||
let task = tokio::spawn(serve_connection(
|
||||
server,
|
||||
peer,
|
||||
WebClientIpSource::XForwardedFor,
|
||||
Arc::from(["127.0.0.1/32".parse().unwrap()]),
|
||||
Arc::clone(runtime),
|
||||
CancellationToken::new(),
|
||||
permit,
|
||||
));
|
||||
client.write_all(&head).await.unwrap();
|
||||
tokio::time::sleep(delay).await;
|
||||
let _ = client.write_all(body).await;
|
||||
let mut response = Vec::new();
|
||||
client.read_to_end(&mut response).await.unwrap();
|
||||
task.await.unwrap();
|
||||
response
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn issued_bootstrap_timeouts_survive_reload_before_session_creation() {
|
||||
let capability = [20u8; 32];
|
||||
let mut initial_config = runtime_config(capability, WebCarrier::Https);
|
||||
initial_config.web.timeouts.body_secs = 3;
|
||||
initial_config.web.timeouts.long_poll_secs = 3;
|
||||
initial_config.web.timeouts.bootstrap_lifetime_secs = 5;
|
||||
let generation = test_runtime_generation(1, initial_config);
|
||||
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||
let runtime = WebProcessRuntime::start(Arc::clone(&active_runtime));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||
let root = format!(
|
||||
"GET /?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
let root_response = request(&listener, &runtime, root).await;
|
||||
let (_, root_body) = split_response(&root_response);
|
||||
let bootstrap = std::str::from_utf8(root_body)
|
||||
.unwrap()
|
||||
.split_once("bootstrap=\"")
|
||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||
.map(|(token, _)| token.to_string())
|
||||
.unwrap();
|
||||
|
||||
let mut replacement_config = runtime_config(capability, WebCarrier::Https);
|
||||
replacement_config.web.timeouts.body_secs = 1;
|
||||
replacement_config.web.timeouts.long_poll_secs = 1;
|
||||
replacement_config.web.timeouts.bootstrap_lifetime_secs = 1;
|
||||
let replacement = test_runtime_generation(2, replacement_config);
|
||||
active_runtime.store(Arc::clone(&replacement));
|
||||
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
let create_head = format!(
|
||||
"POST /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {bootstrap}\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
hello.len()
|
||||
)
|
||||
.into_bytes();
|
||||
let create_response = request_with_body_delay(
|
||||
&listener,
|
||||
&runtime,
|
||||
create_head,
|
||||
&hello,
|
||||
std::time::Duration::from_millis(1200),
|
||||
)
|
||||
.await;
|
||||
assert!(create_response.starts_with(b"HTTP/1.1 200"));
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
replacement.stop_sessions().await;
|
||||
replacement.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn live_session_body_and_closed_token_timeouts_survive_reload() {
|
||||
let capability = [21u8; 32];
|
||||
let mut initial_config = runtime_config(capability, WebCarrier::Https);
|
||||
initial_config.web.timeouts.body_secs = 3;
|
||||
initial_config.web.timeouts.header_secs = 1;
|
||||
initial_config.web.timeouts.http_idle_secs = 4;
|
||||
initial_config.web.timeouts.long_poll_secs = 3;
|
||||
initial_config.web.timeouts.bootstrap_lifetime_secs = 5;
|
||||
let generation = test_runtime_generation(1, initial_config);
|
||||
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||
let runtime = WebProcessRuntime::start(Arc::clone(&active_runtime));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||
let root = format!(
|
||||
"GET /?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
let root_response = request(&listener, &runtime, root).await;
|
||||
let (_, root_body) = split_response(&root_response);
|
||||
let bootstrap = std::str::from_utf8(root_body)
|
||||
.unwrap()
|
||||
.split_once("bootstrap=\"")
|
||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||
.map(|(token, _)| token.to_string())
|
||||
.unwrap();
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
let create_head = format!(
|
||||
"POST /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {bootstrap}\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
hello.len()
|
||||
)
|
||||
.into_bytes();
|
||||
let mut create = create_head.clone();
|
||||
create.extend_from_slice(&hello);
|
||||
let create_response = request(&listener, &runtime, create).await;
|
||||
let (create_headers, _) = split_response(&create_response);
|
||||
assert!(create_headers.starts_with(b"HTTP/1.1 200"));
|
||||
let session = response_header(create_headers, "x-session-token").to_string();
|
||||
|
||||
let mut replacement_config = runtime_config(capability, WebCarrier::Https);
|
||||
replacement_config.web.timeouts.body_secs = 1;
|
||||
replacement_config.web.timeouts.header_secs = 1;
|
||||
replacement_config.web.timeouts.http_idle_secs = 2;
|
||||
replacement_config.web.timeouts.long_poll_secs = 1;
|
||||
replacement_config.web.timeouts.bootstrap_lifetime_secs = 1;
|
||||
let replacement = test_runtime_generation(2, replacement_config);
|
||||
active_runtime.store(Arc::clone(&replacement));
|
||||
|
||||
let retry_response = request_with_body_delay(
|
||||
&listener,
|
||||
&runtime,
|
||||
create_head,
|
||||
&hello,
|
||||
std::time::Duration::from_millis(1200),
|
||||
)
|
||||
.await;
|
||||
let (retry_headers, _) = split_response(&retry_response);
|
||||
assert!(retry_headers.starts_with(b"HTTP/1.1 200"));
|
||||
assert_eq!(response_header(retry_headers, "x-session-token"), session);
|
||||
|
||||
let down = format!(
|
||||
"POST /api/v1/down HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {session}\r\nX-Down-Cursor: 0\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
let down_response = request(&listener, &runtime, down).await;
|
||||
assert!(down_response.starts_with(b"HTTP/1.1 204"));
|
||||
|
||||
let close = format!(
|
||||
"DELETE /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {session}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
.into_bytes();
|
||||
let close_response = request(&listener, &runtime, close.clone()).await;
|
||||
assert!(close_response.starts_with(b"HTTP/1.1 204"));
|
||||
tokio::time::sleep(std::time::Duration::from_millis(1500)).await;
|
||||
let close_retry_response = request(&listener, &runtime, close).await;
|
||||
assert!(close_retry_response.starts_with(b"HTTP/1.1 204"));
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
replacement.stop_sessions().await;
|
||||
replacement.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn active_body_deadline_survives_reload_on_old_keepalive_connection() {
|
||||
let capability = [22u8; 32];
|
||||
let mut initial_config = runtime_config(capability, WebCarrier::Https);
|
||||
initial_config.web.timeouts.header_secs = 1;
|
||||
initial_config.web.timeouts.body_secs = 1;
|
||||
initial_config.web.timeouts.long_poll_secs = 1;
|
||||
initial_config.web.timeouts.http_idle_secs = 2;
|
||||
let generation = test_runtime_generation(1, initial_config);
|
||||
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||
let runtime = WebProcessRuntime::start(Arc::clone(&active_runtime));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let (mut client, cancellation, task) = open_keepalive(&listener, &runtime).await;
|
||||
client
|
||||
.write_all(
|
||||
b"GET / HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\n\r\n",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
read_http_response(&mut client)
|
||||
.await
|
||||
.starts_with(b"HTTP/1.1 200")
|
||||
);
|
||||
|
||||
let mut replacement_config = runtime_config(capability, WebCarrier::Https);
|
||||
replacement_config.web.timeouts.header_secs = 1;
|
||||
replacement_config.web.timeouts.body_secs = 3;
|
||||
replacement_config.web.timeouts.long_poll_secs = 1;
|
||||
replacement_config.web.timeouts.http_idle_secs = 4;
|
||||
let replacement = test_runtime_generation(2, replacement_config);
|
||||
active_runtime.store(Arc::clone(&replacement));
|
||||
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||
client
|
||||
.write_all(
|
||||
format!(
|
||||
"GET /?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\n\r\n"
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let bridge = read_http_response(&mut client).await;
|
||||
let (_, bridge_body) = split_response(&bridge);
|
||||
let bootstrap = std::str::from_utf8(bridge_body)
|
||||
.unwrap()
|
||||
.split_once("bootstrap=\"")
|
||||
.and_then(|(_, suffix)| suffix.split_once('"'))
|
||||
.map(|(token, _)| token.to_string())
|
||||
.unwrap();
|
||||
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||
client
|
||||
.write_all(
|
||||
format!(
|
||||
"POST /api/v1/session HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.10\r\nAuthorization: Bearer {bootstrap}\r\nContent-Type: application/octet-stream\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
hello.len()
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(2200)).await;
|
||||
client.write_all(&hello).await.unwrap();
|
||||
|
||||
assert!(
|
||||
read_http_response(&mut client)
|
||||
.await
|
||||
.starts_with(b"HTTP/1.1 200")
|
||||
);
|
||||
|
||||
cancellation.cancel();
|
||||
drop(client);
|
||||
task.await.unwrap();
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
replacement.stop_sessions().await;
|
||||
replacement.stop_background_tasks().await;
|
||||
}
|
||||
+30
-1
@@ -15,12 +15,21 @@ use crate::config::{
|
||||
};
|
||||
use crate::maestro::generation::test_runtime_generation;
|
||||
use crate::web::frame::{self, FrameType};
|
||||
use crate::web::manager::WebProcessRuntime;
|
||||
use crate::web::manager::{
|
||||
CloseOperationSelector, ControlError, SessionDetail, SessionFilter, SessionListRequest,
|
||||
SessionRefError, WebProcessRuntime,
|
||||
};
|
||||
|
||||
#[path = "legacy_tests.rs"]
|
||||
mod legacy_tests;
|
||||
#[path = "negotiation_tests.rs"]
|
||||
mod negotiation_tests;
|
||||
// Reload-stability tests for session-owned timeout policy.
|
||||
#[path = "session_policy_tests.rs"]
|
||||
mod session_policy_tests;
|
||||
// Runtime control integration stays separate from carrier protocol scenarios.
|
||||
#[path = "control_tests.rs"]
|
||||
mod control_tests;
|
||||
|
||||
const TEST_CARRIER_DEADLINES_SECS: [u64; 4] = [3, 5, 8, 12];
|
||||
|
||||
@@ -391,6 +400,26 @@ async fn unused_bootstrap_survives_equivalent_runtime_generation_swap() {
|
||||
replacement.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bridge_bootstrap_uses_the_generation_that_selected_its_profile() {
|
||||
let initial = test_runtime_generation(1, runtime_config([21; 32], WebCarrier::Https));
|
||||
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&initial)));
|
||||
let runtime = WebProcessRuntime::start(Arc::clone(&active_runtime));
|
||||
let profile = initial.config().web.runtime.as_ref().unwrap().profiles[0].clone();
|
||||
let replacement = test_runtime_generation(2, runtime_config([22; 32], WebCarrier::HttpsLanes));
|
||||
active_runtime.store(Arc::clone(&replacement));
|
||||
|
||||
let result =
|
||||
runtime.issue_bootstrap_for_generation(&initial, profile, "192.0.2.10".parse().unwrap());
|
||||
|
||||
assert!(result.is_ok());
|
||||
runtime.shutdown().await;
|
||||
initial.stop_sessions().await;
|
||||
initial.stop_background_tasks().await;
|
||||
replacement.stop_sessions().await;
|
||||
replacement.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unused_bootstrap_is_rejected_after_profile_identity_change() {
|
||||
let capability = [11u8; 32];
|
||||
|
||||
@@ -254,6 +254,7 @@ pub(super) async fn handle(
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
vhost: Arc<WebRuntimeVhost>,
|
||||
) -> HttpResponse {
|
||||
let request_deadline = super::request_deadline(&request);
|
||||
let Some(parsed) = parse_upgrade(&request) else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
@@ -284,7 +285,16 @@ pub(super) async fn handle(
|
||||
},
|
||||
};
|
||||
let timeouts = session.timeouts().clone();
|
||||
let connection = match runtime
|
||||
let admission_lease = match request_deadline.as_ref() {
|
||||
Some(deadline) => {
|
||||
match deadline.lease_for(Duration::from_secs(timeouts.websocket_eviction_secs)) {
|
||||
Some(lease) => Some(lease),
|
||||
None => return serve_decoy(request, vhost, true, &runtime).await,
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let admitted = runtime
|
||||
.admit_websocket(
|
||||
session.profile_key(),
|
||||
session.trace_session_id(),
|
||||
@@ -296,11 +306,17 @@ pub(super) async fn handle(
|
||||
Duration::from_secs(timeouts.websocket_eviction_secs),
|
||||
session.carrier_cancellation(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
.await;
|
||||
drop(admission_lease);
|
||||
let connection = match admitted {
|
||||
Ok(connection) => connection,
|
||||
Err(_) => return serve_decoy(request, vhost, true, &runtime).await,
|
||||
};
|
||||
if let Some(reservation) = lane_reservation.as_mut()
|
||||
&& reservation.bind(connection.id()).is_err()
|
||||
{
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
}
|
||||
if let Some(reservation) = probe_reservation.as_mut()
|
||||
&& reservation.bind(connection.id()).is_err()
|
||||
{
|
||||
@@ -323,6 +339,20 @@ pub(super) async fn handle(
|
||||
trace.bind_identity(session.trace_identity());
|
||||
trace.register_redaction(parsed.protocol.as_bytes());
|
||||
}
|
||||
let upgrade_deadline = match request_deadline.as_ref() {
|
||||
Some(deadline) => {
|
||||
let Some(until) = tokio::time::Instant::now()
|
||||
.checked_add(Duration::from_secs(timeouts.websocket_upgrade_secs))
|
||||
else {
|
||||
return serve_decoy(request, vhost, true, &runtime).await;
|
||||
};
|
||||
match deadline.upgrade_until(until) {
|
||||
Some(lease) => Some(lease),
|
||||
None => return serve_decoy(request, vhost, true, &runtime).await,
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let on_upgrade = hyper::upgrade::on(&mut request);
|
||||
let protocol = parsed.protocol;
|
||||
let accept = parsed.accept;
|
||||
@@ -331,6 +361,7 @@ pub(super) async fn handle(
|
||||
runtime.spawn_auxiliary(async move {
|
||||
driver::run_upgraded(
|
||||
on_upgrade,
|
||||
upgrade_deadline,
|
||||
driver_runtime,
|
||||
driver_session,
|
||||
connection,
|
||||
|
||||
@@ -8,6 +8,7 @@ use tokio_tungstenite::tungstenite::protocol::{Message, Role, WebSocketConfig};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::ConnectionIo;
|
||||
use crate::web::http::activity::UpgradeDeadlineLease;
|
||||
use crate::web::manager::{WebProcessRuntime, WebSocketBudgetLease, WebSocketConnection};
|
||||
use crate::web::session::{WebSession, WebSocketLaneReservation, WebSocketProbeReservation};
|
||||
use crate::web::trace::{TraceDirection, TraceWebSocketContext};
|
||||
@@ -22,8 +23,11 @@ mod lane;
|
||||
use io::{flush, process_multiplex, read_message, record_message, reserve_data, send};
|
||||
use lane::run_lane;
|
||||
|
||||
// Upgrade ownership remains explicit across cancellation and reservation boundaries.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn run_upgraded(
|
||||
on_upgrade: hyper::upgrade::OnUpgrade,
|
||||
upgrade_deadline: Option<UpgradeDeadlineLease>,
|
||||
runtime: Arc<WebProcessRuntime>,
|
||||
session: Arc<WebSession>,
|
||||
connection: WebSocketConnection,
|
||||
@@ -34,13 +38,15 @@ pub(super) async fn run_upgraded(
|
||||
) {
|
||||
let cancellation = connection.cancellation();
|
||||
let timeouts = session.timeouts().clone();
|
||||
let deadline = upgrade_deadline.as_ref().map_or_else(
|
||||
|| tokio::time::Instant::now() + Duration::from_secs(timeouts.websocket_upgrade_secs),
|
||||
UpgradeDeadlineLease::deadline,
|
||||
);
|
||||
let upgraded = tokio::select! {
|
||||
_ = cancellation.cancelled() => return,
|
||||
result = tokio::time::timeout(
|
||||
Duration::from_secs(timeouts.websocket_upgrade_secs),
|
||||
on_upgrade,
|
||||
) => result,
|
||||
result = tokio::time::timeout_at(deadline, on_upgrade) => result,
|
||||
};
|
||||
drop(upgrade_deadline);
|
||||
let Ok(Ok(upgraded)) = upgraded else {
|
||||
return;
|
||||
};
|
||||
@@ -95,8 +101,7 @@ pub(super) async fn run_upgraded(
|
||||
_ = tokio::time::timeout(eviction, socket.close(None)) => {}
|
||||
}
|
||||
if let Some(reservation) = lane_reservation {
|
||||
session.close_websocket_lane(reservation.lane_id());
|
||||
drop(reservation);
|
||||
session.close_websocket_lane(reservation);
|
||||
} else if !acknowledge_commit || session.is_carrier_committed() {
|
||||
session.close();
|
||||
}
|
||||
@@ -192,10 +197,12 @@ async fn run_multiplex(
|
||||
session.close();
|
||||
return Err(());
|
||||
}
|
||||
} else if acknowledge_commit && sequence > 1 && progressed {
|
||||
if !session.websocket_peer_after_commit_ack(connection.id()) {
|
||||
return Err(());
|
||||
}
|
||||
} else if acknowledge_commit
|
||||
&& sequence > 1
|
||||
&& progressed
|
||||
&& !session.websocket_peer_after_commit_ack(connection.id())
|
||||
{
|
||||
return Err(());
|
||||
}
|
||||
if !active && progressed {
|
||||
if !connection.mark_active() {
|
||||
|
||||
@@ -12,6 +12,7 @@ use crate::web::session::{WebSession, WebSocketLaneReservation};
|
||||
use crate::web::trace::{TraceDirection, TraceWebSocketContext};
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
/// Drives one exact WebSocket lane until its isolated failure boundary closes.
|
||||
pub(super) async fn run_lane(
|
||||
socket: &mut CarrierSocket,
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
@@ -35,7 +36,7 @@ pub(super) async fn run_lane(
|
||||
let maximum_message = session.limits().carrier_batch_bytes;
|
||||
let mut active = false;
|
||||
loop {
|
||||
let down = session.poll_down_lane(reservation.lane_id(), cursor);
|
||||
let down = session.poll_down_websocket_lane(reservation.lane_identity(), cursor);
|
||||
tokio::pin!(down);
|
||||
let event = tokio::select! {
|
||||
_ = cancellation.cancelled() => return Err(()),
|
||||
@@ -106,10 +107,12 @@ pub(super) async fn run_lane(
|
||||
session.close();
|
||||
return Err(());
|
||||
}
|
||||
} else if acknowledge_commit && sequence > 1 && progressed {
|
||||
if !session.websocket_peer_after_commit_ack(connection.id()) {
|
||||
return Err(());
|
||||
}
|
||||
} else if acknowledge_commit
|
||||
&& sequence > 1
|
||||
&& progressed
|
||||
&& !session.websocket_peer_after_commit_ack(connection.id())
|
||||
{
|
||||
return Err(());
|
||||
}
|
||||
if !active && progressed {
|
||||
if !connection.mark_active() {
|
||||
|
||||
+41
-6
@@ -33,12 +33,21 @@ mod carrier_outcome;
|
||||
mod admission;
|
||||
// Shutdown and expiry work remain outside request-path coordination.
|
||||
mod lifecycle;
|
||||
pub(crate) use lifecycle::WebShutdownOutcome;
|
||||
// Queue and WebSocket allocations share one process-owned data-plane budget.
|
||||
mod budget;
|
||||
// WebSocket admission, replacement, and liveness are process-scoped.
|
||||
mod websocket;
|
||||
// Bounded read-only snapshots and opaque session references serve the API.
|
||||
mod status;
|
||||
pub(crate) use status::{
|
||||
SessionDetail, SessionFilter, SessionListRequest, SessionRefError, WebRuntimeStatus,
|
||||
};
|
||||
// Asynchronous bounded close operations isolate mutation lifecycle from HTTP requests.
|
||||
mod control;
|
||||
pub(crate) use budget::WebSocketBudgetLease;
|
||||
use budget::{WebDataBudget, WebSocketBudgetClass};
|
||||
pub(crate) use control::{CloseOperationSelector, ControlError};
|
||||
pub(crate) use negotiation::{
|
||||
CarrierCapabilities, CarrierClientClass, CarrierFailure, CarrierLearningContext, CarrierRequest,
|
||||
};
|
||||
@@ -127,6 +136,7 @@ pub(crate) struct BootstrapResult {
|
||||
|
||||
/// Process-owned bounded WEB credential, session, and memory coordinator.
|
||||
pub(crate) struct WebProcessRuntime {
|
||||
runtime_instance: Arc<str>,
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
trace: Arc<WebTraceStore>,
|
||||
limits: WebLimitsConfig,
|
||||
@@ -146,6 +156,8 @@ pub(crate) struct WebProcessRuntime {
|
||||
websocket_clock: std::time::Instant,
|
||||
websocket_notify: Arc<Notify>,
|
||||
data_budget: Arc<WebDataBudget>,
|
||||
control_operations: Mutex<control::ControlOperationRegistry>,
|
||||
next_control_operation_id: AtomicU64,
|
||||
shutdown: CancellationToken,
|
||||
tasks: TaskTracker,
|
||||
sessions_created: AtomicU64,
|
||||
@@ -171,7 +183,9 @@ impl WebProcessRuntime {
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
trace: Arc<WebTraceStore>,
|
||||
) -> Arc<Self> {
|
||||
let config = active_runtime.load().config();
|
||||
let initial_generation = active_runtime.load_full();
|
||||
let config = initial_generation.config();
|
||||
trace.apply_policy(initial_generation.id, &config.web.debug);
|
||||
let limits = config.web.limits.clone();
|
||||
let learning_capacity = limits.max_carrier_learning_entries;
|
||||
let mut carrier_learning = learning::CarrierLearning::new(learning_capacity);
|
||||
@@ -187,6 +201,7 @@ impl WebProcessRuntime {
|
||||
let lane_poll_limit = limits.max_http_handlers / 2;
|
||||
let lane_aux_poll_limit = (lane_poll_limit / 2).max(1);
|
||||
let runtime = Arc::new(Self {
|
||||
runtime_instance: Arc::from(format!("{:032x}", rand::random::<u128>())),
|
||||
active_runtime,
|
||||
trace,
|
||||
http_connections: Arc::new(Semaphore::new(limits.max_http_connections)),
|
||||
@@ -202,8 +217,10 @@ impl WebProcessRuntime {
|
||||
websocket_clock: std::time::Instant::now(),
|
||||
websocket_notify: Arc::new(Notify::new()),
|
||||
data_budget: WebDataBudget::new(limits.clone()),
|
||||
control_operations: Mutex::new(control::ControlOperationRegistry::default()),
|
||||
next_control_operation_id: AtomicU64::new(1),
|
||||
limits,
|
||||
state: Mutex::new(ManagerState::default()),
|
||||
state: Mutex::new(ManagerState::new(initial_generation.id, config.web.enabled)),
|
||||
stream_admission: Mutex::new(StreamAdmissionState::default()),
|
||||
learning: Mutex::new(carrier_learning),
|
||||
shutdown: CancellationToken::new(),
|
||||
@@ -228,8 +245,11 @@ impl WebProcessRuntime {
|
||||
let Some(runtime) = weak.upgrade() else {
|
||||
break;
|
||||
};
|
||||
let policy = runtime.active_generation().config().web.debug.clone();
|
||||
runtime.trace.apply_policy(&policy);
|
||||
let generation = runtime.active_generation();
|
||||
let policy = generation.config().web.debug.clone();
|
||||
runtime
|
||||
.trace
|
||||
.apply_policy(generation.id, &policy);
|
||||
runtime.cleanup();
|
||||
}
|
||||
}
|
||||
@@ -243,6 +263,11 @@ impl WebProcessRuntime {
|
||||
self.active_runtime.load_full()
|
||||
}
|
||||
|
||||
/// Returns the random process-instance fence used by control-plane references.
|
||||
pub(crate) fn runtime_instance(&self) -> &str {
|
||||
&self.runtime_instance
|
||||
}
|
||||
|
||||
/// Returns the process-owned WEB debug trace store.
|
||||
pub(crate) fn trace(&self) -> &Arc<WebTraceStore> {
|
||||
&self.trace
|
||||
@@ -294,13 +319,23 @@ impl WebProcessRuntime {
|
||||
where
|
||||
F: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
if self.shutdown.is_cancelled() {
|
||||
drop(future);
|
||||
return;
|
||||
}
|
||||
let shutdown = self.shutdown.clone();
|
||||
self.tasks.spawn(async move {
|
||||
let tracked = self.tasks.track_future(async move {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.cancelled() => {}
|
||||
_ = future => {}
|
||||
}
|
||||
});
|
||||
if self.shutdown.is_cancelled() {
|
||||
drop(tracked);
|
||||
return;
|
||||
}
|
||||
drop(tokio::spawn(tracked));
|
||||
}
|
||||
|
||||
/// Reserves one body reader and its declared bounded body allocation.
|
||||
@@ -393,7 +428,7 @@ impl WebProcessRuntime {
|
||||
.try_reserve_websocket(owner, bytes, WebSocketBudgetClass::Data)
|
||||
}
|
||||
|
||||
/// Admits one WebSocket with owner-first bounded replacement.
|
||||
/// Admits one WebSocket with dead-first, then owner-local bounded replacement.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn admit_websocket(
|
||||
self: &Arc<Self>,
|
||||
|
||||
+49
-11
@@ -52,10 +52,33 @@ pub(crate) struct WebDataBudgetSnapshot {
|
||||
pub(crate) queue_bytes: usize,
|
||||
/// Total queue items currently retained.
|
||||
pub(crate) queue_items: usize,
|
||||
/// Control bytes included in the queue total.
|
||||
pub(crate) queue_control_bytes: usize,
|
||||
/// Control items included in the queue total.
|
||||
pub(crate) queue_control_items: usize,
|
||||
/// Total WebSocket bytes currently retained.
|
||||
pub(crate) websocket_bytes: usize,
|
||||
/// Largest combined byte usage observed since process start.
|
||||
pub(crate) high_water_bytes: usize,
|
||||
/// Distinct profile owners currently charged.
|
||||
pub(crate) owners: usize,
|
||||
/// Whether shutdown closed this allocation authority.
|
||||
pub(crate) closed: bool,
|
||||
}
|
||||
|
||||
/// Bounded owner-usage view captured before WebSocket registry selection.
|
||||
pub(super) struct WebSocketFairnessSnapshot {
|
||||
/// Equal byte share at the admission watermark for captured owners.
|
||||
pub(super) fair_share: usize,
|
||||
/// Captured shared-budget use indexed by profile owner.
|
||||
pub(super) owner_bytes: HashMap<ProfileKey, usize>,
|
||||
}
|
||||
|
||||
impl WebSocketFairnessSnapshot {
|
||||
/// Returns the captured byte usage for one quota owner.
|
||||
pub(super) fn owner_usage(&self, owner: ProfileKey) -> usize {
|
||||
self.owner_bytes.get(&owner).copied().unwrap_or(0)
|
||||
}
|
||||
}
|
||||
|
||||
impl WebDataBudget {
|
||||
@@ -220,16 +243,10 @@ impl WebDataBudget {
|
||||
self.pressured.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub(super) fn owner_usage(&self, owner: ProfileKey) -> usize {
|
||||
self.state
|
||||
.lock()
|
||||
.owner_bytes
|
||||
.get(&owner)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
pub(super) fn fair_share(&self, additional_owner: Option<ProfileKey>) -> usize {
|
||||
pub(super) fn fairness_snapshot(
|
||||
&self,
|
||||
additional_owner: Option<ProfileKey>,
|
||||
) -> WebSocketFairnessSnapshot {
|
||||
let state = self.state.lock();
|
||||
let mut owners = state.owner_bytes.len();
|
||||
if additional_owner.is_some_and(|owner| !state.owner_bytes.contains_key(&owner)) {
|
||||
@@ -239,7 +256,10 @@ impl WebDataBudget {
|
||||
self.limits.websocket_bytes_global,
|
||||
self.limits.websocket_admission_watermark_pct,
|
||||
);
|
||||
admission / owners.max(1)
|
||||
WebSocketFairnessSnapshot {
|
||||
fair_share: admission / owners.max(1),
|
||||
owner_bytes: state.owner_bytes.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn snapshot(&self) -> WebDataBudgetSnapshot {
|
||||
@@ -247,11 +267,29 @@ impl WebDataBudget {
|
||||
WebDataBudgetSnapshot {
|
||||
queue_bytes: state.queue_bytes,
|
||||
queue_items: state.queue_items,
|
||||
queue_control_bytes: state.queue_control_bytes,
|
||||
queue_control_items: state.queue_control_items,
|
||||
websocket_bytes: state.websocket_bytes,
|
||||
high_water_bytes: state.high_water_bytes,
|
||||
owners: state.owner_bytes.len(),
|
||||
closed: state.closed,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn try_snapshot(&self) -> Option<WebDataBudgetSnapshot> {
|
||||
let state = self.state.try_lock()?;
|
||||
Some(WebDataBudgetSnapshot {
|
||||
queue_bytes: state.queue_bytes,
|
||||
queue_items: state.queue_items,
|
||||
queue_control_bytes: state.queue_control_bytes,
|
||||
queue_control_items: state.queue_control_items,
|
||||
websocket_bytes: state.websocket_bytes,
|
||||
high_water_bytes: state.high_water_bytes,
|
||||
owners: state.owner_bytes.len(),
|
||||
closed: state.closed,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn close(&self) {
|
||||
self.state.lock().closed = true;
|
||||
self.notify.notify_waiters();
|
||||
|
||||
@@ -163,6 +163,34 @@ pub(super) struct CarrierLearning {
|
||||
policy_started_at: Instant,
|
||||
}
|
||||
|
||||
/// Bounded control-plane summary of carrier-learning state.
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct CarrierLearningStatus {
|
||||
/// Whether outcome learning is active in the effective policy.
|
||||
pub(crate) enabled: bool,
|
||||
/// Effective evidence thresholds.
|
||||
pub(crate) aggressiveness: WebCarrierNegotiationAggressiveness,
|
||||
/// Current evidence epoch, or none after counter exhaustion.
|
||||
pub(crate) epoch: Option<u64>,
|
||||
/// Retained evidence entries.
|
||||
pub(crate) entries: usize,
|
||||
/// Restart-owned evidence ceiling.
|
||||
pub(crate) capacity: usize,
|
||||
/// Effective evidence lifetime.
|
||||
pub(crate) lifetime_secs: u64,
|
||||
/// Monotonic age of the current policy epoch.
|
||||
pub(crate) age_ms: u64,
|
||||
}
|
||||
|
||||
/// Result of one epoch-fenced learning reset.
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct CarrierLearningResetOutcome {
|
||||
/// Evidence entries detached by the reset.
|
||||
pub(crate) entries_cleared: usize,
|
||||
/// New epoch fencing pre-reset outcomes.
|
||||
pub(crate) epoch: u64,
|
||||
}
|
||||
|
||||
impl CarrierLearning {
|
||||
/// Creates an empty store under the restart-owned capacity ceiling.
|
||||
pub(super) fn new(capacity: usize) -> Self {
|
||||
@@ -177,6 +205,23 @@ impl CarrierLearning {
|
||||
}
|
||||
}
|
||||
|
||||
fn status(&self, now: Instant) -> CarrierLearningStatus {
|
||||
let policy = self.policy.unwrap_or(LearningPolicy {
|
||||
enabled: false,
|
||||
aggressiveness: WebCarrierNegotiationAggressiveness::Conservative,
|
||||
lifetime: Duration::ZERO,
|
||||
});
|
||||
CarrierLearningStatus {
|
||||
enabled: policy.enabled,
|
||||
aggressiveness: policy.aggressiveness,
|
||||
epoch: self.epoch,
|
||||
entries: self.entries.len(),
|
||||
capacity: self.capacity,
|
||||
lifetime_secs: policy.lifetime.as_secs(),
|
||||
age_ms: millis(now.saturating_duration_since(self.policy_started_at)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies hot-reloaded learning policy and returns its outcome epoch.
|
||||
pub(super) fn apply_policy(
|
||||
&mut self,
|
||||
@@ -424,6 +469,51 @@ impl CarrierLearning {
|
||||
}
|
||||
}
|
||||
|
||||
impl super::WebProcessRuntime {
|
||||
/// Captures learning state without waiting for a contended evidence lock.
|
||||
pub(crate) fn try_carrier_learning_status(&self) -> Option<CarrierLearningStatus> {
|
||||
self.learning
|
||||
.try_lock()
|
||||
.map(|learning| learning.status(Instant::now()))
|
||||
}
|
||||
|
||||
/// Clears all evidence under a new epoch without changing the active policy.
|
||||
pub(crate) fn reset_carrier_learning(
|
||||
&self,
|
||||
) -> Result<CarrierLearningResetOutcome, super::ManagerError> {
|
||||
let control = self
|
||||
.control_mutation_guard()
|
||||
.map_err(|_| super::ManagerError::Closed)?;
|
||||
let (outcome, retired_entries, retired_order) = {
|
||||
let mut learning = self.learning.lock();
|
||||
let epoch = learning
|
||||
.epoch
|
||||
.and_then(|epoch| epoch.checked_add(1))
|
||||
.ok_or(super::ManagerError::Closed)?;
|
||||
learning.epoch = Some(epoch);
|
||||
learning.insertion_sequence = 1;
|
||||
learning.policy_started_at = Instant::now();
|
||||
let retired_entries = std::mem::take(&mut learning.entries);
|
||||
let retired_order = std::mem::take(&mut learning.insertion_order);
|
||||
(
|
||||
CarrierLearningResetOutcome {
|
||||
entries_cleared: retired_entries.len(),
|
||||
epoch,
|
||||
},
|
||||
retired_entries,
|
||||
retired_order,
|
||||
)
|
||||
};
|
||||
drop(control);
|
||||
drop((retired_entries, retired_order));
|
||||
Ok(outcome)
|
||||
}
|
||||
}
|
||||
|
||||
fn millis(duration: Duration) -> u64 {
|
||||
duration.as_millis().min(u128::from(u64::MAX)) as u64
|
||||
}
|
||||
|
||||
fn supported(configured: &[WebCarrier], request: super::CarrierRequest) -> Vec<WebCarrier> {
|
||||
configured
|
||||
.iter()
|
||||
|
||||
@@ -220,3 +220,52 @@ fn fifo_metadata_stays_within_the_entry_capacity() {
|
||||
assert!(learning.insertion_order.len() <= 3);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_reset_preserves_policy_and_rejects_old_epoch_outcomes() {
|
||||
let generation = crate::maestro::generation::test_runtime_generation(
|
||||
1,
|
||||
crate::config::ProxyConfig::default(),
|
||||
);
|
||||
let runtime = crate::web::manager::WebProcessRuntime::start(std::sync::Arc::new(
|
||||
arc_swap::ArcSwap::from(generation.clone()),
|
||||
));
|
||||
let now = Instant::now();
|
||||
let old_epoch = {
|
||||
let mut learning = runtime.learning.lock();
|
||||
let epoch = learning
|
||||
.apply_policy(
|
||||
now,
|
||||
true,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
learning.record_chain(now, epoch, context(7), &[], WebCarrier::Websocket);
|
||||
epoch
|
||||
};
|
||||
|
||||
let outcome = runtime.reset_carrier_learning().unwrap();
|
||||
{
|
||||
let mut learning = runtime.learning.lock();
|
||||
learning.record_chain(
|
||||
Instant::now(),
|
||||
old_epoch,
|
||||
context(7),
|
||||
&[],
|
||||
WebCarrier::Https,
|
||||
);
|
||||
let status = learning.status(Instant::now());
|
||||
assert!(status.enabled);
|
||||
assert_eq!(
|
||||
status.aggressiveness,
|
||||
WebCarrierNegotiationAggressiveness::Aggressive
|
||||
);
|
||||
assert_eq!(status.entries, 0);
|
||||
assert_eq!(status.epoch, Some(outcome.epoch));
|
||||
}
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,441 @@
|
||||
use std::collections::{BTreeSet, VecDeque};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::Instant;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
use super::status::immutable_matches;
|
||||
use super::{SessionFilter, WebProcessRuntime};
|
||||
|
||||
const OPERATION_REF_VERSION: &str = "wo1";
|
||||
const OPERATION_RETENTION: usize = 32;
|
||||
const CLOSE_CHUNK: usize = 128;
|
||||
|
||||
/// Validated bulk-close selector.
|
||||
#[derive(Clone)]
|
||||
pub(crate) enum CloseOperationSelector {
|
||||
/// Exact logical session references resolved by the API.
|
||||
Refs(Vec<u64>),
|
||||
/// Point-in-time sessions matching a bounded filter.
|
||||
Filter(SessionFilter),
|
||||
/// Every logical session at or below the submission high-water mark.
|
||||
All,
|
||||
}
|
||||
|
||||
/// Stable close-operation state.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(crate) enum ControlOperationState {
|
||||
/// Accepted but not yet executing.
|
||||
Queued,
|
||||
/// Scanning the bounded point-in-time registry.
|
||||
Running,
|
||||
/// Finished the complete bounded scan.
|
||||
Completed,
|
||||
/// Stopped because process shutdown began.
|
||||
Cancelled,
|
||||
/// Stopped on one sanitized internal failure.
|
||||
Failed,
|
||||
}
|
||||
|
||||
/// Retained status for one asynchronous close operation.
|
||||
#[derive(Clone, Serialize)]
|
||||
pub(crate) struct ControlOperationStatus {
|
||||
/// Opaque process-fenced operation reference.
|
||||
pub(crate) operation_id: String,
|
||||
/// Current lifecycle state.
|
||||
pub(crate) state: ControlOperationState,
|
||||
/// Highest logical session eligible for this point-in-time operation.
|
||||
pub(crate) high_water_session_ref: Option<String>,
|
||||
/// Exact submitted reference count, or zero for filter/all selectors.
|
||||
pub(crate) requested: usize,
|
||||
/// Registry candidates visited so far.
|
||||
pub(crate) scanned: usize,
|
||||
/// Candidates that matched the complete selector.
|
||||
pub(crate) matched: usize,
|
||||
/// Matching session incarnations sent a close signal.
|
||||
pub(crate) close_signalled: usize,
|
||||
/// Matching candidates replaced or closed before signalling.
|
||||
pub(crate) conflicted: usize,
|
||||
/// Wall-clock creation timestamp for operator correlation.
|
||||
pub(crate) created_epoch_millis: u64,
|
||||
/// Wall-clock timestamp of the latest status mutation.
|
||||
pub(crate) updated_epoch_millis: u64,
|
||||
/// Stable sanitized failure token.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) failure: Option<&'static str>,
|
||||
}
|
||||
|
||||
/// Runtime control validation failure.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ControlError {
|
||||
/// The supplied process-instance fence is stale.
|
||||
StaleInstance,
|
||||
/// The selector is empty or exceeds its bound.
|
||||
InvalidSelector,
|
||||
/// Close-all was requested before issuance stopped.
|
||||
IssuanceEnabled,
|
||||
/// The single operation slot is occupied.
|
||||
OperationInProgress,
|
||||
/// The operation reference is not canonical.
|
||||
InvalidOperation,
|
||||
/// The canonical operation is outside retained history.
|
||||
OperationNotFound,
|
||||
/// Process shutdown has closed the mutation gate.
|
||||
Closed,
|
||||
}
|
||||
|
||||
/// Single-slot execution gate with bounded terminal history.
|
||||
#[derive(Default)]
|
||||
pub(super) struct ControlOperationRegistry {
|
||||
active: Option<u64>,
|
||||
retained: VecDeque<(u64, ControlOperationStatus)>,
|
||||
closed: bool,
|
||||
}
|
||||
|
||||
struct WorkCandidate {
|
||||
trace_session_id: u64,
|
||||
session: Arc<crate::web::session::WebSession>,
|
||||
bootstrap_hash: super::TokenHash,
|
||||
}
|
||||
|
||||
impl WebProcessRuntime {
|
||||
/// Acquires the process mutation gate without permitting post-shutdown work.
|
||||
pub(super) fn control_mutation_guard(
|
||||
&self,
|
||||
) -> Result<parking_lot::MutexGuard<'_, ControlOperationRegistry>, ControlError> {
|
||||
let operations = self.control_operations.lock();
|
||||
if operations.closed || self.shutdown.is_cancelled() {
|
||||
return Err(ControlError::Closed);
|
||||
}
|
||||
Ok(operations)
|
||||
}
|
||||
|
||||
/// Clears debug records under the process mutation gate.
|
||||
pub(crate) fn clear_debug(&self) -> Result<crate::web::trace::TraceClearOutcome, ControlError> {
|
||||
let _control = self.control_mutation_guard()?;
|
||||
Ok(self.trace.clear())
|
||||
}
|
||||
|
||||
/// Starts one bounded point-in-time close sweep.
|
||||
pub(crate) fn start_close_operation(
|
||||
self: &Arc<Self>,
|
||||
runtime_instance: &str,
|
||||
selector: CloseOperationSelector,
|
||||
) -> Result<ControlOperationStatus, ControlError> {
|
||||
if runtime_instance != self.runtime_instance() {
|
||||
return Err(ControlError::StaleInstance);
|
||||
}
|
||||
if matches!(&selector, CloseOperationSelector::Refs(refs) if refs.is_empty() || refs.len() > 200)
|
||||
|| matches!(&selector, CloseOperationSelector::Filter(filter) if filter.is_empty())
|
||||
{
|
||||
return Err(ControlError::InvalidSelector);
|
||||
}
|
||||
let generation = self.active_generation();
|
||||
let web_enabled = generation.config().web.enabled;
|
||||
let (high_water, issuance_enabled) = {
|
||||
let mut state = self.state.lock();
|
||||
state.apply_issuance_policy(generation.id, web_enabled);
|
||||
(
|
||||
state.session_index.last_key_value().map(|(id, _)| *id),
|
||||
state.issuance_enabled,
|
||||
)
|
||||
};
|
||||
if matches!(selector, CloseOperationSelector::All) && issuance_enabled {
|
||||
return Err(ControlError::IssuanceEnabled);
|
||||
}
|
||||
if self.shutdown.is_cancelled() {
|
||||
return Err(ControlError::Closed);
|
||||
}
|
||||
let sequence = self
|
||||
.next_control_operation_id
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
let operation_id = self.operation_ref(sequence);
|
||||
let now = crate::web::trace::store_epoch_millis();
|
||||
let requested = match &selector {
|
||||
CloseOperationSelector::Refs(refs) => refs.len(),
|
||||
CloseOperationSelector::Filter(_) | CloseOperationSelector::All => 0,
|
||||
};
|
||||
let status = ControlOperationStatus {
|
||||
operation_id,
|
||||
state: ControlOperationState::Queued,
|
||||
high_water_session_ref: high_water.map(|id| self.session_ref(id)),
|
||||
requested,
|
||||
scanned: 0,
|
||||
matched: 0,
|
||||
close_signalled: 0,
|
||||
conflicted: 0,
|
||||
created_epoch_millis: now,
|
||||
updated_epoch_millis: now,
|
||||
failure: None,
|
||||
};
|
||||
let tracked = {
|
||||
let mut operations = self.control_operations.lock();
|
||||
if operations.closed || self.shutdown.is_cancelled() {
|
||||
return Err(ControlError::Closed);
|
||||
}
|
||||
if operations.active.is_some() {
|
||||
return Err(ControlError::OperationInProgress);
|
||||
}
|
||||
operations.active = Some(sequence);
|
||||
operations.retained.push_back((sequence, status.clone()));
|
||||
trim_operations(&mut operations);
|
||||
let weak = Arc::downgrade(self);
|
||||
self.tasks.track_future(async move {
|
||||
if let Some(runtime) = weak.upgrade() {
|
||||
runtime
|
||||
.run_close_operation(sequence, high_water, selector)
|
||||
.await;
|
||||
}
|
||||
})
|
||||
};
|
||||
drop(tokio::spawn(tracked));
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
/// Returns one retained operation status under the process-instance fence.
|
||||
pub(crate) fn control_operation(
|
||||
&self,
|
||||
operation_id: &str,
|
||||
) -> Result<ControlOperationStatus, ControlError> {
|
||||
let sequence = self.parse_operation_ref(operation_id)?;
|
||||
self.control_operations
|
||||
.lock()
|
||||
.retained
|
||||
.iter()
|
||||
.find_map(|(id, status)| (*id == sequence).then(|| status.clone()))
|
||||
.ok_or(ControlError::OperationNotFound)
|
||||
}
|
||||
|
||||
/// Prevents new control work from racing process-runtime shutdown.
|
||||
pub(super) fn close_control_submission_gate(&self) {
|
||||
self.control_operations.lock().closed = true;
|
||||
}
|
||||
|
||||
async fn run_close_operation(
|
||||
self: &Arc<Self>,
|
||||
sequence: u64,
|
||||
high_water: Option<u64>,
|
||||
selector: CloseOperationSelector,
|
||||
) {
|
||||
self.update_operation(sequence, |status| {
|
||||
status.state = ControlOperationState::Running
|
||||
});
|
||||
let refs = match &selector {
|
||||
CloseOperationSelector::Refs(refs) => {
|
||||
Some(refs.iter().copied().collect::<BTreeSet<_>>())
|
||||
}
|
||||
CloseOperationSelector::Filter(_) | CloseOperationSelector::All => None,
|
||||
};
|
||||
let filter = match &selector {
|
||||
CloseOperationSelector::Filter(filter) => Some(filter),
|
||||
CloseOperationSelector::Refs(_) | CloseOperationSelector::All => None,
|
||||
};
|
||||
let mut cursor = None;
|
||||
loop {
|
||||
if self.shutdown.is_cancelled() {
|
||||
self.finish_operation(sequence, ControlOperationState::Cancelled, None);
|
||||
return;
|
||||
}
|
||||
let mut direct = Vec::new();
|
||||
let mut state_filtered = Vec::new();
|
||||
let (scanned, next_cursor, reached_end) = {
|
||||
let mut state = self.state.lock();
|
||||
let mut scanned = 0usize;
|
||||
let mut next_cursor = cursor;
|
||||
let mut reached_end = true;
|
||||
let ids = state
|
||||
.session_index
|
||||
.range((
|
||||
cursor.map_or(std::ops::Bound::Unbounded, std::ops::Bound::Excluded),
|
||||
std::ops::Bound::Unbounded,
|
||||
))
|
||||
.take(CLOSE_CHUNK)
|
||||
.map(|(id, _)| *id)
|
||||
.collect::<Vec<_>>();
|
||||
for trace_session_id in ids {
|
||||
if high_water.is_some_and(|high_water| trace_session_id > high_water) {
|
||||
break;
|
||||
}
|
||||
scanned += 1;
|
||||
next_cursor = Some(trace_session_id);
|
||||
let Some(index) = state.session_index.get(&trace_session_id) else {
|
||||
continue;
|
||||
};
|
||||
let selected = refs
|
||||
.as_ref()
|
||||
.is_none_or(|refs| refs.contains(&trace_session_id));
|
||||
let Some(session) = selected
|
||||
.then(|| state.sessions.get(&index.session_hash).cloned())
|
||||
.flatten()
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if filter.is_some_and(|filter| !immutable_matches(&session, index, filter)) {
|
||||
continue;
|
||||
}
|
||||
let candidate = WorkCandidate {
|
||||
trace_session_id,
|
||||
session,
|
||||
bootstrap_hash: index.bootstrap_hash,
|
||||
};
|
||||
if filter.and_then(|filter| filter.state.as_ref()).is_some() {
|
||||
state_filtered.push(candidate);
|
||||
} else {
|
||||
mark_close_locked(&mut state, &candidate);
|
||||
direct.push(candidate);
|
||||
}
|
||||
}
|
||||
if scanned == CLOSE_CHUNK {
|
||||
reached_end = false;
|
||||
}
|
||||
(scanned, next_cursor, reached_end)
|
||||
};
|
||||
self.update_operation(sequence, |status| {
|
||||
status.scanned = status.scanned.saturating_add(scanned);
|
||||
status.matched = status.matched.saturating_add(direct.len());
|
||||
});
|
||||
for candidate in direct {
|
||||
candidate.session.close();
|
||||
self.update_operation(sequence, |status| {
|
||||
status.close_signalled = status.close_signalled.saturating_add(1)
|
||||
});
|
||||
}
|
||||
if let Some(expected_state) = filter.and_then(|filter| filter.state.as_deref()) {
|
||||
for candidate in state_filtered {
|
||||
let matches_state = candidate
|
||||
.session
|
||||
.try_status(Instant::now())
|
||||
.is_some_and(|status| status.state == expected_state);
|
||||
if !matches_state {
|
||||
continue;
|
||||
}
|
||||
let session = {
|
||||
let mut state = self.state.lock();
|
||||
let current = state
|
||||
.session_index
|
||||
.get(&candidate.trace_session_id)
|
||||
.and_then(|index| state.sessions.get(&index.session_hash))
|
||||
.filter(|current| Arc::ptr_eq(current, &candidate.session))
|
||||
.cloned();
|
||||
if current.is_some() {
|
||||
mark_close_locked(&mut state, &candidate);
|
||||
}
|
||||
current
|
||||
};
|
||||
if let Some(session) = session {
|
||||
session.close();
|
||||
self.update_operation(sequence, |status| {
|
||||
status.matched = status.matched.saturating_add(1);
|
||||
status.close_signalled = status.close_signalled.saturating_add(1);
|
||||
});
|
||||
} else {
|
||||
self.update_operation(sequence, |status| {
|
||||
status.matched = status.matched.saturating_add(1);
|
||||
status.conflicted = status.conflicted.saturating_add(1);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
cursor = next_cursor;
|
||||
if reached_end || cursor.is_none() {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
self.finish_operation(sequence, ControlOperationState::Completed, None);
|
||||
}
|
||||
|
||||
fn update_operation(&self, sequence: u64, update: impl FnOnce(&mut ControlOperationStatus)) {
|
||||
let mut operations = self.control_operations.lock();
|
||||
if let Some((_, status)) = operations
|
||||
.retained
|
||||
.iter_mut()
|
||||
.find(|(id, _)| *id == sequence)
|
||||
{
|
||||
update(status);
|
||||
status.updated_epoch_millis = crate::web::trace::store_epoch_millis();
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_operation(
|
||||
&self,
|
||||
sequence: u64,
|
||||
state: ControlOperationState,
|
||||
failure: Option<&'static str>,
|
||||
) {
|
||||
let mut operations = self.control_operations.lock();
|
||||
if let Some((_, status)) = operations
|
||||
.retained
|
||||
.iter_mut()
|
||||
.find(|(id, _)| *id == sequence)
|
||||
{
|
||||
status.state = failure.map_or(state, |_| ControlOperationState::Failed);
|
||||
status.failure = failure;
|
||||
status.updated_epoch_millis = crate::web::trace::store_epoch_millis();
|
||||
}
|
||||
if operations.active == Some(sequence) {
|
||||
operations.active = None;
|
||||
}
|
||||
}
|
||||
|
||||
fn operation_ref(&self, sequence: u64) -> String {
|
||||
format!(
|
||||
"{OPERATION_REF_VERSION}.{}.{sequence:016x}",
|
||||
self.runtime_instance()
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_operation_ref(&self, value: &str) -> Result<u64, ControlError> {
|
||||
let mut parts = value.split('.');
|
||||
if parts.next() != Some(OPERATION_REF_VERSION) {
|
||||
return Err(ControlError::InvalidOperation);
|
||||
}
|
||||
let instance = parts.next().ok_or(ControlError::InvalidOperation)?;
|
||||
if instance.len() != 32
|
||||
|| !instance
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
{
|
||||
return Err(ControlError::InvalidOperation);
|
||||
}
|
||||
let sequence = parts
|
||||
.next()
|
||||
.filter(|value| {
|
||||
value.len() == 16
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
})
|
||||
.and_then(|value| u64::from_str_radix(value, 16).ok())
|
||||
.filter(|value| *value != 0)
|
||||
.ok_or(ControlError::InvalidOperation)?;
|
||||
if parts.next().is_some() {
|
||||
return Err(ControlError::InvalidOperation);
|
||||
}
|
||||
if instance != self.runtime_instance() {
|
||||
return Err(ControlError::StaleInstance);
|
||||
}
|
||||
Ok(sequence)
|
||||
}
|
||||
}
|
||||
|
||||
fn mark_close_locked(state: &mut super::state::ManagerState, candidate: &WorkCandidate) {
|
||||
if let Some(bootstrap) = state.bootstraps.get_mut(&candidate.bootstrap_hash) {
|
||||
bootstrap.close_requested = true;
|
||||
}
|
||||
}
|
||||
|
||||
fn trim_operations(operations: &mut ControlOperationRegistry) {
|
||||
while operations.retained.len() > OPERATION_RETENTION {
|
||||
if operations
|
||||
.retained
|
||||
.front()
|
||||
.is_some_and(|(id, _)| Some(*id) == operations.active)
|
||||
{
|
||||
break;
|
||||
}
|
||||
operations.retained.pop_front();
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::state::{
|
||||
@@ -11,16 +12,50 @@ use super::state::{
|
||||
};
|
||||
use super::{BootstrapResult, ManagerError, TOKEN_BYTES, TokenHash, WebProcessRuntime};
|
||||
use crate::config::WebRuntimeProfile;
|
||||
use crate::maestro::generation::RuntimeGeneration;
|
||||
use crate::web::session::WebSession;
|
||||
|
||||
impl WebProcessRuntime {
|
||||
/// Issues a one-use bootstrap credential for an active compatible profile.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn issue_bootstrap(
|
||||
&self,
|
||||
profile: Arc<WebRuntimeProfile>,
|
||||
client_ip: IpAddr,
|
||||
) -> std::result::Result<BootstrapResult, ManagerError> {
|
||||
let generation = self.active_generation();
|
||||
self.issue_bootstrap_inner(&generation, profile, client_ip, None)
|
||||
}
|
||||
|
||||
/// Issues one bootstrap against the generation that selected the bridge profile.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn issue_bootstrap_for_generation(
|
||||
&self,
|
||||
generation: &Arc<RuntimeGeneration>,
|
||||
profile: Arc<WebRuntimeProfile>,
|
||||
client_ip: IpAddr,
|
||||
) -> std::result::Result<BootstrapResult, ManagerError> {
|
||||
self.issue_bootstrap_inner(generation, profile, client_ip, None)
|
||||
}
|
||||
|
||||
/// Issues one bridge bootstrap with bounded non-secret request metadata.
|
||||
pub(crate) fn issue_bootstrap_for_request(
|
||||
&self,
|
||||
generation: &Arc<RuntimeGeneration>,
|
||||
profile: Arc<WebRuntimeProfile>,
|
||||
client_ip: IpAddr,
|
||||
user_agent: Option<&str>,
|
||||
) -> std::result::Result<BootstrapResult, ManagerError> {
|
||||
self.issue_bootstrap_inner(generation, profile, client_ip, user_agent)
|
||||
}
|
||||
|
||||
fn issue_bootstrap_inner(
|
||||
&self,
|
||||
generation: &Arc<RuntimeGeneration>,
|
||||
profile: Arc<WebRuntimeProfile>,
|
||||
client_ip: IpAddr,
|
||||
user_agent: Option<&str>,
|
||||
) -> std::result::Result<BootstrapResult, ManagerError> {
|
||||
let config = generation.config();
|
||||
let profile = config
|
||||
.web
|
||||
@@ -34,7 +69,9 @@ impl WebProcessRuntime {
|
||||
let now = Instant::now();
|
||||
let mut state = self.state.lock();
|
||||
remove_expired_locked(&mut state, now);
|
||||
state.apply_issuance_policy(generation.id, config.web.enabled);
|
||||
if state.closed
|
||||
|| !state.issuance_enabled
|
||||
|| state
|
||||
.bootstraps_per_ip
|
||||
.get(&client_ip)
|
||||
@@ -57,11 +94,12 @@ impl WebProcessRuntime {
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
let Some((token, hash)) = new_unique_token(&generation, &state) else {
|
||||
let Some((token, hash)) = new_unique_token(generation, &state) else {
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
let trace_session_id = self.trace.next_session_id();
|
||||
let (user_agent, user_agent_id) = bounded_user_agent(user_agent);
|
||||
state.bootstraps.insert(
|
||||
hash,
|
||||
Bootstrap {
|
||||
@@ -69,7 +107,10 @@ impl WebProcessRuntime {
|
||||
issued_at: now,
|
||||
issuance_ip: client_ip,
|
||||
profile,
|
||||
timeouts: config.web.timeouts.clone(),
|
||||
trace_session_id,
|
||||
user_agent,
|
||||
user_agent_id,
|
||||
body_digest: [0; TOKEN_BYTES],
|
||||
session_token: Zeroizing::new(String::new()),
|
||||
session: None,
|
||||
@@ -110,19 +151,28 @@ impl WebProcessRuntime {
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolves non-secret bootstrap trace identity without exposing its credential.
|
||||
/// Resolves bootstrap trace identity and its issuance-frozen body timeout.
|
||||
pub(crate) fn bootstrap_trace_identity(
|
||||
&self,
|
||||
hash: TokenHash,
|
||||
host: &str,
|
||||
) -> Option<(u64, Arc<WebRuntimeProfile>)> {
|
||||
) -> Option<(u64, Arc<WebRuntimeProfile>, Duration)> {
|
||||
let now = Instant::now();
|
||||
self.state
|
||||
.lock()
|
||||
.bootstraps
|
||||
.get(&hash)
|
||||
.filter(|entry| entry.profile.host == host && now <= entry.expires_at)
|
||||
.map(|entry| (entry.trace_session_id, Arc::clone(&entry.profile)))
|
||||
.map(|entry| {
|
||||
(
|
||||
entry.trace_session_id,
|
||||
Arc::clone(&entry.profile),
|
||||
entry.session.as_ref().map_or_else(
|
||||
|| Duration::from_secs(entry.timeouts.body_secs),
|
||||
|session| Duration::from_secs(session.timeouts().body_secs),
|
||||
),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolves an authenticated session token.
|
||||
@@ -176,3 +226,30 @@ impl WebProcessRuntime {
|
||||
closed.then_some(()).ok_or(ManagerError::Authentication)
|
||||
}
|
||||
}
|
||||
|
||||
fn bounded_user_agent(value: Option<&str>) -> (Option<Arc<str>>, Option<[u8; 16]>) {
|
||||
const DISPLAY_BYTES: usize = 256;
|
||||
const HASH_CONTEXT: &[u8] = b"telemt-web-user-agent-v1\0";
|
||||
let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else {
|
||||
return (None, None);
|
||||
};
|
||||
let mut digest = Sha256::new();
|
||||
digest.update(HASH_CONTEXT);
|
||||
digest.update(value.as_bytes());
|
||||
let digest = digest.finalize();
|
||||
let mut id = [0; 16];
|
||||
id.copy_from_slice(&digest[..16]);
|
||||
let mut display = String::with_capacity(value.len().min(DISPLAY_BYTES));
|
||||
for character in value.chars() {
|
||||
let character = if character.is_control() {
|
||||
'\u{fffd}'
|
||||
} else {
|
||||
character
|
||||
};
|
||||
if display.len().saturating_add(character.len_utf8()) > DISPLAY_BYTES {
|
||||
break;
|
||||
}
|
||||
display.push(character);
|
||||
}
|
||||
(Some(Arc::from(display)), Some(id))
|
||||
}
|
||||
|
||||
@@ -1,346 +0,0 @@
|
||||
/// Telemt Carrier Selection and Failure Dampening - Copyright 2077
|
||||
/// anhand des Kundenverhaltens Rückschlüsse gegen DSGVO ziehen...?!
|
||||
use std::collections::HashMap;
|
||||
use std::net::IpAddr;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::negotiation::{CarrierClientClass, CarrierLearningContext};
|
||||
use super::ProfileKey;
|
||||
use crate::config::WebCarrier;
|
||||
|
||||
const PROFILE_WEIGHT: i16 = 4;
|
||||
const USER_AGENT_WEIGHT: i16 = 4;
|
||||
const IP_WEIGHT: i16 = 1;
|
||||
const SCORE_MIN: i8 = -8;
|
||||
const SCORE_MAX: i8 = 8;
|
||||
const PROFILE_MIN_OUTCOMES: u8 = 8;
|
||||
const PROFILE_MIN_COHORTS: usize = 4;
|
||||
const COHORT_CONTEXT: &[u8] = b"telemt-web-carrier-cohort-v1\0";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
enum EvidenceKey {
|
||||
Profile(ProfileKey),
|
||||
UserAgent(ProfileKey, CarrierClientClass, [u8; 32]),
|
||||
Ip(ProfileKey, IpAddr),
|
||||
}
|
||||
|
||||
struct Evidence {
|
||||
created_at: Instant,
|
||||
lifetime: Duration,
|
||||
scores: [i8; 4],
|
||||
outcomes: u8,
|
||||
cohorts: [Option<[u8; 32]>; PROFILE_MIN_COHORTS],
|
||||
}
|
||||
|
||||
impl Evidence {
|
||||
fn new(created_at: Instant, lifetime: Duration) -> Self {
|
||||
Self {
|
||||
created_at,
|
||||
lifetime,
|
||||
scores: [0; 4],
|
||||
outcomes: 0,
|
||||
cohorts: [None; PROFILE_MIN_COHORTS],
|
||||
}
|
||||
}
|
||||
|
||||
fn update(&mut self, carrier: WebCarrier, delta: i8, cohort: Option<[u8; 32]>) {
|
||||
let score = &mut self.scores[carrier.index()];
|
||||
*score = score.saturating_add(delta).clamp(SCORE_MIN, SCORE_MAX);
|
||||
self.outcomes = self.outcomes.saturating_add(1).min(PROFILE_MIN_OUTCOMES);
|
||||
if let Some(cohort) = cohort
|
||||
&& !self.cohorts.contains(&Some(cohort))
|
||||
&& let Some(slot) = self.cohorts.iter_mut().find(|slot| slot.is_none())
|
||||
{
|
||||
*slot = Some(cohort);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Process-local bounded fixed-window carrier evidence store.
|
||||
pub(super) struct CarrierLearning {
|
||||
entries: HashMap<EvidenceKey, Evidence>,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
impl CarrierLearning {
|
||||
/// Creates an empty store under the restart-owned capacity ceiling.
|
||||
pub(super) fn new(capacity: usize) -> Self {
|
||||
Self {
|
||||
entries: HashMap::with_capacity(capacity),
|
||||
capacity,
|
||||
}
|
||||
}
|
||||
|
||||
/// Ranks supported configured candidates using only unexpired evidence.
|
||||
pub(super) fn rank(
|
||||
&mut self,
|
||||
now: Instant,
|
||||
configured: &[WebCarrier],
|
||||
request: super::CarrierRequest,
|
||||
profile_key: ProfileKey,
|
||||
client_ip: IpAddr,
|
||||
) -> (Vec<WebCarrier>, [i16; 4]) {
|
||||
self.prune(now);
|
||||
let mut scores = [0i16; 4];
|
||||
let profile = self.entries.get(&EvidenceKey::Profile(profile_key));
|
||||
let profile_ready = profile.is_some_and(|entry| {
|
||||
entry.outcomes >= PROFILE_MIN_OUTCOMES
|
||||
&& entry.cohorts.iter().flatten().count() >= PROFILE_MIN_COHORTS
|
||||
});
|
||||
let user_agent = self.entries.get(&EvidenceKey::UserAgent(
|
||||
profile_key,
|
||||
request.class(),
|
||||
request.user_agent_hash(),
|
||||
));
|
||||
let ip = self.entries.get(&EvidenceKey::Ip(profile_key, client_ip));
|
||||
for carrier in WebCarrier::ALL {
|
||||
let index = carrier.index();
|
||||
if profile_ready {
|
||||
scores[index] += i16::from(profile.map_or(0, |entry| entry.scores[index]))
|
||||
* PROFILE_WEIGHT;
|
||||
}
|
||||
scores[index] += i16::from(user_agent.map_or(0, |entry| entry.scores[index]))
|
||||
* USER_AGENT_WEIGHT;
|
||||
scores[index] +=
|
||||
i16::from(ip.map_or(0, |entry| entry.scores[index])) * IP_WEIGHT;
|
||||
}
|
||||
let mut ranked = configured
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|carrier| request.supports(*carrier))
|
||||
.collect::<Vec<_>>();
|
||||
ranked.sort_by_key(|carrier| std::cmp::Reverse(scores[carrier.index()]));
|
||||
(ranked, scores)
|
||||
}
|
||||
|
||||
/// Records one committed success or one server-accepted supersession failure.
|
||||
pub(super) fn record(
|
||||
&mut self,
|
||||
now: Instant,
|
||||
lifetime: Duration,
|
||||
context: CarrierLearningContext,
|
||||
carrier: WebCarrier,
|
||||
success: bool,
|
||||
) {
|
||||
self.prune(now);
|
||||
let delta = if success { 1 } else { -1 };
|
||||
let cohort = cohort_hash(context);
|
||||
self.update(
|
||||
EvidenceKey::Profile(context.profile_key),
|
||||
now,
|
||||
lifetime,
|
||||
carrier,
|
||||
delta,
|
||||
Some(cohort),
|
||||
);
|
||||
self.update(
|
||||
EvidenceKey::UserAgent(
|
||||
context.profile_key,
|
||||
context.class,
|
||||
context.user_agent_hash,
|
||||
),
|
||||
now,
|
||||
lifetime,
|
||||
carrier,
|
||||
delta,
|
||||
None,
|
||||
);
|
||||
self.update(
|
||||
EvidenceKey::Ip(context.profile_key, context.client_ip),
|
||||
now,
|
||||
lifetime,
|
||||
carrier,
|
||||
delta,
|
||||
None,
|
||||
);
|
||||
}
|
||||
|
||||
/// Removes fixed-window entries after their creation-time expiry.
|
||||
pub(super) fn prune(&mut self, now: Instant) {
|
||||
self.entries.retain(|_, entry| {
|
||||
now.saturating_duration_since(entry.created_at) <= entry.lifetime
|
||||
});
|
||||
}
|
||||
|
||||
fn update(
|
||||
&mut self,
|
||||
key: EvidenceKey,
|
||||
now: Instant,
|
||||
lifetime: Duration,
|
||||
carrier: WebCarrier,
|
||||
delta: i8,
|
||||
cohort: Option<[u8; 32]>,
|
||||
) {
|
||||
if !self.entries.contains_key(&key) && self.entries.len() >= self.capacity {
|
||||
let oldest = self
|
||||
.entries
|
||||
.iter()
|
||||
.min_by_key(|(_, entry)| entry.created_at)
|
||||
.map(|(key, _)| *key);
|
||||
if let Some(oldest) = oldest {
|
||||
self.entries.remove(&oldest);
|
||||
}
|
||||
}
|
||||
self.entries
|
||||
.entry(key)
|
||||
.or_insert_with(|| Evidence::new(now, lifetime))
|
||||
.update(carrier, delta, cohort);
|
||||
}
|
||||
}
|
||||
|
||||
fn cohort_hash(context: CarrierLearningContext) -> [u8; 32] {
|
||||
let mut digest = Sha256::new();
|
||||
digest.update(COHORT_CONTEXT);
|
||||
digest.update(context.profile_key);
|
||||
digest.update([match context.class {
|
||||
CarrierClientClass::Legacy => 0,
|
||||
CarrierClientClass::Bridge => 1,
|
||||
CarrierClientClass::BrowserHint => 2,
|
||||
}]);
|
||||
digest.update(context.user_agent_hash);
|
||||
match context.client_ip {
|
||||
IpAddr::V4(address) => {
|
||||
digest.update([4]);
|
||||
digest.update(address.octets());
|
||||
}
|
||||
IpAddr::V6(address) => {
|
||||
digest.update([6]);
|
||||
digest.update(address.octets());
|
||||
}
|
||||
}
|
||||
digest.finalize().into()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::web::manager::{CarrierCapabilities, CarrierRequest};
|
||||
|
||||
fn request(hash: u8) -> CarrierRequest {
|
||||
CarrierRequest::automatic(
|
||||
CarrierClientClass::Bridge,
|
||||
CarrierCapabilities::all(),
|
||||
1,
|
||||
None,
|
||||
[hash; 32],
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn evidence_is_bounded_and_expires_without_sliding() {
|
||||
let start = Instant::now();
|
||||
let mut learning = CarrierLearning::new(3);
|
||||
let context = CarrierLearningContext {
|
||||
profile_key: [1; 32],
|
||||
client_ip: "192.0.2.1".parse().unwrap(),
|
||||
class: CarrierClientClass::Bridge,
|
||||
user_agent_hash: [2; 32],
|
||||
};
|
||||
learning.record(
|
||||
start,
|
||||
Duration::from_secs(10),
|
||||
context,
|
||||
WebCarrier::Websocket,
|
||||
true,
|
||||
);
|
||||
assert_eq!(learning.entries.len(), 3);
|
||||
learning.record(
|
||||
start + Duration::from_secs(5),
|
||||
Duration::from_secs(10),
|
||||
context,
|
||||
WebCarrier::Websocket,
|
||||
true,
|
||||
);
|
||||
learning.prune(start + Duration::from_secs(11));
|
||||
assert!(learning.entries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_agent_and_ip_evidence_rank_stably() {
|
||||
let now = Instant::now();
|
||||
let mut learning = CarrierLearning::new(16);
|
||||
let context = CarrierLearningContext {
|
||||
profile_key: [1; 32],
|
||||
client_ip: "192.0.2.1".parse().unwrap(),
|
||||
class: CarrierClientClass::Bridge,
|
||||
user_agent_hash: [2; 32],
|
||||
};
|
||||
learning.record(
|
||||
now,
|
||||
Duration::from_secs(10),
|
||||
context,
|
||||
WebCarrier::Websocket,
|
||||
true,
|
||||
);
|
||||
let (ranked, scores) = learning.rank(
|
||||
now,
|
||||
&[WebCarrier::Https, WebCarrier::Websocket],
|
||||
request(2),
|
||||
context.profile_key,
|
||||
context.client_ip,
|
||||
);
|
||||
assert_eq!(ranked, [WebCarrier::Websocket, WebCarrier::Https]);
|
||||
assert_eq!(scores[WebCarrier::Websocket.index()], 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_evidence_requires_outcome_and_cohort_thresholds() {
|
||||
let now = Instant::now();
|
||||
let profile_key = [1; 32];
|
||||
let configured = [WebCarrier::Https, WebCarrier::Websocket];
|
||||
let unrelated_ip = "198.51.100.10".parse().unwrap();
|
||||
let mut learning = CarrierLearning::new(64);
|
||||
for cohort in 1..=3u8 {
|
||||
let context = CarrierLearningContext {
|
||||
profile_key,
|
||||
client_ip: IpAddr::V4(std::net::Ipv4Addr::new(192, 0, 2, cohort)),
|
||||
class: CarrierClientClass::Bridge,
|
||||
user_agent_hash: [cohort; 32],
|
||||
};
|
||||
for _ in 0..2 {
|
||||
learning.record(
|
||||
now,
|
||||
Duration::from_secs(10),
|
||||
context,
|
||||
WebCarrier::Websocket,
|
||||
true,
|
||||
);
|
||||
}
|
||||
}
|
||||
let (ranked, _) = learning.rank(
|
||||
now,
|
||||
&configured,
|
||||
request(99),
|
||||
profile_key,
|
||||
unrelated_ip,
|
||||
);
|
||||
assert_eq!(ranked, configured);
|
||||
|
||||
let fourth = CarrierLearningContext {
|
||||
profile_key,
|
||||
client_ip: "192.0.2.4".parse().unwrap(),
|
||||
class: CarrierClientClass::Bridge,
|
||||
user_agent_hash: [4; 32],
|
||||
};
|
||||
for _ in 0..2 {
|
||||
learning.record(
|
||||
now,
|
||||
Duration::from_secs(10),
|
||||
fourth,
|
||||
WebCarrier::Websocket,
|
||||
true,
|
||||
);
|
||||
}
|
||||
let (ranked, scores) = learning.rank(
|
||||
now,
|
||||
&configured,
|
||||
request(99),
|
||||
profile_key,
|
||||
unrelated_ip,
|
||||
);
|
||||
assert_eq!(ranked, [WebCarrier::Websocket, WebCarrier::Https]);
|
||||
assert_eq!(scores[WebCarrier::Websocket.index()], 32);
|
||||
}
|
||||
}
|
||||
+339
-42
@@ -2,13 +2,31 @@ use std::net::IpAddr;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use tracing::info;
|
||||
use tokio::time::Instant as TokioInstant;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use super::state::{
|
||||
decrement_map, remember_closed_token_locked, remove_bootstrap_locked, remove_expired_locked,
|
||||
decrement_map, remember_closed_session_locked, remember_closed_token_locked,
|
||||
remove_bootstrap_locked, remove_expired_locked,
|
||||
};
|
||||
use super::{ProfileKey, TokenHash, WebProcessRuntime};
|
||||
|
||||
/// Result of draining all process-owned WEB work under one absolute deadline.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum WebShutdownOutcome {
|
||||
/// Every registered session and auxiliary task completed.
|
||||
Drained,
|
||||
/// Cancellation was asserted, but registered work remained at the deadline.
|
||||
DeadlineExceeded,
|
||||
}
|
||||
|
||||
/// Owned shutdown snapshot retained after sessions leave the live registry.
|
||||
pub(crate) struct WebShutdownDrain {
|
||||
runtime: std::sync::Arc<WebProcessRuntime>,
|
||||
sessions: Vec<std::sync::Arc<crate::web::session::WebSession>>,
|
||||
started: TokioInstant,
|
||||
}
|
||||
|
||||
impl WebProcessRuntime {
|
||||
/// Removes one closed session and retains a bounded host-bound replay marker.
|
||||
pub(crate) fn session_finished(
|
||||
@@ -17,27 +35,36 @@ impl WebProcessRuntime {
|
||||
client_ip: IpAddr,
|
||||
profile_key: ProfileKey,
|
||||
profile_host: &str,
|
||||
closed_token_lifetime: Duration,
|
||||
) {
|
||||
let mut state = self.state.lock();
|
||||
if state.sessions.remove(&hash).is_none() {
|
||||
let Some(session) = state.sessions.remove(&hash) else {
|
||||
return;
|
||||
}
|
||||
};
|
||||
decrement_map(&mut state.sessions_per_ip, &client_ip);
|
||||
decrement_map(&mut state.sessions_per_profile, &profile_key);
|
||||
remember_closed_token_locked(
|
||||
&mut state,
|
||||
hash,
|
||||
profile_host,
|
||||
Duration::from_secs(
|
||||
self.active_runtime
|
||||
.load()
|
||||
.config()
|
||||
.web
|
||||
.timeouts
|
||||
.bootstrap_lifetime_secs,
|
||||
),
|
||||
closed_token_lifetime,
|
||||
self.limits.max_sessions_global.saturating_mul(16),
|
||||
);
|
||||
let trace_session_id = session.trace_session_id();
|
||||
if state
|
||||
.session_index
|
||||
.get(&trace_session_id)
|
||||
.is_some_and(|index| index.session_hash == hash)
|
||||
{
|
||||
state.session_index.remove(&trace_session_id);
|
||||
remember_closed_session_locked(
|
||||
&mut state,
|
||||
trace_session_id,
|
||||
session.carrier_attempt(),
|
||||
closed_token_lifetime,
|
||||
self.limits.max_sessions_global,
|
||||
);
|
||||
}
|
||||
let bootstrap_hashes = state
|
||||
.bootstraps
|
||||
.iter()
|
||||
@@ -55,11 +82,21 @@ impl WebProcessRuntime {
|
||||
self.sessions_closed.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Stops issuance, closes all sessions, and joins bounded child work.
|
||||
pub(crate) async fn shutdown(&self) {
|
||||
/// Closes every WEB authority gate before any graceful wait begins.
|
||||
pub(crate) fn begin_shutdown(self: &std::sync::Arc<Self>) -> WebShutdownDrain {
|
||||
let started = TokioInstant::now();
|
||||
self.shutdown.cancel();
|
||||
self.close_control_submission_gate();
|
||||
self.close_websockets();
|
||||
self.data_budget.close();
|
||||
self.http_connections.close();
|
||||
self.http_handlers.close();
|
||||
self.lane_polls.close();
|
||||
self.lane_aux_polls.close();
|
||||
self.body_readers.close();
|
||||
self.body_bytes.close();
|
||||
self.stream_handshakes.close();
|
||||
self.websocket_connections.close();
|
||||
let sessions = {
|
||||
let mut state = self.state.lock();
|
||||
state.closed = true;
|
||||
@@ -71,6 +108,24 @@ impl WebProcessRuntime {
|
||||
for session in &sessions {
|
||||
session.close();
|
||||
}
|
||||
self.tasks.close();
|
||||
WebShutdownDrain {
|
||||
runtime: std::sync::Arc::clone(self),
|
||||
sessions,
|
||||
started,
|
||||
}
|
||||
}
|
||||
|
||||
/// Stops issuance and drains all WEB work until one absolute deadline.
|
||||
pub(crate) async fn shutdown_until(
|
||||
self: &std::sync::Arc<Self>,
|
||||
deadline: TokioInstant,
|
||||
) -> WebShutdownOutcome {
|
||||
self.begin_shutdown().wait_until(deadline).await
|
||||
}
|
||||
|
||||
/// Stops issuance and drains WEB work under the currently configured budget.
|
||||
pub(crate) async fn shutdown(self: &std::sync::Arc<Self>) -> WebShutdownOutcome {
|
||||
let timeout_secs = self
|
||||
.active_runtime
|
||||
.load()
|
||||
@@ -78,34 +133,11 @@ impl WebProcessRuntime {
|
||||
.web
|
||||
.timeouts
|
||||
.shutdown_secs;
|
||||
let waits = async {
|
||||
for session in sessions {
|
||||
session.wait().await;
|
||||
}
|
||||
};
|
||||
let _ = tokio::time::timeout(Duration::from_secs(timeout_secs), waits).await;
|
||||
self.tasks.close();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(timeout_secs), self.tasks.wait()).await;
|
||||
let sessions_live = self.state.lock().sessions.len();
|
||||
let streams_live = self.stream_admission.lock().streams_live;
|
||||
let budget = self.data_budget.snapshot();
|
||||
info!(
|
||||
target: "telemt::web",
|
||||
sessions_created = self.sessions_created.load(Ordering::Relaxed),
|
||||
sessions_closed = self.sessions_closed.load(Ordering::Relaxed),
|
||||
sessions_live,
|
||||
streams_opened = self.streams_opened.load(Ordering::Relaxed),
|
||||
streams_rejected = self.streams_rejected.load(Ordering::Relaxed),
|
||||
streams_live,
|
||||
pending_bytes = budget.queue_bytes,
|
||||
pending_items = budget.queue_items,
|
||||
websocket_bytes = budget.websocket_bytes,
|
||||
data_high_water_bytes = budget.high_water_bytes,
|
||||
bytes_up = self.bytes_up.load(Ordering::Relaxed),
|
||||
bytes_down = self.bytes_down.load(Ordering::Relaxed),
|
||||
limit_hits = self.limit_hits.load(Ordering::Relaxed),
|
||||
"WEB runtime stopped"
|
||||
);
|
||||
let now = TokioInstant::now();
|
||||
let deadline = now
|
||||
.checked_add(Duration::from_secs(timeout_secs))
|
||||
.unwrap_or(now);
|
||||
self.shutdown_until(deadline).await
|
||||
}
|
||||
|
||||
/// Expires credentials and closes idle sessions without holding locks across callbacks.
|
||||
@@ -126,6 +158,7 @@ impl WebProcessRuntime {
|
||||
drop(learning);
|
||||
let (sessions, expired_chains) = {
|
||||
let mut state = self.state.lock();
|
||||
state.apply_issuance_policy(generation.id, config.enabled);
|
||||
let expired = state
|
||||
.bootstraps
|
||||
.iter()
|
||||
@@ -162,3 +195,267 @@ impl WebProcessRuntime {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl WebShutdownDrain {
|
||||
/// Waits for frozen session ownership and process auxiliary tasks concurrently.
|
||||
pub(crate) async fn wait_until(self, deadline: TokioInstant) -> WebShutdownOutcome {
|
||||
let sessions = &self.sessions;
|
||||
let session_waits = async {
|
||||
for session in sessions {
|
||||
session.wait().await;
|
||||
}
|
||||
};
|
||||
let outcome = wait_for_drain(deadline, session_waits, self.runtime.tasks.wait()).await;
|
||||
self.log_outcome(outcome, deadline);
|
||||
outcome
|
||||
}
|
||||
|
||||
fn log_outcome(&self, outcome: WebShutdownOutcome, deadline: TokioInstant) {
|
||||
let sessions_live = self.runtime.state.lock().sessions.len();
|
||||
let streams_live = self.runtime.stream_admission.lock().streams_live;
|
||||
let session_tasks_live = self.sessions.iter().fold(0usize, |total, session| {
|
||||
total.saturating_add(session.tasks_live())
|
||||
});
|
||||
let sessions_pending = self
|
||||
.sessions
|
||||
.iter()
|
||||
.filter(|session| session.tasks_live() != 0)
|
||||
.count();
|
||||
let auxiliary_tasks_live = self.runtime.tasks.len();
|
||||
let budget = self.runtime.data_budget.snapshot();
|
||||
let budget_ms = deadline
|
||||
.saturating_duration_since(self.started)
|
||||
.as_millis()
|
||||
.min(u128::from(u64::MAX)) as u64;
|
||||
let elapsed_ms = TokioInstant::now()
|
||||
.saturating_duration_since(self.started)
|
||||
.as_millis()
|
||||
.min(u128::from(u64::MAX)) as u64;
|
||||
match outcome {
|
||||
WebShutdownOutcome::Drained => info!(
|
||||
target: "telemt::web",
|
||||
shutdown_drained = true,
|
||||
shutdown_budget_ms = budget_ms,
|
||||
shutdown_elapsed_ms = elapsed_ms,
|
||||
sessions_created = self.runtime.sessions_created.load(Ordering::Relaxed),
|
||||
sessions_closed = self.runtime.sessions_closed.load(Ordering::Relaxed),
|
||||
sessions_live,
|
||||
sessions_pending,
|
||||
session_tasks_live,
|
||||
auxiliary_tasks_live,
|
||||
streams_opened = self.runtime.streams_opened.load(Ordering::Relaxed),
|
||||
streams_rejected = self.runtime.streams_rejected.load(Ordering::Relaxed),
|
||||
streams_live,
|
||||
pending_bytes = budget.queue_bytes,
|
||||
pending_items = budget.queue_items,
|
||||
websocket_bytes = budget.websocket_bytes,
|
||||
data_high_water_bytes = budget.high_water_bytes,
|
||||
bytes_up = self.runtime.bytes_up.load(Ordering::Relaxed),
|
||||
bytes_down = self.runtime.bytes_down.load(Ordering::Relaxed),
|
||||
limit_hits = self.runtime.limit_hits.load(Ordering::Relaxed),
|
||||
"WEB runtime stopped"
|
||||
),
|
||||
WebShutdownOutcome::DeadlineExceeded => warn!(
|
||||
target: "telemt::web",
|
||||
shutdown_drained = false,
|
||||
shutdown_budget_ms = budget_ms,
|
||||
shutdown_elapsed_ms = elapsed_ms,
|
||||
sessions_created = self.runtime.sessions_created.load(Ordering::Relaxed),
|
||||
sessions_closed = self.runtime.sessions_closed.load(Ordering::Relaxed),
|
||||
sessions_live,
|
||||
sessions_pending,
|
||||
session_tasks_live,
|
||||
auxiliary_tasks_live,
|
||||
streams_opened = self.runtime.streams_opened.load(Ordering::Relaxed),
|
||||
streams_rejected = self.runtime.streams_rejected.load(Ordering::Relaxed),
|
||||
streams_live,
|
||||
pending_bytes = budget.queue_bytes,
|
||||
pending_items = budget.queue_items,
|
||||
websocket_bytes = budget.websocket_bytes,
|
||||
data_high_water_bytes = budget.high_water_bytes,
|
||||
bytes_up = self.runtime.bytes_up.load(Ordering::Relaxed),
|
||||
bytes_down = self.runtime.bytes_down.load(Ordering::Relaxed),
|
||||
limit_hits = self.runtime.limit_hits.load(Ordering::Relaxed),
|
||||
"WEB runtime shutdown deadline exceeded"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_drain<S, T>(deadline: TokioInstant, sessions: S, tasks: T) -> WebShutdownOutcome
|
||||
where
|
||||
S: std::future::Future<Output = ()>,
|
||||
T: std::future::Future<Output = ()>,
|
||||
{
|
||||
let waits = async {
|
||||
tokio::join!(sessions, tasks);
|
||||
};
|
||||
if tokio::time::timeout_at(deadline, waits).await.is_ok() {
|
||||
WebShutdownOutcome::Drained
|
||||
} else {
|
||||
WebShutdownOutcome::DeadlineExceeded
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
use super::*;
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::maestro::generation::test_runtime_generation;
|
||||
use crate::web::manager::{CloseOperationSelector, ControlError};
|
||||
|
||||
struct DropProbe {
|
||||
polls: Arc<AtomicUsize>,
|
||||
drops: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Future for DropProbe {
|
||||
type Output = ();
|
||||
|
||||
fn poll(self: Pin<&mut Self>, _context: &mut Context<'_>) -> Poll<Self::Output> {
|
||||
self.polls.fetch_add(1, Ordering::AcqRel);
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DropProbe {
|
||||
fn drop(&mut self) {
|
||||
self.drops.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
fn runtime() -> (
|
||||
Arc<WebProcessRuntime>,
|
||||
Arc<crate::maestro::generation::RuntimeGeneration>,
|
||||
) {
|
||||
let generation = test_runtime_generation(1, ProxyConfig::default());
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
(runtime, generation)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn drain_uses_one_absolute_deadline_for_both_wait_groups() {
|
||||
let started = TokioInstant::now();
|
||||
let outcome = wait_for_drain(
|
||||
started + Duration::from_secs(5),
|
||||
tokio::time::sleep(Duration::from_secs(4)),
|
||||
tokio::time::sleep(Duration::from_secs(9)),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(outcome, WebShutdownOutcome::DeadlineExceeded);
|
||||
assert_eq!(TokioInstant::now() - started, Duration::from_secs(5));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn drain_returns_when_both_wait_groups_finish() {
|
||||
let started = TokioInstant::now();
|
||||
let outcome = wait_for_drain(
|
||||
started + Duration::from_secs(5),
|
||||
tokio::time::sleep(Duration::from_secs(3)),
|
||||
tokio::time::sleep(Duration::from_secs(2)),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(outcome, WebShutdownOutcome::Drained);
|
||||
assert_eq!(TokioInstant::now() - started, Duration::from_secs(3));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn post_shutdown_auxiliary_is_dropped_without_polling() {
|
||||
let (runtime, generation) = runtime();
|
||||
let polls = Arc::new(AtomicUsize::new(0));
|
||||
let drops = Arc::new(AtomicUsize::new(0));
|
||||
let drain = runtime.begin_shutdown();
|
||||
|
||||
runtime.spawn_auxiliary(DropProbe {
|
||||
polls: Arc::clone(&polls),
|
||||
drops: Arc::clone(&drops),
|
||||
});
|
||||
|
||||
assert_eq!(polls.load(Ordering::Acquire), 0);
|
||||
assert_eq!(drops.load(Ordering::Acquire), 1);
|
||||
assert_eq!(
|
||||
drain
|
||||
.wait_until(TokioInstant::now() + Duration::from_secs(1))
|
||||
.await,
|
||||
WebShutdownOutcome::Drained
|
||||
);
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn initial_trace_policy_is_attributed_to_active_generation() {
|
||||
let (runtime, generation) = runtime();
|
||||
|
||||
assert_eq!(runtime.trace().status().policy_generation, generation.id);
|
||||
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_closes_the_control_submission_gate() {
|
||||
let (runtime, generation) = runtime();
|
||||
let drain = runtime.begin_shutdown();
|
||||
|
||||
assert!(matches!(
|
||||
runtime.start_close_operation(
|
||||
runtime.runtime_instance(),
|
||||
CloseOperationSelector::Refs(vec![1]),
|
||||
),
|
||||
Err(ControlError::Closed)
|
||||
));
|
||||
assert!(matches!(
|
||||
runtime.reset_carrier_learning(),
|
||||
Err(crate::web::manager::ManagerError::Closed)
|
||||
));
|
||||
assert!(matches!(runtime.clear_debug(), Err(ControlError::Closed)));
|
||||
assert_eq!(
|
||||
drain
|
||||
.wait_until(TokioInstant::now() + Duration::from_secs(1))
|
||||
.await,
|
||||
WebShutdownOutcome::Drained
|
||||
);
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn expired_deadline_still_closes_every_runtime_gate() {
|
||||
let (runtime, generation) = runtime();
|
||||
let existing = runtime.try_http_connection().unwrap();
|
||||
let release = Arc::new(Notify::new());
|
||||
let release_task = Arc::clone(&release);
|
||||
runtime.tasks.spawn(async move {
|
||||
release_task.notified().await;
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
let drain = runtime.begin_shutdown();
|
||||
|
||||
assert!(runtime.try_http_connection().is_none());
|
||||
assert!(runtime.try_http_handler().is_none());
|
||||
assert!(runtime.try_lane_poll(false).is_none());
|
||||
assert_eq!(
|
||||
drain.wait_until(TokioInstant::now()).await,
|
||||
WebShutdownOutcome::DeadlineExceeded
|
||||
);
|
||||
|
||||
drop(existing);
|
||||
release.notify_waiters();
|
||||
runtime.tasks.wait().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,6 +77,11 @@ impl CarrierCapabilities {
|
||||
Self(0b1111)
|
||||
}
|
||||
|
||||
/// Returns the current server-authoritative native iOS capability ceiling.
|
||||
pub(crate) const fn ios() -> Self {
|
||||
Self(1 << WebCarrier::Https.index())
|
||||
}
|
||||
|
||||
/// Builds a set from a validated bit representation.
|
||||
pub(crate) const fn from_bits(bits: u8) -> Option<Self> {
|
||||
if bits != 0 && bits & !0b1111 == 0 {
|
||||
@@ -90,6 +95,11 @@ impl CarrierCapabilities {
|
||||
pub(crate) const fn contains(self, carrier: WebCarrier) -> bool {
|
||||
self.0 & (1 << carrier.index()) != 0
|
||||
}
|
||||
|
||||
/// Intersects declared capabilities with an authoritative server ceiling.
|
||||
pub(crate) const fn intersection(self, ceiling: Self) -> Option<Self> {
|
||||
Self::from_bits(self.0 & ceiling.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Immutable metadata attached to one session-creation attempt.
|
||||
|
||||
@@ -10,8 +10,8 @@ use zeroize::Zeroizing;
|
||||
use super::negotiation::carrier_attempt_deadline_index;
|
||||
use super::session_admission::admit_initial;
|
||||
use super::state::{
|
||||
CarrierChainPhase, decrement_map, matching_profile, new_unique_token, profile_key,
|
||||
remember_closed_token_locked, remove_expired_locked,
|
||||
CarrierChainPhase, LiveSessionIndex, decrement_map, matching_profile, new_unique_token,
|
||||
profile_key, remember_closed_token_locked, remove_expired_locked,
|
||||
};
|
||||
use super::{
|
||||
CarrierLearningContext, CarrierRequest, CreateResult, ManagerError, TokenHash,
|
||||
@@ -56,6 +56,10 @@ impl WebProcessRuntime {
|
||||
let now = Instant::now();
|
||||
let mut state = self.state.lock();
|
||||
remove_expired_locked(&mut state, now);
|
||||
state.apply_issuance_policy(generation.id, config.web.enabled);
|
||||
if state.closed || !state.issuance_enabled {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let Some(entry) = state.bootstraps.get(&bootstrap_hash) else {
|
||||
return Err(ManagerError::Authentication);
|
||||
};
|
||||
@@ -201,9 +205,7 @@ impl WebProcessRuntime {
|
||||
}
|
||||
let trace_session_id = entry.trace_session_id;
|
||||
let issued_profile = Arc::clone(&entry.profile);
|
||||
if state.closed || !config.web.enabled {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let issued_timeouts = entry.timeouts.clone();
|
||||
let profile = config
|
||||
.web
|
||||
.runtime
|
||||
@@ -303,33 +305,39 @@ impl WebProcessRuntime {
|
||||
learning_context,
|
||||
carrier_request.is_automatic(),
|
||||
self.limits.clone(),
|
||||
config.web.timeouts.clone(),
|
||||
issued_timeouts.clone(),
|
||||
);
|
||||
state.sessions.insert(session_hash, Arc::clone(&session));
|
||||
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
|
||||
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
|
||||
let entry = state
|
||||
.bootstraps
|
||||
.get_mut(&bootstrap_hash)
|
||||
.ok_or(ManagerError::Authentication)?;
|
||||
entry.used = true;
|
||||
entry.body_digest = body_digest;
|
||||
entry.session_token = Zeroizing::new(session_token.clone());
|
||||
entry.session = Some(Arc::clone(&session));
|
||||
entry.carrier_request = Some(carrier_request);
|
||||
entry.carrier_candidates = candidates.into();
|
||||
entry.carrier_scores = scores;
|
||||
entry.carrier_attempt = 1;
|
||||
entry.carrier_phase = CarrierChainPhase::Provisional;
|
||||
entry.carrier_started_at = carrier_request.is_automatic().then_some(now);
|
||||
entry.carrier_deadline_at = carrier_deadline_at;
|
||||
entry.carrier_failures = [None; 3];
|
||||
entry.carrier_learning_epoch = learning_epoch.unwrap_or(0);
|
||||
entry.expires_at = now + Duration::from_secs(config.web.timeouts.bootstrap_lifetime_secs);
|
||||
entry.session_client_ip = Some(client_ip);
|
||||
entry.session_ip_learning_eligible = ip_learning_eligible;
|
||||
let issuance_ip = entry.issuance_ip;
|
||||
let candidate_count = u8::try_from(entry.carrier_candidates.len()).unwrap_or(4);
|
||||
let (issuance_ip, candidate_count, user_agent, user_agent_id) = {
|
||||
let entry = state
|
||||
.bootstraps
|
||||
.get_mut(&bootstrap_hash)
|
||||
.ok_or(ManagerError::Authentication)?;
|
||||
entry.used = true;
|
||||
entry.body_digest = body_digest;
|
||||
entry.session_token = Zeroizing::new(session_token.clone());
|
||||
entry.session = Some(Arc::clone(&session));
|
||||
entry.carrier_request = Some(carrier_request);
|
||||
entry.carrier_candidates = candidates.into();
|
||||
entry.carrier_scores = scores;
|
||||
entry.carrier_attempt = 1;
|
||||
entry.carrier_phase = CarrierChainPhase::Provisional;
|
||||
entry.carrier_started_at = carrier_request.is_automatic().then_some(now);
|
||||
entry.carrier_deadline_at = carrier_deadline_at;
|
||||
entry.carrier_failures = [None; 3];
|
||||
entry.carrier_learning_epoch = learning_epoch.unwrap_or(0);
|
||||
entry.expires_at = now + Duration::from_secs(issued_timeouts.bootstrap_lifetime_secs);
|
||||
entry.session_client_ip = Some(client_ip);
|
||||
entry.session_ip_learning_eligible = ip_learning_eligible;
|
||||
(
|
||||
entry.issuance_ip,
|
||||
u8::try_from(entry.carrier_candidates.len()).unwrap_or(4),
|
||||
entry.user_agent.clone(),
|
||||
entry.user_agent_id,
|
||||
)
|
||||
};
|
||||
decrement_map(&mut state.bootstraps_per_ip, &issuance_ip);
|
||||
self.sessions_created.fetch_add(1, Ordering::Relaxed);
|
||||
let identity = session.trace_identity();
|
||||
@@ -345,6 +353,16 @@ impl WebProcessRuntime {
|
||||
.is_automatic()
|
||||
.then_some(CarrierChainPhase::Provisional.as_str()),
|
||||
};
|
||||
state.session_index.insert(
|
||||
trace_session_id,
|
||||
LiveSessionIndex {
|
||||
session_hash,
|
||||
bootstrap_hash,
|
||||
attempt: 1,
|
||||
user_agent,
|
||||
user_agent_id,
|
||||
},
|
||||
);
|
||||
drop(state);
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
@@ -376,178 +394,7 @@ impl WebProcessRuntime {
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn replace_session(
|
||||
self: &Arc<Self>,
|
||||
bootstrap_hash: TokenHash,
|
||||
client_ip: IpAddr,
|
||||
replacement: Replacement,
|
||||
) -> std::result::Result<CreateResult, ManagerError> {
|
||||
if !replacement.old_session.begin_carrier_supersede() {
|
||||
let committed = replacement.old_session.is_carrier_committed();
|
||||
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
|
||||
return Err(if committed {
|
||||
ManagerError::Committed
|
||||
} else {
|
||||
ManagerError::Closed
|
||||
});
|
||||
}
|
||||
let generation = self.active_generation();
|
||||
let config = generation.config();
|
||||
let now = Instant::now();
|
||||
let mut state = self.state.lock();
|
||||
remove_expired_locked(&mut state, now);
|
||||
let valid = state.bootstraps.get(&bootstrap_hash).is_some_and(|entry| {
|
||||
entry.carrier_transitioning
|
||||
&& entry.carrier_phase == CarrierChainPhase::Provisional
|
||||
&& !entry.close_requested
|
||||
&& entry.carrier_attempt.saturating_add(1) == replacement.attempt
|
||||
&& now < replacement.carrier_deadline_at
|
||||
&& entry
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| Arc::ptr_eq(session, &replacement.old_session))
|
||||
}) && state
|
||||
.sessions
|
||||
.get(&replacement.old_session.token_hash())
|
||||
.is_some_and(|session| Arc::ptr_eq(session, &replacement.old_session));
|
||||
if !valid
|
||||
|| state.closed
|
||||
|| !config.web.enabled
|
||||
|| !generation
|
||||
.proxy_shared
|
||||
.is_user_enabled(&replacement.profile.user)
|
||||
{
|
||||
drop(state);
|
||||
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
drop(state);
|
||||
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
let learning_context = (replacement.profile.carrier_learning
|
||||
&& replacement.learning_epoch != 0)
|
||||
.then_some(CarrierLearningContext {
|
||||
profile_key: replacement.profile_key,
|
||||
client_ip,
|
||||
class: replacement.request.class(),
|
||||
user_agent_hash: replacement.request.user_agent_hash(),
|
||||
epoch: replacement.learning_epoch,
|
||||
ip_learning_eligible: replacement.ip_learning_eligible,
|
||||
});
|
||||
let session = WebSession::new(
|
||||
Arc::downgrade(self),
|
||||
session_hash,
|
||||
client_ip,
|
||||
replacement.trace_session_id,
|
||||
Arc::clone(&replacement.profile),
|
||||
replacement.profile_key,
|
||||
replacement.carrier,
|
||||
replacement.attempt,
|
||||
bootstrap_hash,
|
||||
Some(replacement.carrier_deadline_at),
|
||||
replacement.request.class(),
|
||||
learning_context,
|
||||
true,
|
||||
self.limits.clone(),
|
||||
replacement.old_session.timeouts().clone(),
|
||||
);
|
||||
let Some(supersede) = replacement.old_session.prepare_carrier_supersede() else {
|
||||
drop(state);
|
||||
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
|
||||
session.close();
|
||||
return Err(ManagerError::Closed);
|
||||
};
|
||||
let old_hash = replacement.old_session.token_hash();
|
||||
state.sessions.remove(&old_hash);
|
||||
remember_closed_token_locked(
|
||||
&mut state,
|
||||
old_hash,
|
||||
&replacement.profile.host,
|
||||
Duration::from_secs(config.web.timeouts.bootstrap_lifetime_secs),
|
||||
self.limits.max_sessions_global.saturating_mul(16),
|
||||
);
|
||||
state.sessions.insert(session_hash, Arc::clone(&session));
|
||||
let entry = state
|
||||
.bootstraps
|
||||
.get_mut(&bootstrap_hash)
|
||||
.ok_or(ManagerError::Authentication)?;
|
||||
entry.session_token = Zeroizing::new(session_token.clone());
|
||||
entry.session = Some(Arc::clone(&session));
|
||||
entry.carrier_request = Some(replacement.request);
|
||||
entry.carrier_attempt = replacement.attempt;
|
||||
entry.carrier_transitioning = false;
|
||||
entry.carrier_phase = CarrierChainPhase::Provisional;
|
||||
if let Some(slot) = entry
|
||||
.carrier_failures
|
||||
.get_mut(usize::from(replacement.attempt.saturating_sub(2)))
|
||||
{
|
||||
*slot = Some(replacement.old_session.carrier());
|
||||
}
|
||||
self.sessions_created.fetch_add(1, Ordering::Relaxed);
|
||||
self.sessions_closed.fetch_add(1, Ordering::Relaxed);
|
||||
let result = CreateResult {
|
||||
token: session_token,
|
||||
carrier: replacement.carrier,
|
||||
attempt: Some(replacement.attempt),
|
||||
candidate_count: Some(u8::try_from(entry.carrier_candidates.len()).unwrap_or(4)),
|
||||
deadline_secs: Some(entry.profile.carrier_negotiation_deadlines_secs[3]),
|
||||
carrier_state: Some(CarrierChainPhase::Provisional.as_str()),
|
||||
};
|
||||
let identity = session.trace_identity();
|
||||
let old_identity = replacement.old_session.trace_identity();
|
||||
drop(state);
|
||||
supersede.finish();
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
old_identity.clone(),
|
||||
TraceLifecycleEvent::CarrierFailed,
|
||||
replacement.request.class().as_str(),
|
||||
replacement.old_session.carrier(),
|
||||
replacement.attempt - 1,
|
||||
replacement.scores,
|
||||
replacement
|
||||
.request
|
||||
.failure()
|
||||
.map(|failure| failure.as_str()),
|
||||
);
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
old_identity,
|
||||
TraceLifecycleEvent::CarrierSuperseded,
|
||||
replacement.request.class().as_str(),
|
||||
replacement.old_session.carrier(),
|
||||
replacement.attempt - 1,
|
||||
replacement.scores,
|
||||
replacement
|
||||
.request
|
||||
.failure()
|
||||
.map(|failure| failure.as_str()),
|
||||
);
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
identity.clone(),
|
||||
TraceLifecycleEvent::CarrierSelected,
|
||||
replacement.request.class().as_str(),
|
||||
replacement.carrier,
|
||||
replacement.attempt,
|
||||
replacement.scores,
|
||||
None,
|
||||
);
|
||||
self.trace.record_lifecycle(
|
||||
None,
|
||||
Some(client_ip),
|
||||
identity,
|
||||
TraceLifecycleEvent::SessionCreated,
|
||||
None,
|
||||
replacement
|
||||
.request
|
||||
.failure()
|
||||
.map(|failure| failure.as_str()),
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
// Atomic pre-commit carrier replacement and frozen-policy transfer.
|
||||
mod replacement;
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
use super::*;
|
||||
|
||||
impl WebProcessRuntime {
|
||||
pub(super) fn replace_session(
|
||||
self: &Arc<Self>,
|
||||
bootstrap_hash: TokenHash,
|
||||
client_ip: IpAddr,
|
||||
replacement: Replacement,
|
||||
) -> std::result::Result<CreateResult, ManagerError> {
|
||||
if !replacement.old_session.begin_carrier_supersede() {
|
||||
let committed = replacement.old_session.is_carrier_committed();
|
||||
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
|
||||
return Err(if committed {
|
||||
ManagerError::Committed
|
||||
} else {
|
||||
ManagerError::Closed
|
||||
});
|
||||
}
|
||||
let generation = self.active_generation();
|
||||
let config = generation.config();
|
||||
let now = Instant::now();
|
||||
let mut state = self.state.lock();
|
||||
remove_expired_locked(&mut state, now);
|
||||
state.apply_issuance_policy(generation.id, config.web.enabled);
|
||||
let valid = state.bootstraps.get(&bootstrap_hash).is_some_and(|entry| {
|
||||
entry.carrier_transitioning
|
||||
&& entry.carrier_phase == CarrierChainPhase::Provisional
|
||||
&& !entry.close_requested
|
||||
&& entry.carrier_attempt.saturating_add(1) == replacement.attempt
|
||||
&& now < replacement.carrier_deadline_at
|
||||
&& entry
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(|session| Arc::ptr_eq(session, &replacement.old_session))
|
||||
}) && state
|
||||
.sessions
|
||||
.get(&replacement.old_session.token_hash())
|
||||
.is_some_and(|session| Arc::ptr_eq(session, &replacement.old_session));
|
||||
if !valid
|
||||
|| state.closed
|
||||
|| !state.issuance_enabled
|
||||
|| !generation
|
||||
.proxy_shared
|
||||
.is_user_enabled(&replacement.profile.user)
|
||||
{
|
||||
drop(state);
|
||||
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let Some((session_token, session_hash)) = new_unique_token(&generation, &state) else {
|
||||
self.limit_hits.fetch_add(1, Ordering::Relaxed);
|
||||
drop(state);
|
||||
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
|
||||
return Err(ManagerError::Limit);
|
||||
};
|
||||
let learning_context = (replacement.profile.carrier_learning
|
||||
&& replacement.learning_epoch != 0)
|
||||
.then_some(CarrierLearningContext {
|
||||
profile_key: replacement.profile_key,
|
||||
client_ip,
|
||||
class: replacement.request.class(),
|
||||
user_agent_hash: replacement.request.user_agent_hash(),
|
||||
epoch: replacement.learning_epoch,
|
||||
ip_learning_eligible: replacement.ip_learning_eligible,
|
||||
});
|
||||
let session = WebSession::new(
|
||||
Arc::downgrade(self),
|
||||
session_hash,
|
||||
client_ip,
|
||||
replacement.trace_session_id,
|
||||
Arc::clone(&replacement.profile),
|
||||
replacement.profile_key,
|
||||
replacement.carrier,
|
||||
replacement.attempt,
|
||||
bootstrap_hash,
|
||||
Some(replacement.carrier_deadline_at),
|
||||
replacement.request.class(),
|
||||
learning_context,
|
||||
true,
|
||||
self.limits.clone(),
|
||||
replacement.old_session.timeouts().clone(),
|
||||
);
|
||||
let Some(supersede) = replacement.old_session.prepare_carrier_supersede() else {
|
||||
drop(state);
|
||||
self.cancel_replacement(bootstrap_hash, &replacement.old_session);
|
||||
session.close();
|
||||
return Err(ManagerError::Closed);
|
||||
};
|
||||
let old_hash = replacement.old_session.token_hash();
|
||||
state.sessions.remove(&old_hash);
|
||||
remember_closed_token_locked(
|
||||
&mut state,
|
||||
old_hash,
|
||||
&replacement.profile.host,
|
||||
Duration::from_secs(replacement.old_session.timeouts().bootstrap_lifetime_secs),
|
||||
self.limits.max_sessions_global.saturating_mul(16),
|
||||
);
|
||||
state.sessions.insert(session_hash, Arc::clone(&session));
|
||||
let entry = state
|
||||
.bootstraps
|
||||
.get_mut(&bootstrap_hash)
|
||||
.ok_or(ManagerError::Authentication)?;
|
||||
entry.session_token = Zeroizing::new(session_token.clone());
|
||||
entry.session = Some(Arc::clone(&session));
|
||||
entry.carrier_request = Some(replacement.request);
|
||||
entry.carrier_attempt = replacement.attempt;
|
||||
entry.carrier_transitioning = false;
|
||||
entry.carrier_phase = CarrierChainPhase::Provisional;
|
||||
if let Some(slot) = entry
|
||||
.carrier_failures
|
||||
.get_mut(usize::from(replacement.attempt.saturating_sub(2)))
|
||||
{
|
||||
*slot = Some(replacement.old_session.carrier());
|
||||
}
|
||||
self.sessions_created.fetch_add(1, Ordering::Relaxed);
|
||||
self.sessions_closed.fetch_add(1, Ordering::Relaxed);
|
||||
let result = CreateResult {
|
||||
token: session_token,
|
||||
carrier: replacement.carrier,
|
||||
attempt: Some(replacement.attempt),
|
||||
candidate_count: Some(u8::try_from(entry.carrier_candidates.len()).unwrap_or(4)),
|
||||
deadline_secs: Some(entry.profile.carrier_negotiation_deadlines_secs[3]),
|
||||
carrier_state: Some(CarrierChainPhase::Provisional.as_str()),
|
||||
};
|
||||
if let Some(index) = state.session_index.get_mut(&replacement.trace_session_id)
|
||||
&& index.session_hash == old_hash
|
||||
{
|
||||
index.session_hash = session_hash;
|
||||
index.bootstrap_hash = bootstrap_hash;
|
||||
index.attempt = replacement.attempt;
|
||||
}
|
||||
let identity = session.trace_identity();
|
||||
let old_identity = replacement.old_session.trace_identity();
|
||||
drop(state);
|
||||
supersede.finish();
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
old_identity.clone(),
|
||||
TraceLifecycleEvent::CarrierFailed,
|
||||
replacement.request.class().as_str(),
|
||||
replacement.old_session.carrier(),
|
||||
replacement.attempt - 1,
|
||||
replacement.scores,
|
||||
replacement
|
||||
.request
|
||||
.failure()
|
||||
.map(|failure| failure.as_str()),
|
||||
);
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
old_identity,
|
||||
TraceLifecycleEvent::CarrierSuperseded,
|
||||
replacement.request.class().as_str(),
|
||||
replacement.old_session.carrier(),
|
||||
replacement.attempt - 1,
|
||||
replacement.scores,
|
||||
replacement
|
||||
.request
|
||||
.failure()
|
||||
.map(|failure| failure.as_str()),
|
||||
);
|
||||
self.trace.record_carrier_lifecycle(
|
||||
client_ip,
|
||||
identity.clone(),
|
||||
TraceLifecycleEvent::CarrierSelected,
|
||||
replacement.request.class().as_str(),
|
||||
replacement.carrier,
|
||||
replacement.attempt,
|
||||
replacement.scores,
|
||||
None,
|
||||
);
|
||||
self.trace.record_lifecycle(
|
||||
None,
|
||||
Some(client_ip),
|
||||
identity,
|
||||
TraceLifecycleEvent::SessionCreated,
|
||||
None,
|
||||
replacement
|
||||
.request
|
||||
.failure()
|
||||
.map(|failure| failure.as_str()),
|
||||
);
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
+113
-3
@@ -1,4 +1,4 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -8,7 +8,7 @@ use sha2::{Digest, Sha256};
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
use super::{CarrierRequest, ProfileKey, TOKEN_BYTES, TokenHash};
|
||||
use crate::config::{WebCarrier, WebRuntimeConfig, WebRuntimeProfile};
|
||||
use crate::config::{WebCarrier, WebRuntimeConfig, WebRuntimeProfile, WebTimeoutsConfig};
|
||||
use crate::maestro::generation::RuntimeGeneration;
|
||||
use crate::web::session::WebSession;
|
||||
|
||||
@@ -41,8 +41,14 @@ pub(super) struct Bootstrap {
|
||||
pub(super) issuance_ip: IpAddr,
|
||||
/// Immutable profile selected during capability validation.
|
||||
pub(super) profile: Arc<WebRuntimeProfile>,
|
||||
/// Request and session deadlines frozen with the generated bridge.
|
||||
pub(super) timeouts: WebTimeoutsConfig,
|
||||
/// Process-unique non-secret identifier shared by bootstrap and session traces.
|
||||
pub(super) trace_session_id: u64,
|
||||
/// Bounded display form of the issuing User-Agent.
|
||||
pub(super) user_agent: Option<Arc<str>>,
|
||||
/// Opaque non-secret identifier used for exact User-Agent filtering.
|
||||
pub(super) user_agent_id: Option<[u8; 16]>,
|
||||
/// Digest of the accepted HELLO body for idempotent retry matching.
|
||||
pub(super) body_digest: TokenHash,
|
||||
/// Zeroizing copy returned only for an exact session-creation retry.
|
||||
@@ -87,6 +93,28 @@ pub(super) struct ClosedToken {
|
||||
pub(super) host: String,
|
||||
}
|
||||
|
||||
/// Current logical-session owner stored without exposing bearer credentials.
|
||||
pub(super) struct LiveSessionIndex {
|
||||
/// Current bearer hash used only for internal pointer revalidation.
|
||||
pub(super) session_hash: TokenHash,
|
||||
/// Bootstrap chain that owns carrier replacement and close intent.
|
||||
pub(super) bootstrap_hash: TokenHash,
|
||||
/// Current carrier incarnation in the logical trace session.
|
||||
pub(super) attempt: u8,
|
||||
/// Bounded display form of the issuing User-Agent.
|
||||
pub(super) user_agent: Option<Arc<str>>,
|
||||
/// Opaque non-secret identifier used for exact User-Agent filtering.
|
||||
pub(super) user_agent_id: Option<[u8; 16]>,
|
||||
}
|
||||
|
||||
/// Bounded logical-session tombstone used for exact detail semantics.
|
||||
pub(super) struct ClosedSession {
|
||||
/// Tombstone expiry deadline.
|
||||
pub(super) expires_at: Instant,
|
||||
/// Last carrier incarnation closed for this logical session.
|
||||
pub(super) attempt: u8,
|
||||
}
|
||||
|
||||
/// Token-bucket state for one process-wide creation class.
|
||||
#[derive(Default)]
|
||||
pub(super) struct RateState {
|
||||
@@ -114,7 +142,6 @@ pub(super) struct StreamAdmissionState {
|
||||
}
|
||||
|
||||
/// Process-wide WEB registries and quota accounting protected by one short lock.
|
||||
#[derive(Default)]
|
||||
pub(super) struct ManagerState {
|
||||
/// Bootstrap credentials indexed by their SHA-256 token hash.
|
||||
pub(super) bootstraps: HashMap<TokenHash, Bootstrap>,
|
||||
@@ -122,6 +149,12 @@ pub(super) struct ManagerState {
|
||||
pub(super) bootstraps_per_ip: HashMap<IpAddr, usize>,
|
||||
/// Live sessions indexed by bearer-token hash.
|
||||
pub(super) sessions: HashMap<TokenHash, Arc<WebSession>>,
|
||||
/// Stable ordered logical-session lookup independent from bearer hashes.
|
||||
pub(super) session_index: BTreeMap<u64, LiveSessionIndex>,
|
||||
/// Recently closed logical sessions retained for exact detail responses.
|
||||
pub(super) closed_sessions: HashMap<u64, ClosedSession>,
|
||||
/// Insertion order for bounded logical-session tombstones.
|
||||
pub(super) closed_session_order: VecDeque<u64>,
|
||||
/// Recently closed token hashes retained for idempotent DELETE semantics.
|
||||
pub(super) closed_tokens: HashMap<TokenHash, ClosedToken>,
|
||||
/// Live session counts by forwarded client address.
|
||||
@@ -132,10 +165,48 @@ pub(super) struct ManagerState {
|
||||
pub(super) bootstrap_rate: RateState,
|
||||
/// Session creation rate limiter.
|
||||
pub(super) session_rate: RateState,
|
||||
/// Generation-fenced issuance gate mirrored from the effective WEB policy.
|
||||
pub(super) issuance_enabled: bool,
|
||||
/// Generation that last authored `issuance_enabled`.
|
||||
pub(super) issuance_generation: u64,
|
||||
/// Process shutdown admission latch.
|
||||
pub(super) closed: bool,
|
||||
}
|
||||
|
||||
impl ManagerState {
|
||||
pub(super) fn new(issuance_generation: u64, issuance_enabled: bool) -> Self {
|
||||
Self {
|
||||
bootstraps: HashMap::new(),
|
||||
bootstraps_per_ip: HashMap::new(),
|
||||
sessions: HashMap::new(),
|
||||
session_index: BTreeMap::new(),
|
||||
closed_sessions: HashMap::new(),
|
||||
closed_session_order: VecDeque::new(),
|
||||
closed_tokens: HashMap::new(),
|
||||
sessions_per_ip: HashMap::new(),
|
||||
sessions_per_profile: HashMap::new(),
|
||||
bootstrap_rate: RateState::default(),
|
||||
session_rate: RateState::default(),
|
||||
issuance_enabled,
|
||||
issuance_generation,
|
||||
closed: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn apply_issuance_policy(&mut self, generation: u64, enabled: bool) {
|
||||
if generation >= self.issuance_generation {
|
||||
self.issuance_generation = generation;
|
||||
self.issuance_enabled = enabled;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ManagerState {
|
||||
fn default() -> Self {
|
||||
Self::new(0, false)
|
||||
}
|
||||
}
|
||||
|
||||
/// Generates one collision-checked credential and its stable hash key.
|
||||
pub(super) fn new_unique_token(
|
||||
generation: &RuntimeGeneration,
|
||||
@@ -241,6 +312,16 @@ pub(super) fn remove_expired_locked(state: &mut ManagerState, now: Instant) {
|
||||
state
|
||||
.closed_tokens
|
||||
.retain(|_, closed| now <= closed.expires_at);
|
||||
while state
|
||||
.closed_session_order
|
||||
.front()
|
||||
.and_then(|trace_session_id| state.closed_sessions.get(trace_session_id))
|
||||
.is_some_and(|closed| now > closed.expires_at)
|
||||
{
|
||||
if let Some(trace_session_id) = state.closed_session_order.pop_front() {
|
||||
state.closed_sessions.remove(&trace_session_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Removes one bootstrap and releases its per-address issuance quota when unused.
|
||||
@@ -281,6 +362,35 @@ pub(super) fn remember_closed_token_locked(
|
||||
}
|
||||
}
|
||||
|
||||
/// Retains one bounded logical-session marker without storing its bearer identity.
|
||||
pub(super) fn remember_closed_session_locked(
|
||||
state: &mut ManagerState,
|
||||
trace_session_id: u64,
|
||||
attempt: u8,
|
||||
lifetime: Duration,
|
||||
capacity: usize,
|
||||
) {
|
||||
if state
|
||||
.closed_sessions
|
||||
.insert(
|
||||
trace_session_id,
|
||||
ClosedSession {
|
||||
expires_at: Instant::now() + lifetime,
|
||||
attempt,
|
||||
},
|
||||
)
|
||||
.is_none()
|
||||
{
|
||||
state.closed_session_order.push_back(trace_session_id);
|
||||
}
|
||||
while state.closed_sessions.len() > capacity {
|
||||
let Some(oldest) = state.closed_session_order.pop_front() else {
|
||||
break;
|
||||
};
|
||||
state.closed_sessions.remove(&oldest);
|
||||
}
|
||||
}
|
||||
|
||||
/// Decrements one counted owner and removes its map entry at zero.
|
||||
pub(super) fn decrement_map<K, Q>(values: &mut HashMap<K, usize>, key: &Q)
|
||||
where
|
||||
|
||||
@@ -0,0 +1,560 @@
|
||||
use std::net::IpAddr;
|
||||
use std::ops::Bound::{Excluded, Unbounded};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::Instant;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
use super::WebProcessRuntime;
|
||||
use crate::config::{
|
||||
WebCarrier, WebCarrierNegotiationAggressiveness, WebDebugConfig, WebLimitsConfig,
|
||||
};
|
||||
use crate::web::session::WebSessionStatus;
|
||||
|
||||
const SESSION_REF_VERSION: &str = "ws1";
|
||||
const MAX_SESSION_SCAN: usize = 1000;
|
||||
|
||||
/// Current usage of one process-owned semaphore.
|
||||
#[derive(Clone, Serialize)]
|
||||
struct PermitStatus {
|
||||
used: usize,
|
||||
available: usize,
|
||||
capacity: usize,
|
||||
closed: bool,
|
||||
}
|
||||
|
||||
/// Short-lock manager registry counts.
|
||||
#[derive(Clone, Serialize)]
|
||||
struct ManagerStatus {
|
||||
issuance_enabled: bool,
|
||||
issuance_generation: u64,
|
||||
shutdown: bool,
|
||||
bootstraps: usize,
|
||||
sessions: usize,
|
||||
closed_tokens: usize,
|
||||
closed_sessions: usize,
|
||||
client_ips: usize,
|
||||
profiles: usize,
|
||||
}
|
||||
|
||||
/// Logical-stream admission counters.
|
||||
#[derive(Clone, Serialize)]
|
||||
struct StreamStatus {
|
||||
live: usize,
|
||||
profiles: usize,
|
||||
closed: bool,
|
||||
}
|
||||
|
||||
/// Shared queue and WebSocket byte-budget counters.
|
||||
#[derive(Clone, Serialize)]
|
||||
struct BudgetStatus {
|
||||
queue_bytes: usize,
|
||||
queue_items: usize,
|
||||
control_bytes: usize,
|
||||
control_items: usize,
|
||||
websocket_bytes: usize,
|
||||
high_water_bytes: usize,
|
||||
owners: usize,
|
||||
closed: bool,
|
||||
}
|
||||
|
||||
/// Process WebSocket registry counters.
|
||||
#[derive(Clone, Serialize)]
|
||||
struct WebSocketStatus {
|
||||
entries: usize,
|
||||
claims: usize,
|
||||
evictions_in_flight: usize,
|
||||
closed: bool,
|
||||
}
|
||||
|
||||
/// Process-local carrier-learning summary.
|
||||
#[derive(Clone, Serialize)]
|
||||
struct LearningStatus {
|
||||
enabled: bool,
|
||||
aggressiveness: WebCarrierNegotiationAggressiveness,
|
||||
epoch: Option<u64>,
|
||||
entries: usize,
|
||||
capacity: usize,
|
||||
lifetime_secs: u64,
|
||||
age_ms: u64,
|
||||
}
|
||||
|
||||
/// Effective trace policy and bounded ring counters.
|
||||
#[derive(Clone, Serialize)]
|
||||
struct DebugStatus {
|
||||
policy: WebDebugConfig,
|
||||
policy_generation: u64,
|
||||
epoch: u64,
|
||||
records: usize,
|
||||
records_capacity: usize,
|
||||
used_bytes: usize,
|
||||
bytes_capacity: usize,
|
||||
contention_drops: u64,
|
||||
evictions: u64,
|
||||
byte_truncations: u64,
|
||||
earliest_seq: Option<u64>,
|
||||
latest_seq: Option<u64>,
|
||||
}
|
||||
|
||||
/// One non-blocking multi-plane WEB runtime snapshot.
|
||||
#[derive(Clone, Serialize)]
|
||||
pub(crate) struct WebRuntimeStatus {
|
||||
runtime_instance: String,
|
||||
generation_id: u64,
|
||||
limits: WebLimitsConfig,
|
||||
manager: Option<ManagerStatus>,
|
||||
streams: Option<StreamStatus>,
|
||||
budget: Option<BudgetStatus>,
|
||||
websockets: Option<WebSocketStatus>,
|
||||
learning: Option<LearningStatus>,
|
||||
debug: Option<DebugStatus>,
|
||||
permits: Vec<(&'static str, PermitStatus)>,
|
||||
auxiliary_tasks: usize,
|
||||
session_incarnations_created: u64,
|
||||
session_incarnations_closed: u64,
|
||||
streams_opened: u64,
|
||||
streams_rejected: u64,
|
||||
bytes_up: u64,
|
||||
bytes_down: u64,
|
||||
limit_hits: u64,
|
||||
partial: Vec<&'static str>,
|
||||
}
|
||||
|
||||
/// Strict bounded filters for session enumeration and bulk close.
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct SessionFilter {
|
||||
/// Exact process-local logical session identifier.
|
||||
pub(crate) trace_session_id: Option<u64>,
|
||||
/// Exact forwarded client address.
|
||||
pub(crate) client_ip: Option<IpAddr>,
|
||||
/// Exact canonical virtual host.
|
||||
pub(crate) host: Option<String>,
|
||||
/// Exact configured user label.
|
||||
pub(crate) user: Option<String>,
|
||||
/// Exact non-secret User-Agent identifier.
|
||||
pub(crate) user_agent_id: Option<[u8; 16]>,
|
||||
/// Exact non-secret profile-key fingerprint.
|
||||
pub(crate) key_id: Option<String>,
|
||||
/// Exact current carrier.
|
||||
pub(crate) carrier: Option<WebCarrier>,
|
||||
/// Exact point-in-time lifecycle token.
|
||||
pub(crate) state: Option<String>,
|
||||
}
|
||||
|
||||
impl SessionFilter {
|
||||
/// Returns whether the selector would match every live session.
|
||||
pub(crate) fn is_empty(&self) -> bool {
|
||||
self.trace_session_id.is_none()
|
||||
&& self.client_ip.is_none()
|
||||
&& self.host.is_none()
|
||||
&& self.user.is_none()
|
||||
&& self.user_agent_id.is_none()
|
||||
&& self.key_id.is_none()
|
||||
&& self.carrier.is_none()
|
||||
&& self.state.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
/// Validated bounded list request.
|
||||
pub(crate) struct SessionListRequest {
|
||||
/// Maximum returned rows.
|
||||
pub(crate) limit: usize,
|
||||
/// Exclusive ordered logical-session cursor.
|
||||
pub(crate) cursor: Option<u64>,
|
||||
/// Exact bounded filters.
|
||||
pub(crate) filter: SessionFilter,
|
||||
}
|
||||
|
||||
/// One API-safe session row with optional User-Agent metadata.
|
||||
#[derive(Clone, Serialize)]
|
||||
pub(crate) struct SessionRow {
|
||||
/// Opaque process-fenced logical-session reference.
|
||||
pub(crate) session_ref: String,
|
||||
/// Bounded sanitized User-Agent display value.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) user_agent: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
user_agent_id: Option<String>,
|
||||
#[serde(flatten)]
|
||||
status: WebSessionStatus,
|
||||
}
|
||||
|
||||
/// Bounded session page and continuation metadata.
|
||||
#[derive(Clone, Serialize)]
|
||||
pub(crate) struct SessionPage {
|
||||
/// Ordered live-session rows captured without blocking.
|
||||
pub(crate) sessions: Vec<SessionRow>,
|
||||
next_cursor: Option<String>,
|
||||
scanned: usize,
|
||||
scan_truncated: bool,
|
||||
partial_sessions: usize,
|
||||
partial: Vec<&'static str>,
|
||||
}
|
||||
|
||||
/// Exact detail lookup outcome.
|
||||
pub(crate) enum SessionDetail {
|
||||
/// One exact live-session snapshot.
|
||||
Active(Box<SessionRow>),
|
||||
/// One bounded retained closed-session tombstone.
|
||||
Gone { attempt: u8 },
|
||||
/// A required short lock was contended.
|
||||
Busy,
|
||||
/// Neither a live session nor a retained tombstone exists.
|
||||
NotFound,
|
||||
}
|
||||
|
||||
struct Candidate {
|
||||
trace_session_id: u64,
|
||||
session: Arc<crate::web::session::WebSession>,
|
||||
user_agent: Option<Arc<str>>,
|
||||
user_agent_id: Option<[u8; 16]>,
|
||||
}
|
||||
|
||||
impl WebProcessRuntime {
|
||||
/// Captures every independent plane without blocking on a contended lock.
|
||||
pub(crate) fn try_status(&self) -> WebRuntimeStatus {
|
||||
let generation_id = self.active_generation().id;
|
||||
let mut partial = Vec::new();
|
||||
let manager = self.state.try_lock().map(|state| ManagerStatus {
|
||||
issuance_enabled: state.issuance_enabled,
|
||||
issuance_generation: state.issuance_generation,
|
||||
shutdown: state.closed,
|
||||
bootstraps: state.bootstraps.len(),
|
||||
sessions: state.sessions.len(),
|
||||
closed_tokens: state.closed_tokens.len(),
|
||||
closed_sessions: state.closed_sessions.len(),
|
||||
client_ips: state.sessions_per_ip.len(),
|
||||
profiles: state.sessions_per_profile.len(),
|
||||
});
|
||||
if manager.is_none() {
|
||||
partial.push("manager");
|
||||
}
|
||||
let streams = self.stream_admission.try_lock().map(|state| StreamStatus {
|
||||
live: state.streams_live,
|
||||
profiles: state.streams_per_profile.len(),
|
||||
closed: state.closed,
|
||||
});
|
||||
if streams.is_none() {
|
||||
partial.push("streams");
|
||||
}
|
||||
let budget = self.data_budget.try_snapshot().map(|status| BudgetStatus {
|
||||
queue_bytes: status.queue_bytes,
|
||||
queue_items: status.queue_items,
|
||||
control_bytes: status.queue_control_bytes,
|
||||
control_items: status.queue_control_items,
|
||||
websocket_bytes: status.websocket_bytes,
|
||||
high_water_bytes: status.high_water_bytes,
|
||||
owners: status.owners,
|
||||
closed: status.closed,
|
||||
});
|
||||
if budget.is_none() {
|
||||
partial.push("budget");
|
||||
}
|
||||
let websockets = self.websockets.try_lock().map(|registry| {
|
||||
let status = registry.status();
|
||||
WebSocketStatus {
|
||||
entries: status.entries,
|
||||
claims: status.claims,
|
||||
evictions_in_flight: status.evictions_in_flight,
|
||||
closed: status.closed,
|
||||
}
|
||||
});
|
||||
if websockets.is_none() {
|
||||
partial.push("websockets");
|
||||
}
|
||||
let learning = self
|
||||
.try_carrier_learning_status()
|
||||
.map(|status| LearningStatus {
|
||||
enabled: status.enabled,
|
||||
aggressiveness: status.aggressiveness,
|
||||
epoch: status.epoch,
|
||||
entries: status.entries,
|
||||
capacity: status.capacity,
|
||||
lifetime_secs: status.lifetime_secs,
|
||||
age_ms: status.age_ms,
|
||||
});
|
||||
if learning.is_none() {
|
||||
partial.push("learning");
|
||||
}
|
||||
let debug = self.trace.try_status().map(|status| DebugStatus {
|
||||
policy: status.policy.as_ref().clone(),
|
||||
policy_generation: status.policy_generation,
|
||||
epoch: status.epoch,
|
||||
records: status.records,
|
||||
records_capacity: status.records_capacity,
|
||||
used_bytes: status.used_bytes,
|
||||
bytes_capacity: status.bytes_capacity,
|
||||
contention_drops: status.contention_drops,
|
||||
evictions: status.evictions,
|
||||
byte_truncations: status.byte_truncations,
|
||||
earliest_seq: status.earliest_seq,
|
||||
latest_seq: status.latest_seq,
|
||||
});
|
||||
if debug.is_none() {
|
||||
partial.push("debug");
|
||||
}
|
||||
let websocket_capacity = self
|
||||
.limits
|
||||
.max_http_connections
|
||||
.saturating_sub(self.limits.websocket_http_connection_reserve);
|
||||
WebRuntimeStatus {
|
||||
runtime_instance: self.runtime_instance().to_string(),
|
||||
generation_id,
|
||||
limits: self.limits.clone(),
|
||||
manager,
|
||||
streams,
|
||||
budget,
|
||||
websockets,
|
||||
learning,
|
||||
debug,
|
||||
permits: vec![
|
||||
(
|
||||
"http_connections",
|
||||
permits(&self.http_connections, self.limits.max_http_connections),
|
||||
),
|
||||
(
|
||||
"http_handlers",
|
||||
permits(&self.http_handlers, self.limits.max_http_handlers),
|
||||
),
|
||||
(
|
||||
"lane_polls",
|
||||
permits(&self.lane_polls, self.limits.max_http_handlers / 2),
|
||||
),
|
||||
(
|
||||
"lane_aux_polls",
|
||||
permits(
|
||||
&self.lane_aux_polls,
|
||||
(self.limits.max_http_handlers / 4).max(1),
|
||||
),
|
||||
),
|
||||
(
|
||||
"body_readers",
|
||||
permits(&self.body_readers, self.limits.max_body_readers),
|
||||
),
|
||||
(
|
||||
"body_bytes",
|
||||
permits(&self.body_bytes, self.limits.max_body_bytes_global),
|
||||
),
|
||||
(
|
||||
"stream_handshakes",
|
||||
permits(&self.stream_handshakes, self.limits.max_stream_handshakes),
|
||||
),
|
||||
(
|
||||
"websocket_connections",
|
||||
permits(&self.websocket_connections, websocket_capacity),
|
||||
),
|
||||
],
|
||||
auxiliary_tasks: self.tasks.len(),
|
||||
session_incarnations_created: self.sessions_created.load(Ordering::Relaxed),
|
||||
session_incarnations_closed: self.sessions_closed.load(Ordering::Relaxed),
|
||||
streams_opened: self.streams_opened.load(Ordering::Relaxed),
|
||||
streams_rejected: self.streams_rejected.load(Ordering::Relaxed),
|
||||
bytes_up: self.bytes_up.load(Ordering::Relaxed),
|
||||
bytes_down: self.bytes_down.load(Ordering::Relaxed),
|
||||
limit_hits: self.limit_hits.load(Ordering::Relaxed),
|
||||
partial,
|
||||
}
|
||||
}
|
||||
|
||||
/// Formats an opaque process-fenced session reference.
|
||||
pub(crate) fn session_ref(&self, trace_session_id: u64) -> String {
|
||||
format!(
|
||||
"{SESSION_REF_VERSION}.{}.{trace_session_id:016x}",
|
||||
self.runtime_instance()
|
||||
)
|
||||
}
|
||||
|
||||
/// Parses an exact reference and distinguishes stale process instances.
|
||||
pub(crate) fn parse_session_ref(&self, value: &str) -> Result<u64, SessionRefError> {
|
||||
let mut parts = value.split('.');
|
||||
let version = parts.next();
|
||||
let instance = parts.next();
|
||||
let id = parts.next();
|
||||
if version != Some(SESSION_REF_VERSION) || parts.next().is_some() {
|
||||
return Err(SessionRefError::Invalid);
|
||||
}
|
||||
if !instance.is_some_and(|instance| {
|
||||
instance.len() == 32
|
||||
&& instance
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
}) {
|
||||
return Err(SessionRefError::Invalid);
|
||||
}
|
||||
let id = id
|
||||
.filter(|id| {
|
||||
id.len() == 16
|
||||
&& id
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
})
|
||||
.and_then(|id| u64::from_str_radix(id, 16).ok())
|
||||
.filter(|id| *id != 0)
|
||||
.ok_or(SessionRefError::Invalid)?;
|
||||
if instance != Some(self.runtime_instance()) {
|
||||
return Err(SessionRefError::StaleInstance);
|
||||
}
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Lists live sessions under an ordered bounded scan.
|
||||
pub(crate) fn list_sessions(&self, request: SessionListRequest) -> SessionPage {
|
||||
let Some(state) = self.state.try_lock() else {
|
||||
return SessionPage {
|
||||
sessions: Vec::new(),
|
||||
next_cursor: request.cursor.map(|id| self.session_ref(id)),
|
||||
scanned: 0,
|
||||
scan_truncated: false,
|
||||
partial_sessions: 0,
|
||||
partial: vec!["manager"],
|
||||
};
|
||||
};
|
||||
let mut candidates = Vec::with_capacity(request.limit);
|
||||
let mut scanned = 0usize;
|
||||
let mut last_scanned = request.cursor;
|
||||
for (&trace_session_id, index) in state
|
||||
.session_index
|
||||
.range((request.cursor.map_or(Unbounded, Excluded), Unbounded))
|
||||
{
|
||||
if scanned >= MAX_SESSION_SCAN || candidates.len() >= request.limit {
|
||||
break;
|
||||
}
|
||||
scanned += 1;
|
||||
last_scanned = Some(trace_session_id);
|
||||
let Some(session) = state.sessions.get(&index.session_hash).cloned() else {
|
||||
continue;
|
||||
};
|
||||
if !immutable_matches(&session, index, &request.filter) {
|
||||
continue;
|
||||
}
|
||||
candidates.push(Candidate {
|
||||
trace_session_id,
|
||||
session,
|
||||
user_agent: index.user_agent.clone(),
|
||||
user_agent_id: index.user_agent_id,
|
||||
});
|
||||
}
|
||||
drop(state);
|
||||
let mut rows = Vec::with_capacity(candidates.len());
|
||||
let mut partial_sessions = 0usize;
|
||||
let now = Instant::now();
|
||||
for candidate in candidates {
|
||||
let Some(status) = candidate.session.try_status(now) else {
|
||||
partial_sessions += 1;
|
||||
continue;
|
||||
};
|
||||
if request
|
||||
.filter
|
||||
.state
|
||||
.as_deref()
|
||||
.is_some_and(|expected| expected != status.state)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
rows.push(self.row(candidate, status));
|
||||
}
|
||||
let scan_truncated = scanned >= MAX_SESSION_SCAN;
|
||||
SessionPage {
|
||||
sessions: rows,
|
||||
next_cursor: (scan_truncated || scanned >= request.limit)
|
||||
.then(|| last_scanned.map(|id| self.session_ref(id)))
|
||||
.flatten(),
|
||||
scanned,
|
||||
scan_truncated,
|
||||
partial_sessions,
|
||||
partial: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves one active or recently closed logical session.
|
||||
pub(crate) fn session_detail(&self, trace_session_id: u64) -> SessionDetail {
|
||||
let Some(state) = self.state.try_lock() else {
|
||||
return SessionDetail::Busy;
|
||||
};
|
||||
if let Some(index) = state.session_index.get(&trace_session_id) {
|
||||
let Some(session) = state.sessions.get(&index.session_hash).cloned() else {
|
||||
return SessionDetail::Busy;
|
||||
};
|
||||
let candidate = Candidate {
|
||||
trace_session_id,
|
||||
session,
|
||||
user_agent: index.user_agent.clone(),
|
||||
user_agent_id: index.user_agent_id,
|
||||
};
|
||||
drop(state);
|
||||
return candidate
|
||||
.session
|
||||
.try_status(Instant::now())
|
||||
.map(|status| SessionDetail::Active(Box::new(self.row(candidate, status))))
|
||||
.unwrap_or(SessionDetail::Busy);
|
||||
}
|
||||
let closed = state
|
||||
.closed_sessions
|
||||
.get(&trace_session_id)
|
||||
.map(|closed| closed.attempt);
|
||||
closed.map_or(SessionDetail::NotFound, |attempt| SessionDetail::Gone {
|
||||
attempt,
|
||||
})
|
||||
}
|
||||
|
||||
fn row(&self, candidate: Candidate, status: WebSessionStatus) -> SessionRow {
|
||||
SessionRow {
|
||||
session_ref: self.session_ref(candidate.trace_session_id),
|
||||
user_agent: candidate.user_agent.map(|value| value.to_string()),
|
||||
user_agent_id: candidate.user_agent_id.map(hex::encode),
|
||||
status,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Opaque session-reference validation failure.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum SessionRefError {
|
||||
/// The reference does not use the canonical versioned shape.
|
||||
Invalid,
|
||||
/// The reference belongs to another process runtime.
|
||||
StaleInstance,
|
||||
}
|
||||
|
||||
/// Tests immutable candidate fields before any optional state-lock read.
|
||||
pub(super) fn immutable_matches(
|
||||
session: &crate::web::session::WebSession,
|
||||
index: &super::state::LiveSessionIndex,
|
||||
filter: &SessionFilter,
|
||||
) -> bool {
|
||||
filter
|
||||
.trace_session_id
|
||||
.is_none_or(|value| value == session.trace_session_id())
|
||||
&& filter
|
||||
.client_ip
|
||||
.is_none_or(|value| value == session.client_ip())
|
||||
&& filter
|
||||
.host
|
||||
.as_deref()
|
||||
.is_none_or(|value| value == session.profile_host())
|
||||
&& filter
|
||||
.user
|
||||
.as_deref()
|
||||
.is_none_or(|value| value == session.profile_user())
|
||||
&& filter
|
||||
.key_id
|
||||
.as_deref()
|
||||
.is_none_or(|value| session.key_id() == value)
|
||||
&& filter
|
||||
.carrier
|
||||
.is_none_or(|value| value == session.carrier())
|
||||
&& filter
|
||||
.user_agent_id
|
||||
.is_none_or(|value| index.user_agent_id == Some(value))
|
||||
}
|
||||
|
||||
fn permits(semaphore: &Arc<tokio::sync::Semaphore>, capacity: usize) -> PermitStatus {
|
||||
let available = semaphore.available_permits().min(capacity);
|
||||
PermitStatus {
|
||||
used: capacity.saturating_sub(available),
|
||||
available,
|
||||
capacity,
|
||||
closed: semaphore.is_closed(),
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,9 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::{ManagerError, ProfileKey, WebProcessRuntime, WebSocketBudgetLease};
|
||||
|
||||
// Deterministic victim ordering remains isolated from registry mutation.
|
||||
mod policy;
|
||||
|
||||
/// One process-owned WebSocket carrier class used for eviction priority.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub(crate) enum WebSocketKind {
|
||||
@@ -25,6 +28,7 @@ struct WebSocketClaimKey {
|
||||
}
|
||||
|
||||
#[repr(u8)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum WebSocketPhase {
|
||||
Claimed,
|
||||
Upgraded,
|
||||
@@ -57,6 +61,26 @@ pub(super) struct WebSocketRegistry {
|
||||
closed: bool,
|
||||
}
|
||||
|
||||
/// Point-in-time WebSocket registry counters.
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct WebSocketRegistryStatus {
|
||||
pub(super) entries: usize,
|
||||
pub(super) claims: usize,
|
||||
pub(super) evictions_in_flight: usize,
|
||||
pub(super) closed: bool,
|
||||
}
|
||||
|
||||
impl WebSocketRegistry {
|
||||
pub(super) fn status(&self) -> WebSocketRegistryStatus {
|
||||
WebSocketRegistryStatus {
|
||||
entries: self.entries.len(),
|
||||
claims: self.claims.len(),
|
||||
evictions_in_flight: self.evictions_in_flight,
|
||||
closed: self.closed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Exact process-owned admission retained through the upgraded socket lifetime.
|
||||
pub(crate) struct WebSocketConnection {
|
||||
runtime: std::sync::Weak<WebProcessRuntime>,
|
||||
@@ -239,6 +263,8 @@ enum TryAdmitError {
|
||||
Closed,
|
||||
}
|
||||
|
||||
// Admission inputs stay explicit so quota and cancellation ownership cannot drift.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn try_admit(
|
||||
runtime: &Arc<WebProcessRuntime>,
|
||||
owner: ProfileKey,
|
||||
@@ -353,8 +379,7 @@ fn select_victim(
|
||||
excluded_id: Option<u64>,
|
||||
claim: bool,
|
||||
) -> Option<Arc<WebSocketEntry>> {
|
||||
let fair_share = runtime.data_budget.fair_share(Some(owner));
|
||||
let requester_usage = runtime.data_budget.owner_usage(owner);
|
||||
let fairness = runtime.data_budget.fairness_snapshot(Some(owner));
|
||||
let now = runtime.websocket_tick();
|
||||
let mut registry = runtime.websockets.lock();
|
||||
if claim && registry.evictions_in_flight >= runtime.limits.max_websocket_evictions_in_flight {
|
||||
@@ -366,31 +391,8 @@ fn select_victim(
|
||||
.filter(|entry| Some(entry.id) != excluded_id)
|
||||
.filter(|entry| !entry.closing.load(Ordering::Acquire))
|
||||
.filter_map(|entry| {
|
||||
let owner_rank = if entry.session_id == session_id {
|
||||
0
|
||||
} else if entry.owner == owner {
|
||||
1
|
||||
} else if entry.client_ip == client_ip {
|
||||
2
|
||||
} else {
|
||||
if requester_usage >= fair_share
|
||||
|| runtime.data_budget.owner_usage(entry.owner) <= fair_share
|
||||
{
|
||||
return None;
|
||||
}
|
||||
3
|
||||
};
|
||||
let priority = entry_priority(entry, now);
|
||||
Some((
|
||||
(
|
||||
owner_rank,
|
||||
priority,
|
||||
entry.last_progress_tick.load(Ordering::Acquire),
|
||||
entry.created_tick,
|
||||
entry.id,
|
||||
),
|
||||
Arc::clone(entry),
|
||||
))
|
||||
policy::admission_key(entry, now, owner, session_id, client_ip, &fairness)
|
||||
.map(|key| (key, Arc::clone(entry)))
|
||||
})
|
||||
.min_by_key(|(key, _)| *key)
|
||||
.map(|(_, entry)| entry)?;
|
||||
@@ -405,6 +407,7 @@ fn select_pressure_victim(
|
||||
now: u64,
|
||||
claim: bool,
|
||||
) -> Option<Arc<WebSocketEntry>> {
|
||||
let fairness = runtime.data_budget.fairness_snapshot(None);
|
||||
let mut registry = runtime.websockets.lock();
|
||||
if claim && registry.evictions_in_flight >= runtime.limits.max_websocket_evictions_in_flight {
|
||||
return None;
|
||||
@@ -415,12 +418,7 @@ fn select_pressure_victim(
|
||||
.filter(|entry| !entry.closing.load(Ordering::Acquire))
|
||||
.map(|entry| {
|
||||
(
|
||||
(
|
||||
entry_priority(entry, now),
|
||||
entry.last_progress_tick.load(Ordering::Acquire),
|
||||
entry.created_tick,
|
||||
entry.id,
|
||||
),
|
||||
policy::pressure_key(entry, now, &fairness),
|
||||
Arc::clone(entry),
|
||||
)
|
||||
})
|
||||
@@ -438,18 +436,23 @@ fn claim_stale_victims(runtime: &WebProcessRuntime, now: u64) -> Vec<Arc<WebSock
|
||||
.limits
|
||||
.max_websocket_evictions_in_flight
|
||||
.saturating_sub(registry.evictions_in_flight);
|
||||
let candidates = registry
|
||||
let mut candidates = registry
|
||||
.entries
|
||||
.values()
|
||||
.filter(|entry| !entry.closing.load(Ordering::Acquire))
|
||||
.filter(|entry| {
|
||||
now.saturating_sub(entry.last_peer_tick.load(Ordering::Acquire)) >= dead_after(entry)
|
||||
})
|
||||
.take(available)
|
||||
.filter(|entry| policy::victim_class(entry, now) == policy::VictimClass::Dead)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
candidates.sort_unstable_by_key(|entry| {
|
||||
(
|
||||
entry.last_peer_tick.load(Ordering::Acquire),
|
||||
entry.created_tick,
|
||||
entry.id,
|
||||
)
|
||||
});
|
||||
candidates
|
||||
.into_iter()
|
||||
.take(available)
|
||||
.filter(|entry| claim_entry(&mut registry, entry, runtime))
|
||||
.collect()
|
||||
}
|
||||
@@ -474,18 +477,6 @@ fn claim_entry(
|
||||
true
|
||||
}
|
||||
|
||||
fn entry_priority(entry: &WebSocketEntry, now: u64) -> u8 {
|
||||
if entry.phase.load(Ordering::Acquire) < WebSocketPhase::Active as u8
|
||||
|| now.saturating_sub(entry.last_peer_tick.load(Ordering::Acquire)) >= dead_after(entry)
|
||||
{
|
||||
0
|
||||
} else if matches!(entry.kind, WebSocketKind::Lane(_)) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
}
|
||||
}
|
||||
|
||||
fn dead_after(entry: &WebSocketEntry) -> u64 {
|
||||
entry.liveness_interval_ms.saturating_mul(2)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
use std::net::IpAddr;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use super::{WebSocketEntry, WebSocketKind, WebSocketPhase, dead_after};
|
||||
use crate::web::manager::ProfileKey;
|
||||
use crate::web::manager::budget::WebSocketFairnessSnapshot;
|
||||
|
||||
/// Stable total-order key used by bounded victim selection.
|
||||
pub(super) type VictimKey = (u8, u8, u8, u64, u64, u64);
|
||||
|
||||
/// Lifecycle class used before locality and least-recent-progress ordering.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(super) enum VictimClass {
|
||||
/// Active connection whose peer-liveness deadline elapsed.
|
||||
Dead,
|
||||
/// Claimed or upgraded connection still bounded by its startup deadlines.
|
||||
PreActive,
|
||||
/// Active per-stream lane connection.
|
||||
LiveLane,
|
||||
/// Active multiplexed session connection.
|
||||
LiveMultiplex,
|
||||
}
|
||||
|
||||
impl VictimClass {
|
||||
fn rank(self) -> u8 {
|
||||
match self {
|
||||
Self::Dead | Self::PreActive => 0,
|
||||
Self::LiveLane => 1,
|
||||
Self::LiveMultiplex => 2,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Classifies one non-closing connection without conflating startup with death.
|
||||
pub(super) fn victim_class(entry: &WebSocketEntry, now: u64) -> VictimClass {
|
||||
let phase = entry.phase.load(Ordering::Acquire);
|
||||
if phase == WebSocketPhase::Active as u8
|
||||
&& now.saturating_sub(entry.last_peer_tick.load(Ordering::Acquire)) >= dead_after(entry)
|
||||
{
|
||||
VictimClass::Dead
|
||||
} else if phase < WebSocketPhase::Active as u8 {
|
||||
VictimClass::PreActive
|
||||
} else if matches!(entry.kind, WebSocketKind::Lane(_)) {
|
||||
VictimClass::LiveLane
|
||||
} else {
|
||||
VictimClass::LiveMultiplex
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns one eligible admission-victim key with global dead-first ordering.
|
||||
pub(super) fn admission_key(
|
||||
entry: &WebSocketEntry,
|
||||
now: u64,
|
||||
requester_owner: ProfileKey,
|
||||
requester_session: u64,
|
||||
requester_ip: IpAddr,
|
||||
fairness: &WebSocketFairnessSnapshot,
|
||||
) -> Option<VictimKey> {
|
||||
let class = victim_class(entry, now);
|
||||
if class == VictimClass::Dead {
|
||||
return Some((
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
entry.last_peer_tick.load(Ordering::Acquire),
|
||||
entry.created_tick,
|
||||
entry.id,
|
||||
));
|
||||
}
|
||||
let locality = if entry.session_id == requester_session {
|
||||
0
|
||||
} else if entry.owner == requester_owner {
|
||||
1
|
||||
} else if entry.client_ip == requester_ip {
|
||||
2
|
||||
} else if fairness.owner_usage(requester_owner) < fairness.fair_share
|
||||
&& fairness.owner_usage(entry.owner) > fairness.fair_share
|
||||
{
|
||||
3
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
Some((
|
||||
1,
|
||||
locality,
|
||||
class.rank(),
|
||||
entry.last_progress_tick.load(Ordering::Acquire),
|
||||
entry.created_tick,
|
||||
entry.id,
|
||||
))
|
||||
}
|
||||
|
||||
/// Returns one pressure-victim key preferring dead and over-share owners.
|
||||
pub(super) fn pressure_key(
|
||||
entry: &WebSocketEntry,
|
||||
now: u64,
|
||||
fairness: &WebSocketFairnessSnapshot,
|
||||
) -> VictimKey {
|
||||
let class = victim_class(entry, now);
|
||||
let dead = class == VictimClass::Dead;
|
||||
(
|
||||
u8::from(!dead),
|
||||
if dead {
|
||||
0
|
||||
} else {
|
||||
u8::from(fairness.owner_usage(entry.owner) <= fairness.fair_share)
|
||||
},
|
||||
class.rank(),
|
||||
entry.last_progress_tick.load(Ordering::Acquire),
|
||||
entry.created_tick,
|
||||
entry.id,
|
||||
)
|
||||
}
|
||||
@@ -1,25 +1,39 @@
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
|
||||
fn entry(kind: WebSocketKind, opened: bool, peer_tick: u64) -> WebSocketEntry {
|
||||
let phase = if opened {
|
||||
WebSocketPhase::Active
|
||||
} else {
|
||||
WebSocketPhase::Claimed
|
||||
};
|
||||
use super::policy::{VictimClass, admission_key, pressure_key, victim_class};
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::maestro::generation::test_runtime_generation;
|
||||
use crate::web::manager::budget::WebSocketFairnessSnapshot;
|
||||
use arc_swap::ArcSwap;
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn entry(
|
||||
id: u64,
|
||||
owner: ProfileKey,
|
||||
session_id: u64,
|
||||
client_ip: &str,
|
||||
kind: WebSocketKind,
|
||||
phase: WebSocketPhase,
|
||||
peer_tick: u64,
|
||||
progress_tick: u64,
|
||||
) -> WebSocketEntry {
|
||||
WebSocketEntry {
|
||||
id: 1,
|
||||
owner: [0; 32],
|
||||
session_id: 1,
|
||||
id,
|
||||
owner,
|
||||
session_id,
|
||||
claim: WebSocketClaimKey {
|
||||
session_hash: [0; 32],
|
||||
kind,
|
||||
},
|
||||
client_ip: "192.0.2.10".parse().unwrap(),
|
||||
client_ip: client_ip.parse().unwrap(),
|
||||
kind,
|
||||
liveness_interval_ms: 10,
|
||||
created_tick: 1,
|
||||
last_peer_tick: AtomicU64::new(peer_tick),
|
||||
last_progress_tick: AtomicU64::new(peer_tick),
|
||||
last_progress_tick: AtomicU64::new(progress_tick),
|
||||
phase: AtomicU8::new(phase as u8),
|
||||
closing: AtomicBool::new(false),
|
||||
cancel: CancellationToken::new(),
|
||||
@@ -27,25 +41,259 @@ fn entry(kind: WebSocketKind, opened: bool, peer_tick: u64) -> WebSocketEntry {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preopen_and_dead_entries_precede_live_lane_and_multiplex_victims() {
|
||||
let preopen = entry(WebSocketKind::Multiplex, false, 90);
|
||||
let dead = entry(WebSocketKind::Multiplex, true, 1);
|
||||
let lane = entry(WebSocketKind::Lane(7), true, 90);
|
||||
let multiplex = entry(WebSocketKind::Multiplex, true, 90);
|
||||
|
||||
assert_eq!(entry_priority(&preopen, 100), 0);
|
||||
assert_eq!(entry_priority(&dead, 100), 0);
|
||||
assert_eq!(entry_priority(&lane, 100), 1);
|
||||
assert_eq!(entry_priority(&multiplex, 100), 2);
|
||||
fn fairness(fair_share: usize, usages: &[(ProfileKey, usize)]) -> WebSocketFairnessSnapshot {
|
||||
WebSocketFairnessSnapshot {
|
||||
fair_share,
|
||||
owner_bytes: usages.iter().copied().collect::<HashMap<_, _>>(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dead_classification_keeps_each_connections_creation_time_interval() {
|
||||
let short_interval = entry(WebSocketKind::Multiplex, true, 80);
|
||||
let mut long_interval = entry(WebSocketKind::Multiplex, true, 80);
|
||||
long_interval.liveness_interval_ms = 100;
|
||||
fn preactive_and_dead_are_distinct_lifecycle_classes() {
|
||||
let preactive = entry(
|
||||
1,
|
||||
[1; 32],
|
||||
1,
|
||||
"192.0.2.10",
|
||||
WebSocketKind::Multiplex,
|
||||
WebSocketPhase::Claimed,
|
||||
1,
|
||||
1,
|
||||
);
|
||||
let dead = entry(
|
||||
2,
|
||||
[1; 32],
|
||||
1,
|
||||
"192.0.2.10",
|
||||
WebSocketKind::Multiplex,
|
||||
WebSocketPhase::Active,
|
||||
1,
|
||||
1,
|
||||
);
|
||||
|
||||
assert_eq!(entry_priority(&short_interval, 100), 0);
|
||||
assert_eq!(entry_priority(&long_interval, 100), 2);
|
||||
assert_eq!(victim_class(&preactive, 100), VictimClass::PreActive);
|
||||
assert_eq!(victim_class(&dead, 100), VictimClass::Dead);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dead_other_session_precedes_healthy_same_session() {
|
||||
let requester_owner = [1; 32];
|
||||
let usage = fairness(100, &[(requester_owner, 100), ([2; 32], 100)]);
|
||||
let dead = entry(
|
||||
2,
|
||||
[2; 32],
|
||||
2,
|
||||
"198.51.100.10",
|
||||
WebSocketKind::Multiplex,
|
||||
WebSocketPhase::Active,
|
||||
1,
|
||||
1,
|
||||
);
|
||||
let healthy = entry(
|
||||
1,
|
||||
requester_owner,
|
||||
1,
|
||||
"192.0.2.10",
|
||||
WebSocketKind::Lane(7),
|
||||
WebSocketPhase::Active,
|
||||
99,
|
||||
99,
|
||||
);
|
||||
|
||||
let dead_key = admission_key(
|
||||
&dead,
|
||||
100,
|
||||
requester_owner,
|
||||
1,
|
||||
"192.0.2.10".parse().unwrap(),
|
||||
&usage,
|
||||
)
|
||||
.unwrap();
|
||||
let healthy_key = admission_key(
|
||||
&healthy,
|
||||
100,
|
||||
requester_owner,
|
||||
1,
|
||||
"192.0.2.10".parse().unwrap(),
|
||||
&usage,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(dead_key < healthy_key);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unrelated_live_victim_requires_opposite_fair_share_positions() {
|
||||
let requester_owner = [1; 32];
|
||||
let victim_owner = [2; 32];
|
||||
let candidate = entry(
|
||||
1,
|
||||
victim_owner,
|
||||
2,
|
||||
"198.51.100.10",
|
||||
WebSocketKind::Lane(7),
|
||||
WebSocketPhase::Active,
|
||||
99,
|
||||
99,
|
||||
);
|
||||
let requester_ip = "192.0.2.10".parse().unwrap();
|
||||
|
||||
assert!(
|
||||
admission_key(
|
||||
&candidate,
|
||||
100,
|
||||
requester_owner,
|
||||
1,
|
||||
requester_ip,
|
||||
&fairness(100, &[(requester_owner, 99), (victim_owner, 101)]),
|
||||
)
|
||||
.is_some()
|
||||
);
|
||||
assert!(
|
||||
admission_key(
|
||||
&candidate,
|
||||
100,
|
||||
requester_owner,
|
||||
1,
|
||||
requester_ip,
|
||||
&fairness(100, &[(requester_owner, 100), (victim_owner, 101)]),
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pressure_prefers_over_share_owner_then_lifecycle_and_id() {
|
||||
let over_owner = [1; 32];
|
||||
let under_owner = [2; 32];
|
||||
let usage = fairness(100, &[(over_owner, 101), (under_owner, 99)]);
|
||||
let over = entry(
|
||||
9,
|
||||
over_owner,
|
||||
1,
|
||||
"192.0.2.10",
|
||||
WebSocketKind::Multiplex,
|
||||
WebSocketPhase::Active,
|
||||
99,
|
||||
99,
|
||||
);
|
||||
let under = entry(
|
||||
1,
|
||||
under_owner,
|
||||
2,
|
||||
"198.51.100.10",
|
||||
WebSocketKind::Lane(7),
|
||||
WebSocketPhase::Active,
|
||||
90,
|
||||
90,
|
||||
);
|
||||
|
||||
assert!(pressure_key(&over, 100, &usage) < pressure_key(&under, 100, &usage));
|
||||
|
||||
let equal_usage = fairness(100, &[(over_owner, 100), (under_owner, 100)]);
|
||||
let preactive = entry(
|
||||
2,
|
||||
under_owner,
|
||||
2,
|
||||
"198.51.100.10",
|
||||
WebSocketKind::Multiplex,
|
||||
WebSocketPhase::Upgraded,
|
||||
99,
|
||||
99,
|
||||
);
|
||||
assert!(pressure_key(&preactive, 100, &equal_usage) < pressure_key(&under, 100, &equal_usage));
|
||||
|
||||
let lower_id = entry(
|
||||
1,
|
||||
under_owner,
|
||||
2,
|
||||
"198.51.100.10",
|
||||
WebSocketKind::Lane(7),
|
||||
WebSocketPhase::Active,
|
||||
90,
|
||||
90,
|
||||
);
|
||||
let higher_id = entry(
|
||||
2,
|
||||
under_owner,
|
||||
2,
|
||||
"198.51.100.10",
|
||||
WebSocketKind::Lane(8),
|
||||
WebSocketPhase::Active,
|
||||
90,
|
||||
90,
|
||||
);
|
||||
|
||||
assert!(
|
||||
pressure_key(&lower_id, 100, &equal_usage) < pressure_key(&higher_id, 100, &equal_usage)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn concurrent_victim_claims_stay_bounded_and_return_to_zero() {
|
||||
let config = ProxyConfig::default();
|
||||
let generation = test_runtime_generation(1, config);
|
||||
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||
let limit = runtime.limits.max_websocket_evictions_in_flight;
|
||||
let entries = (0..limit.saturating_mul(2))
|
||||
.map(|index| {
|
||||
Arc::new(entry(
|
||||
index as u64 + 1,
|
||||
[index as u8; 32],
|
||||
index as u64 + 1,
|
||||
"192.0.2.10",
|
||||
WebSocketKind::Lane(index as u32 + 1),
|
||||
WebSocketPhase::Active,
|
||||
1,
|
||||
1,
|
||||
))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let connections = entries
|
||||
.iter()
|
||||
.map(|entry| WebSocketConnection {
|
||||
runtime: Arc::downgrade(&runtime),
|
||||
entry: Arc::clone(entry),
|
||||
slot: None,
|
||||
base_budget: None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
{
|
||||
let mut registry = runtime.websockets.lock();
|
||||
for entry in &entries {
|
||||
registry.claims.insert(entry.claim, entry.id);
|
||||
registry.entries.insert(entry.id, Arc::clone(entry));
|
||||
}
|
||||
}
|
||||
let successes = Arc::new(AtomicUsize::new(0));
|
||||
let mut tasks = Vec::new();
|
||||
for task_id in 0..100usize {
|
||||
let runtime = Arc::clone(&runtime);
|
||||
let entries = entries.clone();
|
||||
let successes = Arc::clone(&successes);
|
||||
tasks.push(tokio::spawn(async move {
|
||||
for attempt in 0..100usize {
|
||||
let entry = &entries[(task_id * 100 + attempt) % entries.len()];
|
||||
{
|
||||
let mut registry = runtime.websockets.lock();
|
||||
if claim_entry(&mut registry, entry, &runtime) {
|
||||
successes.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
for task in tasks {
|
||||
task.await.unwrap();
|
||||
}
|
||||
|
||||
assert_eq!(successes.load(Ordering::Acquire), limit);
|
||||
assert_eq!(runtime.websockets.lock().evictions_in_flight, limit);
|
||||
|
||||
drop(connections);
|
||||
assert_eq!(runtime.websockets.lock().evictions_in_flight, 0);
|
||||
runtime.shutdown().await;
|
||||
generation.stop_sessions().await;
|
||||
generation.stop_background_tasks().await;
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
/// Browser bridge generation for the serialized HTTPS carrier.
|
||||
pub(crate) mod bridge;
|
||||
/// Process lifecycle publication shared with the control plane.
|
||||
pub(crate) mod control;
|
||||
/// Shared binary frame codec and protocol constants.
|
||||
pub(crate) mod frame;
|
||||
/// Plain HTTP ingress and decoy routing behind external TLS termination.
|
||||
|
||||
+32
-3
@@ -24,6 +24,9 @@ mod backend;
|
||||
mod downlink;
|
||||
// Response ownership keeps detached batches charged until the last body clone drops.
|
||||
mod resident;
|
||||
// Read-only control-plane snapshots stay isolated from carrier operations.
|
||||
mod status;
|
||||
pub(crate) use status::WebSessionStatus;
|
||||
// Lane carrier state isolates request sequencing and downlink replay per logical stream.
|
||||
mod lanes;
|
||||
// Lane batch staging transfers queue ownership without escaping process budgets.
|
||||
@@ -62,6 +65,22 @@ pub(crate) struct StreamIdentity {
|
||||
pub(crate) instance: u64,
|
||||
}
|
||||
|
||||
/// Exact server-local identity of one carrier-lane incarnation.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct CarrierLaneIdentity {
|
||||
/// Numeric lane identifier carried on the wire.
|
||||
pub(crate) lane_id: u32,
|
||||
/// Monotonic server-local incarnation of that numeric lane.
|
||||
pub(crate) instance: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
struct WebSocketLaneClaim {
|
||||
lane: CarrierLaneIdentity,
|
||||
peer_port: u16,
|
||||
connection_id: Option<u64>,
|
||||
}
|
||||
|
||||
struct StreamState {
|
||||
instance: u64,
|
||||
inbound: VecDeque<InboundChunk>,
|
||||
@@ -144,7 +163,7 @@ struct SessionState {
|
||||
carrier_lanes: HashMap<u32, CarrierLane>,
|
||||
lane_open_waits: usize,
|
||||
next_lane_instance: u64,
|
||||
websocket_lane_reservations: HashMap<u32, u16>,
|
||||
websocket_lane_reservations: HashMap<u32, WebSocketLaneClaim>,
|
||||
pending_bytes: usize,
|
||||
pending_items: usize,
|
||||
pending_control_bytes: usize,
|
||||
@@ -189,6 +208,7 @@ pub(crate) struct WebSession {
|
||||
carrier_class: CarrierClientClass,
|
||||
learning_context: Option<CarrierLearningContext>,
|
||||
automatic_carrier: bool,
|
||||
created_at: Instant,
|
||||
limits: WebLimitsConfig,
|
||||
timeouts: WebTimeoutsConfig,
|
||||
state: Mutex<SessionState>,
|
||||
@@ -252,6 +272,7 @@ impl WebSession {
|
||||
carrier_class,
|
||||
learning_context,
|
||||
automatic_carrier,
|
||||
created_at: Instant::now(),
|
||||
limits,
|
||||
timeouts,
|
||||
state: Mutex::new(SessionState {
|
||||
@@ -328,6 +349,11 @@ impl WebSession {
|
||||
self.trace_session_id
|
||||
}
|
||||
|
||||
/// Returns the immutable carrier-attempt incarnation number.
|
||||
pub(crate) fn carrier_attempt(&self) -> u8 {
|
||||
self.carrier_attempt
|
||||
}
|
||||
|
||||
/// Creates a child cancellation boundary for one owned carrier task.
|
||||
pub(crate) fn carrier_cancellation(&self) -> CancellationToken {
|
||||
self.cancel.child_token()
|
||||
@@ -506,11 +532,14 @@ fn remember_closed(state: &mut SessionState, stream_id: u32, limit: usize) -> Op
|
||||
evicted
|
||||
}
|
||||
|
||||
fn insert_carrier_lane(state: &mut SessionState, lane_id: u32) -> Option<u64> {
|
||||
fn insert_carrier_lane(state: &mut SessionState, lane_id: u32) -> Option<CarrierLaneIdentity> {
|
||||
if state.carrier_lanes.contains_key(&lane_id) {
|
||||
return None;
|
||||
}
|
||||
let instance = state.next_lane_instance;
|
||||
state.next_lane_instance = instance.checked_add(1)?;
|
||||
state
|
||||
.carrier_lanes
|
||||
.insert(lane_id, CarrierLane::new(instance));
|
||||
Some(instance)
|
||||
Some(CarrierLaneIdentity { lane_id, instance })
|
||||
}
|
||||
|
||||
+12
-14
@@ -42,26 +42,13 @@ impl WebSession {
|
||||
drop(completion);
|
||||
return false;
|
||||
}
|
||||
let Ok(connection_permit) = generation.max_connections.clone().try_acquire_owned() else {
|
||||
manager.record_stream_rejected();
|
||||
self.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamRejected,
|
||||
Some(stream.id),
|
||||
Some("connection_limit"),
|
||||
);
|
||||
completion
|
||||
.retain_rejected
|
||||
.store(retain_reservation_on_reject, Ordering::Release);
|
||||
drop(completion);
|
||||
return false;
|
||||
};
|
||||
let connection_permits = Arc::clone(&generation.max_connections);
|
||||
let deps = generation.client_runtime_deps();
|
||||
let replay_checker = Arc::clone(&generation.replay_checker);
|
||||
let session = Arc::clone(self);
|
||||
let cancel = self.cancel.clone();
|
||||
let retain_rejected = Arc::clone(&completion.retain_rejected);
|
||||
let future = async move {
|
||||
let _connection_permit = connection_permit;
|
||||
let _completion = completion;
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamAdmitted,
|
||||
@@ -77,6 +64,7 @@ impl WebSession {
|
||||
logical_stream,
|
||||
deps,
|
||||
replay_checker,
|
||||
connection_permits,
|
||||
peer_port,
|
||||
) => {}
|
||||
}
|
||||
@@ -219,6 +207,7 @@ async fn run_stream(
|
||||
stream: WebLogicalStream,
|
||||
deps: crate::proxy::authenticated::ClientRuntimeDeps,
|
||||
replay_checker: Arc<crate::stats::ReplayChecker>,
|
||||
connection_permits: Arc<tokio::sync::Semaphore>,
|
||||
peer_port: u16,
|
||||
) {
|
||||
use tokio::io::AsyncReadExt;
|
||||
@@ -267,6 +256,15 @@ async fn run_stream(
|
||||
let Some(manager) = session.manager.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let Ok(_connection_permit) = connection_permits.try_acquire_owned() else {
|
||||
manager.record_stream_rejected();
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamRejected,
|
||||
Some(stream_identity.id),
|
||||
Some("connection_limit_after_first_byte"),
|
||||
);
|
||||
return;
|
||||
};
|
||||
let Some(handshake_permit) = manager.try_stream_handshake() else {
|
||||
session.trace_lifecycle(
|
||||
crate::web::trace::TraceLifecycleEvent::StreamRejected,
|
||||
|
||||
@@ -206,6 +206,58 @@ async fn open_without_data_does_not_start_the_inner_handshake_timeout() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn silent_open_does_not_consume_generation_connection_capacity() {
|
||||
let runtime = test_runtime(WebCarrier::Https, 1);
|
||||
let available = runtime.generation.max_connections.available_permits();
|
||||
|
||||
assert_eq!(runtime.process_frame(1, 1, FrameType::Open, &[]), Ok(1));
|
||||
settle_tasks().await;
|
||||
|
||||
assert_eq!(
|
||||
runtime.generation.max_connections.available_permits(),
|
||||
available
|
||||
);
|
||||
|
||||
assert_eq!(runtime.process_frame(1, 2, FrameType::Data, &[0x5a]), Ok(2));
|
||||
settle_tasks().await;
|
||||
assert_eq!(
|
||||
runtime.generation.max_connections.available_permits(),
|
||||
available - 1
|
||||
);
|
||||
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn first_inner_byte_fails_closed_when_connection_capacity_is_exhausted() {
|
||||
let runtime = test_runtime(WebCarrier::Https, 1);
|
||||
let capacity = runtime.generation.max_connections.available_permits();
|
||||
let permit = runtime
|
||||
.generation
|
||||
.max_connections
|
||||
.clone()
|
||||
.try_acquire_many_owned(capacity as u32)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(runtime.process_frame(1, 1, FrameType::Open, &[]), Ok(1));
|
||||
settle_tasks().await;
|
||||
assert!(runtime.session.state.lock().streams.contains_key(&1));
|
||||
|
||||
assert_eq!(runtime.process_frame(1, 2, FrameType::Data, &[0x5a]), Ok(2));
|
||||
settle_tasks().await;
|
||||
|
||||
assert!(!runtime.session.state.lock().streams.contains_key(&1));
|
||||
assert_eq!(runtime.generation.max_connections.available_permits(), 0);
|
||||
|
||||
drop(permit);
|
||||
assert_eq!(
|
||||
runtime.generation.max_connections.available_permits(),
|
||||
capacity
|
||||
);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn the_first_inner_byte_starts_the_handshake_timeout() {
|
||||
let runtime = test_runtime(WebCarrier::Https, 1);
|
||||
|
||||
@@ -29,8 +29,10 @@ fn session() -> (Arc<WebSession>, Arc<WebProcessRuntime>) {
|
||||
max_streams: 1,
|
||||
max_streams_per_session: 1,
|
||||
});
|
||||
let mut timeouts = WebTimeoutsConfig::default();
|
||||
timeouts.long_poll_secs = 1;
|
||||
let timeouts = WebTimeoutsConfig {
|
||||
long_poll_secs: 1,
|
||||
..WebTimeoutsConfig::default()
|
||||
};
|
||||
let session = WebSession::new(
|
||||
Arc::downgrade(&manager),
|
||||
[1; 32],
|
||||
|
||||
@@ -6,8 +6,8 @@ use tokio::sync::OwnedSemaphorePermit;
|
||||
|
||||
use super::lane_downlink::take_lane_down_batch;
|
||||
use super::{
|
||||
PendingClass, PollResult, QUEUE_ITEM_COST, QueuedFrame, SessionState, WebSession,
|
||||
remember_closed,
|
||||
CarrierLaneIdentity, PendingClass, PollResult, QUEUE_ITEM_COST, QueuedFrame, SessionState,
|
||||
WebSession, remember_closed,
|
||||
};
|
||||
use crate::web::frame::{self, FrameType};
|
||||
use crate::web::manager::ManagerError;
|
||||
@@ -18,15 +18,46 @@ impl WebSession {
|
||||
&self,
|
||||
lane_id: u32,
|
||||
cursor: u64,
|
||||
) -> Result<PollResult, ManagerError> {
|
||||
self.poll_down_lane_inner(lane_id, None, cursor).await
|
||||
}
|
||||
|
||||
/// Polls only the exact lane incarnation owned by one WebSocket driver.
|
||||
pub(crate) async fn poll_down_websocket_lane(
|
||||
&self,
|
||||
lane: CarrierLaneIdentity,
|
||||
cursor: u64,
|
||||
) -> Result<PollResult, ManagerError> {
|
||||
self.poll_down_lane_inner(lane.lane_id, Some(lane.instance), cursor)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn poll_down_lane_inner(
|
||||
&self,
|
||||
lane_id: u32,
|
||||
expected_instance: Option<u64>,
|
||||
cursor: u64,
|
||||
) -> Result<PollResult, ManagerError> {
|
||||
if !self.carrier().uses_lanes() || lane_id > frame::MAX_STREAM_ID {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
if !self.wait_for_lane_open(lane_id, cursor).await? {
|
||||
let lane_ready = if let Some(expected_instance) = expected_instance {
|
||||
let state = self.state.lock();
|
||||
if state.closed {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
state
|
||||
.carrier_lanes
|
||||
.get(&lane_id)
|
||||
.is_some_and(|lane| lane.instance == expected_instance)
|
||||
} else {
|
||||
self.wait_for_lane_open(lane_id, cursor).await?
|
||||
};
|
||||
if !lane_ready {
|
||||
return Ok(PollResult {
|
||||
body: Bytes::new(),
|
||||
next_cursor: cursor,
|
||||
lane_closed: false,
|
||||
lane_closed: expected_instance.is_some(),
|
||||
});
|
||||
}
|
||||
let (instance, epoch, notify, healthy) = {
|
||||
@@ -43,6 +74,13 @@ impl WebSession {
|
||||
lane_closed: true,
|
||||
});
|
||||
};
|
||||
if expected_instance.is_some_and(|instance| lane.instance != instance) {
|
||||
return Ok(PollResult {
|
||||
body: Bytes::new(),
|
||||
next_cursor: cursor,
|
||||
lane_closed: true,
|
||||
});
|
||||
}
|
||||
if let Some(unacked) = &lane.unacked {
|
||||
if cursor == unacked.base_cursor {
|
||||
return Ok(PollResult {
|
||||
|
||||
@@ -84,6 +84,11 @@ impl WebSession {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the current number of registered logical-stream tasks.
|
||||
pub(crate) fn tasks_live(&self) -> usize {
|
||||
self.tasks_live.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
/// Atomically closes a session only when reconnect grace is still due.
|
||||
pub(crate) fn close_if_due(&self, now: Instant) -> bool {
|
||||
let healthy = {
|
||||
@@ -206,6 +211,7 @@ impl WebSession {
|
||||
self.client_ip,
|
||||
self.profile_key,
|
||||
&self.profile.host,
|
||||
Duration::from_secs(self.timeouts.bootstrap_lifetime_secs),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
use std::time::Instant;
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
use super::{SessionNegotiationPhase, WebSession};
|
||||
use crate::config::WebCarrier;
|
||||
|
||||
/// One bounded point-in-time session snapshot without bearer identity.
|
||||
#[derive(Clone, Serialize)]
|
||||
pub(crate) struct WebSessionStatus {
|
||||
/// Stable trace identifier within this process.
|
||||
pub(crate) trace_session_id: u64,
|
||||
/// Forwarded client address frozen at session creation.
|
||||
pub(crate) client_ip: std::net::IpAddr,
|
||||
/// Canonical WEB virtual host.
|
||||
pub(crate) host: String,
|
||||
/// Configured non-secret user label.
|
||||
pub(crate) user: String,
|
||||
/// Non-secret configured key fingerprint.
|
||||
pub(crate) key_id: String,
|
||||
/// Current carrier incarnation.
|
||||
pub(crate) carrier: WebCarrier,
|
||||
/// One-based carrier attempt.
|
||||
pub(crate) attempt: u8,
|
||||
/// Stable client classification token.
|
||||
pub(crate) client_class: &'static str,
|
||||
/// Whether server-side carrier negotiation owns this chain.
|
||||
pub(crate) automatic: bool,
|
||||
/// Current session lifecycle token.
|
||||
pub(crate) state: &'static str,
|
||||
/// Live logical streams.
|
||||
pub(crate) streams: usize,
|
||||
/// Stream relay tasks that have not exited.
|
||||
pub(crate) tasks: usize,
|
||||
/// Carrier lanes currently retained.
|
||||
pub(crate) lanes: usize,
|
||||
/// Lane OPEN polls currently waiting.
|
||||
pub(crate) lane_open_waits: usize,
|
||||
/// WebSocket lane slots reserved before ownership transfer.
|
||||
pub(crate) websocket_lane_reservations: usize,
|
||||
/// Whether the multiplexed WebSocket carrier is active.
|
||||
pub(crate) websocket_active: bool,
|
||||
/// Queued and response-resident bytes charged to this session.
|
||||
pub(crate) pending_bytes: usize,
|
||||
/// Queued and response-resident items charged to this session.
|
||||
pub(crate) pending_items: usize,
|
||||
/// Control bytes included in the pending total.
|
||||
pub(crate) control_bytes: usize,
|
||||
/// Control items included in the pending total.
|
||||
pub(crate) control_items: usize,
|
||||
/// Monotonic age since session creation.
|
||||
pub(crate) age_ms: u64,
|
||||
/// Monotonic age since the latest carrier activity.
|
||||
pub(crate) idle_ms: u64,
|
||||
/// Remaining automatic negotiation deadline.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) negotiation_remaining_ms: Option<u64>,
|
||||
}
|
||||
|
||||
impl WebSession {
|
||||
/// Captures one short-lock session snapshot or reports lock contention.
|
||||
pub(crate) fn try_status(&self, now: Instant) -> Option<WebSessionStatus> {
|
||||
let state = self.state.try_lock()?;
|
||||
let resident = self.resident.snapshot();
|
||||
let state_name = if state.closed {
|
||||
"closed"
|
||||
} else if state.close_requested {
|
||||
"closing"
|
||||
} else if state.carrier_health_reported {
|
||||
"healthy"
|
||||
} else {
|
||||
match state.negotiation_phase {
|
||||
SessionNegotiationPhase::Uncommitted => "provisional",
|
||||
SessionNegotiationPhase::Replacing => "replacing",
|
||||
SessionNegotiationPhase::Committed => "committed",
|
||||
SessionNegotiationPhase::Superseded => "superseded",
|
||||
}
|
||||
};
|
||||
Some(WebSessionStatus {
|
||||
trace_session_id: self.trace_session_id,
|
||||
client_ip: self.client_ip,
|
||||
host: self.profile.host.clone(),
|
||||
user: self.profile.user.clone(),
|
||||
key_id: self.profile.key_fingerprint.clone(),
|
||||
carrier: self.selected_carrier,
|
||||
attempt: self.carrier_attempt,
|
||||
client_class: self.carrier_class.as_str(),
|
||||
automatic: self.automatic_carrier,
|
||||
state: state_name,
|
||||
streams: state.streams.len(),
|
||||
tasks: self.tasks_live(),
|
||||
lanes: state.carrier_lanes.len(),
|
||||
lane_open_waits: state.lane_open_waits,
|
||||
websocket_lane_reservations: state.websocket_lane_reservations.len(),
|
||||
websocket_active: state.websocket_carrier_active,
|
||||
pending_bytes: state.pending_bytes.saturating_add(resident.bytes()),
|
||||
pending_items: state.pending_items.saturating_add(resident.items()),
|
||||
control_bytes: state
|
||||
.pending_control_bytes
|
||||
.saturating_add(resident.control_bytes),
|
||||
control_items: state
|
||||
.pending_control_items
|
||||
.saturating_add(resident.control_items),
|
||||
age_ms: millis(now.saturating_duration_since(self.created_at)),
|
||||
idle_ms: millis(now.saturating_duration_since(state.last_activity)),
|
||||
negotiation_remaining_ms: self
|
||||
.carrier_deadline_at
|
||||
.map(|deadline| millis(deadline.saturating_duration_since(now))),
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the forwarded client address frozen at session creation.
|
||||
pub(crate) fn client_ip(&self) -> std::net::IpAddr {
|
||||
self.client_ip
|
||||
}
|
||||
|
||||
/// Returns the canonical profile host frozen into this session.
|
||||
pub(crate) fn profile_host(&self) -> &str {
|
||||
&self.profile.host
|
||||
}
|
||||
|
||||
/// Returns the configured non-secret user label.
|
||||
pub(crate) fn profile_user(&self) -> &str {
|
||||
&self.profile.user
|
||||
}
|
||||
|
||||
/// Returns the configured non-secret key fingerprint.
|
||||
pub(crate) fn key_id(&self) -> &str {
|
||||
&self.profile.key_fingerprint
|
||||
}
|
||||
}
|
||||
|
||||
fn millis(duration: std::time::Duration) -> u64 {
|
||||
duration.as_millis().min(u128::from(u64::MAX)) as u64
|
||||
}
|
||||
@@ -166,6 +166,8 @@ impl WebSession {
|
||||
result
|
||||
}
|
||||
|
||||
// Batch application keeps every transactional accumulator explicit.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) fn apply_batch_locked(
|
||||
self: &Arc<Self>,
|
||||
state: &mut SessionState,
|
||||
|
||||
+226
-65
@@ -4,7 +4,10 @@ use std::time::Instant;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::uplink::{AppliedProgress, inbound_reservation, validate_batch};
|
||||
use super::{PendingClass, WebSession, inbound_queue_cost, insert_carrier_lane};
|
||||
use super::{
|
||||
CarrierLaneIdentity, PendingClass, StreamIdentity, WebSession, WebSocketLaneClaim,
|
||||
inbound_queue_cost, insert_carrier_lane,
|
||||
};
|
||||
use crate::config::WebCarrier;
|
||||
use crate::web::frame;
|
||||
use crate::web::manager::ManagerError;
|
||||
@@ -12,9 +15,19 @@ use crate::web::manager::ManagerError;
|
||||
/// Pre-OPEN stream quota and synthetic tuple ownership for one WebSocket lane.
|
||||
pub(crate) struct WebSocketLaneReservation {
|
||||
session: Arc<WebSession>,
|
||||
lane_id: u32,
|
||||
peer_port: u16,
|
||||
transferred: bool,
|
||||
claim: WebSocketLaneClaim,
|
||||
stream: Option<StreamIdentity>,
|
||||
phase: WebSocketLaneReservationPhase,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum WebSocketLaneReservationPhase {
|
||||
Reserved,
|
||||
Bound,
|
||||
Transferred,
|
||||
StreamOwned,
|
||||
Closing,
|
||||
Released,
|
||||
}
|
||||
|
||||
/// Session-wide ownership of the only automatic WebSocket carrier probe.
|
||||
@@ -57,28 +70,99 @@ impl Drop for WebSocketProbeReservation {
|
||||
impl WebSocketLaneReservation {
|
||||
/// Returns the logical stream owned by this connection.
|
||||
pub(crate) fn lane_id(&self) -> u32 {
|
||||
self.lane_id
|
||||
self.claim.lane.lane_id
|
||||
}
|
||||
|
||||
fn transfer_to_stream(&mut self) {
|
||||
let removed = self
|
||||
.session
|
||||
.state
|
||||
.lock()
|
||||
.websocket_lane_reservations
|
||||
.remove(&self.lane_id);
|
||||
if removed == Some(self.peer_port) {
|
||||
self.transferred = true;
|
||||
/// Returns the exact lane incarnation owned by this connection.
|
||||
pub(crate) fn lane_identity(&self) -> CarrierLaneIdentity {
|
||||
self.claim.lane
|
||||
}
|
||||
|
||||
/// Binds this pre-upgrade reservation to one admitted process connection.
|
||||
pub(crate) fn bind(&mut self, connection_id: u64) -> Result<(), ManagerError> {
|
||||
if self.phase != WebSocketLaneReservationPhase::Reserved {
|
||||
return Err(ManagerError::Concurrent);
|
||||
}
|
||||
let mut state = self.session.state.lock();
|
||||
if state.closed
|
||||
|| state
|
||||
.carrier_lanes
|
||||
.get(&self.claim.lane.lane_id)
|
||||
.is_none_or(|lane| lane.instance != self.claim.lane.instance)
|
||||
{
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
let Some(current) = state
|
||||
.websocket_lane_reservations
|
||||
.get_mut(&self.claim.lane.lane_id)
|
||||
.filter(|current| **current == self.claim && current.connection_id.is_none())
|
||||
else {
|
||||
return Err(ManagerError::Closed);
|
||||
};
|
||||
current.connection_id = Some(connection_id);
|
||||
self.claim.connection_id = Some(connection_id);
|
||||
self.phase = WebSocketLaneReservationPhase::Bound;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn transfer_to_stream(&mut self, stream: StreamIdentity) -> Result<(), ManagerError> {
|
||||
if self.phase != WebSocketLaneReservationPhase::Bound
|
||||
|| stream.id != self.claim.lane.lane_id
|
||||
{
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let mut state = self.session.state.lock();
|
||||
if state
|
||||
.carrier_lanes
|
||||
.get(&self.claim.lane.lane_id)
|
||||
.is_none_or(|lane| lane.instance != self.claim.lane.instance)
|
||||
|| state
|
||||
.streams
|
||||
.get(&stream.id)
|
||||
.is_none_or(|current| current.instance != stream.instance)
|
||||
|| state
|
||||
.websocket_lane_reservations
|
||||
.get(&self.claim.lane.lane_id)
|
||||
!= Some(&self.claim)
|
||||
{
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
state
|
||||
.websocket_lane_reservations
|
||||
.remove(&self.claim.lane.lane_id);
|
||||
self.stream = Some(stream);
|
||||
self.phase = WebSocketLaneReservationPhase::Transferred;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn mark_stream_owned(&mut self, stream: StreamIdentity) -> Result<(), ManagerError> {
|
||||
if self.phase != WebSocketLaneReservationPhase::Transferred || self.stream != Some(stream) {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
self.phase = WebSocketLaneReservationPhase::StreamOwned;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn retain_after_rejected_spawn(&mut self) {
|
||||
debug_assert_eq!(self.phase, WebSocketLaneReservationPhase::StreamOwned);
|
||||
self.phase = WebSocketLaneReservationPhase::Transferred;
|
||||
}
|
||||
|
||||
fn release(&mut self) {
|
||||
if self.phase == WebSocketLaneReservationPhase::Released {
|
||||
return;
|
||||
}
|
||||
let stream_owned = self.phase == WebSocketLaneReservationPhase::StreamOwned;
|
||||
self.phase = WebSocketLaneReservationPhase::Closing;
|
||||
self.session
|
||||
.release_websocket_lane_claim(self.claim, self.stream, stream_owned);
|
||||
self.phase = WebSocketLaneReservationPhase::Released;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WebSocketLaneReservation {
|
||||
fn drop(&mut self) {
|
||||
if !self.transferred {
|
||||
self.session
|
||||
.release_websocket_lane_reservation(self.lane_id, self.peer_port);
|
||||
}
|
||||
self.release();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,9 +246,7 @@ impl WebSession {
|
||||
);
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
state.websocket_lane_reservations.insert(lane_id, peer_port);
|
||||
if insert_carrier_lane(&mut state, lane_id).is_none() {
|
||||
state.websocket_lane_reservations.remove(&lane_id);
|
||||
let Some(lane) = insert_carrier_lane(&mut state, lane_id) else {
|
||||
state.active_peer_ports.remove(&peer_port);
|
||||
manager.release_stream(
|
||||
self.profile_key,
|
||||
@@ -173,14 +255,37 @@ impl WebSession {
|
||||
peer_port,
|
||||
);
|
||||
return Err(ManagerError::Protocol);
|
||||
};
|
||||
let claim = WebSocketLaneClaim {
|
||||
lane,
|
||||
peer_port,
|
||||
connection_id: None,
|
||||
};
|
||||
let inserted = match state.websocket_lane_reservations.entry(lane_id) {
|
||||
std::collections::hash_map::Entry::Vacant(entry) => {
|
||||
entry.insert(claim);
|
||||
true
|
||||
}
|
||||
std::collections::hash_map::Entry::Occupied(_) => false,
|
||||
};
|
||||
if !inserted {
|
||||
self.release_lane_locked(&mut state, lane_id);
|
||||
state.active_peer_ports.remove(&peer_port);
|
||||
manager.release_stream(
|
||||
self.profile_key,
|
||||
self.client_ip,
|
||||
self.profile.public_addr,
|
||||
peer_port,
|
||||
);
|
||||
return Err(ManagerError::Concurrent);
|
||||
}
|
||||
drop(state);
|
||||
self.lane_open_notify.notify_waiters();
|
||||
Ok(WebSocketLaneReservation {
|
||||
session: Arc::clone(self),
|
||||
lane_id,
|
||||
peer_port,
|
||||
transferred: false,
|
||||
claim,
|
||||
stream: None,
|
||||
phase: WebSocketLaneReservationPhase::Reserved,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -192,12 +297,16 @@ impl WebSession {
|
||||
body: &[u8],
|
||||
) -> Result<bool, ManagerError> {
|
||||
if !Arc::ptr_eq(self, &reservation.session)
|
||||
|| reservation.lane_id == 0
|
||||
|| reservation.lane_id > frame::MAX_STREAM_ID
|
||||
|| reservation.lane_id() == 0
|
||||
|| reservation.lane_id() > frame::MAX_STREAM_ID
|
||||
|| !matches!(
|
||||
reservation.phase,
|
||||
WebSocketLaneReservationPhase::Bound | WebSocketLaneReservationPhase::StreamOwned
|
||||
)
|
||||
{
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
let lane_id = reservation.lane_id;
|
||||
let lane_id = reservation.lane_id();
|
||||
let frames = frame::parse_all(body, &self.limits).map_err(|_| ManagerError::Protocol)?;
|
||||
if frames
|
||||
.iter()
|
||||
@@ -216,8 +325,19 @@ impl WebSession {
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
self.ensure_carrier_active_locked(&state)?;
|
||||
if !reservation.transferred
|
||||
&& state.websocket_lane_reservations.get(&lane_id) != Some(&reservation.peer_port)
|
||||
if state
|
||||
.carrier_lanes
|
||||
.get(&lane_id)
|
||||
.is_none_or(|lane| lane.instance != reservation.claim.lane.instance)
|
||||
|| (reservation.phase == WebSocketLaneReservationPhase::Bound
|
||||
&& state.websocket_lane_reservations.get(&lane_id) != Some(&reservation.claim))
|
||||
|| (reservation.phase == WebSocketLaneReservationPhase::StreamOwned
|
||||
&& reservation.stream.is_none_or(|stream| {
|
||||
state
|
||||
.streams
|
||||
.get(&stream.id)
|
||||
.is_none_or(|current| current.instance != stream.instance)
|
||||
}))
|
||||
{
|
||||
return Err(ManagerError::Closed);
|
||||
}
|
||||
@@ -252,8 +372,8 @@ impl WebSession {
|
||||
let mut unused_bytes = reserve_bytes;
|
||||
let mut unused_items = reserve_items;
|
||||
let mut progress = AppliedProgress::default();
|
||||
let mut reserved_open =
|
||||
(!reservation.transferred).then_some((lane_id, reservation.peer_port));
|
||||
let mut reserved_open = (reservation.phase == WebSocketLaneReservationPhase::Bound)
|
||||
.then_some((lane_id, reservation.claim.peer_port));
|
||||
let applied = self.apply_batch_locked(
|
||||
&mut state,
|
||||
&frames,
|
||||
@@ -287,15 +407,18 @@ impl WebSession {
|
||||
self.finish_carrier_health();
|
||||
}
|
||||
for completion in opened {
|
||||
if completion.stream.id != lane_id || completion.peer_port != reservation.peer_port {
|
||||
let stream = completion.stream;
|
||||
if stream.id != lane_id || completion.peer_port != reservation.claim.peer_port {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
reservation.transfer_to_stream(stream)?;
|
||||
reservation.mark_stream_owned(stream)?;
|
||||
if !self.spawn_stream(completion, true) {
|
||||
reservation.retain_after_rejected_spawn();
|
||||
return Err(ManagerError::Limit);
|
||||
}
|
||||
reservation.transfer_to_stream();
|
||||
}
|
||||
if !reservation.transferred {
|
||||
if reservation.phase != WebSocketLaneReservationPhase::StreamOwned {
|
||||
return Err(ManagerError::Protocol);
|
||||
}
|
||||
if let Some(manager) = self.manager.upgrade() {
|
||||
@@ -304,49 +427,87 @@ impl WebSession {
|
||||
Ok(progressed)
|
||||
}
|
||||
|
||||
/// Ends one failed or disconnected lane without closing its parent session.
|
||||
pub(crate) fn close_websocket_lane(&self, lane_id: u32) {
|
||||
let reserved = {
|
||||
let mut state = self.state.lock();
|
||||
let reserved = state.websocket_lane_reservations.remove(&lane_id);
|
||||
if let Some(stream) = state.streams.remove(&lane_id) {
|
||||
state.closing_streams.insert(lane_id, stream.instance);
|
||||
let (bytes, items) = inbound_queue_cost(&stream.inbound);
|
||||
self.release_locked(&mut state, bytes, items, false);
|
||||
if let Some(waker) = stream.read_waker {
|
||||
waker.wake();
|
||||
}
|
||||
if let Some(waker) = stream.write_waker {
|
||||
waker.wake();
|
||||
}
|
||||
}
|
||||
self.remember_closed_locked(&mut state, lane_id);
|
||||
self.release_lane_locked(&mut state, lane_id);
|
||||
reserved
|
||||
};
|
||||
if let Some(peer_port) = reserved {
|
||||
self.release_websocket_lane_reservation(lane_id, peer_port);
|
||||
/// Ends one exact failed or disconnected lane without closing its parent session.
|
||||
pub(crate) fn close_websocket_lane(&self, mut reservation: WebSocketLaneReservation) {
|
||||
if std::ptr::eq(self, Arc::as_ptr(&reservation.session)) {
|
||||
reservation.release();
|
||||
}
|
||||
self.lane_open_notify.notify_waiters();
|
||||
}
|
||||
|
||||
fn release_websocket_lane_reservation(&self, lane_id: u32, peer_port: u16) {
|
||||
let removed = {
|
||||
fn release_websocket_lane_claim(
|
||||
&self,
|
||||
claim: WebSocketLaneClaim,
|
||||
stream: Option<StreamIdentity>,
|
||||
stream_owned: bool,
|
||||
) {
|
||||
let release_port = {
|
||||
let mut state = self.state.lock();
|
||||
if state.websocket_lane_reservations.get(&lane_id) == Some(&peer_port) {
|
||||
state.websocket_lane_reservations.remove(&lane_id);
|
||||
let lane_matches = state
|
||||
.carrier_lanes
|
||||
.get(&claim.lane.lane_id)
|
||||
.is_some_and(|lane| lane.instance == claim.lane.instance);
|
||||
if !lane_matches && !state.closed {
|
||||
return;
|
||||
}
|
||||
self.release_lane_locked(&mut state, lane_id);
|
||||
state.active_peer_ports.remove(&peer_port)
|
||||
let release_port = if let Some(stream) = stream {
|
||||
if stream.id != claim.lane.lane_id {
|
||||
return;
|
||||
}
|
||||
let current_stream = state
|
||||
.streams
|
||||
.get(&stream.id)
|
||||
.is_some_and(|current| current.instance == stream.instance);
|
||||
if current_stream {
|
||||
let Some(stream_state) = state.streams.remove(&stream.id) else {
|
||||
return;
|
||||
};
|
||||
state
|
||||
.closing_streams
|
||||
.insert(claim.lane.lane_id, stream.instance);
|
||||
let (bytes, items) = inbound_queue_cost(&stream_state.inbound);
|
||||
self.release_locked(&mut state, bytes, items, false);
|
||||
if let Some(waker) = stream_state.read_waker {
|
||||
waker.wake();
|
||||
}
|
||||
if let Some(waker) = stream_state.write_waker {
|
||||
waker.wake();
|
||||
}
|
||||
false
|
||||
} else if stream_owned {
|
||||
false
|
||||
} else {
|
||||
state.active_peer_ports.remove(&claim.peer_port)
|
||||
}
|
||||
} else {
|
||||
if state.websocket_lane_reservations.get(&claim.lane.lane_id) != Some(&claim) {
|
||||
return;
|
||||
}
|
||||
state
|
||||
.websocket_lane_reservations
|
||||
.remove(&claim.lane.lane_id);
|
||||
state.active_peer_ports.remove(&claim.peer_port)
|
||||
};
|
||||
if lane_matches {
|
||||
self.remember_closed_locked(&mut state, claim.lane.lane_id);
|
||||
if state
|
||||
.carrier_lanes
|
||||
.get(&claim.lane.lane_id)
|
||||
.is_some_and(|lane| lane.instance == claim.lane.instance)
|
||||
{
|
||||
self.release_lane_locked(&mut state, claim.lane.lane_id);
|
||||
}
|
||||
}
|
||||
release_port
|
||||
};
|
||||
if removed && let Some(manager) = self.manager.upgrade() {
|
||||
if release_port && let Some(manager) = self.manager.upgrade() {
|
||||
manager.release_stream(
|
||||
self.profile_key,
|
||||
self.client_ip,
|
||||
self.profile.public_addr,
|
||||
peer_port,
|
||||
claim.peer_port,
|
||||
);
|
||||
}
|
||||
self.lane_open_notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -81,10 +81,48 @@ fn runtime(admission: bool) -> TestRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
fn detach_stale_lane(runtime: &TestRuntime, reservation: &WebSocketLaneReservation) {
|
||||
let claim = reservation.claim;
|
||||
{
|
||||
let mut state = runtime.session.state.lock();
|
||||
assert_eq!(
|
||||
state
|
||||
.websocket_lane_reservations
|
||||
.remove(&claim.lane.lane_id),
|
||||
Some(claim)
|
||||
);
|
||||
runtime
|
||||
.session
|
||||
.release_lane_locked(&mut state, claim.lane.lane_id);
|
||||
assert!(state.active_peer_ports.remove(&claim.peer_port));
|
||||
}
|
||||
runtime.manager.release_stream(
|
||||
runtime.session.profile_key,
|
||||
runtime.session.client_ip,
|
||||
runtime.session.profile.public_addr,
|
||||
claim.peer_port,
|
||||
);
|
||||
}
|
||||
|
||||
fn replacement_lane(
|
||||
runtime: &TestRuntime,
|
||||
stale: &WebSocketLaneReservation,
|
||||
) -> WebSocketLaneReservation {
|
||||
detach_stale_lane(runtime, stale);
|
||||
let mut replacement = runtime
|
||||
.session
|
||||
.reserve_websocket_lane(stale.lane_id())
|
||||
.unwrap();
|
||||
replacement.bind(2).unwrap();
|
||||
assert_ne!(replacement.lane_identity(), stale.lane_identity());
|
||||
replacement
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejected_open_retains_stream_quota_until_lane_socket_teardown() {
|
||||
let runtime = runtime(false);
|
||||
let mut reservation = runtime.session.reserve_websocket_lane(7).unwrap();
|
||||
reservation.bind(1).unwrap();
|
||||
let open = frame::encode(FrameType::Open, 7, &[]);
|
||||
|
||||
assert_eq!(
|
||||
@@ -93,6 +131,19 @@ async fn rejected_open_retains_stream_quota_until_lane_socket_teardown() {
|
||||
.process_websocket_lane(&mut reservation, 1, &open),
|
||||
Err(ManagerError::Limit),
|
||||
);
|
||||
assert_eq!(
|
||||
reservation.phase,
|
||||
WebSocketLaneReservationPhase::Transferred
|
||||
);
|
||||
assert!(reservation.stream.is_some());
|
||||
assert!(
|
||||
!runtime
|
||||
.session
|
||||
.state
|
||||
.lock()
|
||||
.websocket_lane_reservations
|
||||
.contains_key(&7)
|
||||
);
|
||||
assert!(
|
||||
runtime
|
||||
.manager
|
||||
@@ -105,8 +156,124 @@ async fn rejected_open_retains_stream_quota_until_lane_socket_teardown() {
|
||||
.is_none()
|
||||
);
|
||||
|
||||
runtime.session.close_websocket_lane(7);
|
||||
runtime.session.close_websocket_lane(reservation);
|
||||
let peer_port = runtime
|
||||
.manager
|
||||
.try_acquire_stream(
|
||||
runtime.session.profile_key,
|
||||
runtime.session.profile.max_streams,
|
||||
runtime.session.client_ip,
|
||||
runtime.session.profile.public_addr,
|
||||
)
|
||||
.unwrap();
|
||||
runtime.manager.release_stream(
|
||||
runtime.session.profile_key,
|
||||
runtime.session.client_ip,
|
||||
runtime.session.profile.public_addr,
|
||||
peer_port,
|
||||
);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn closed_session_releases_bound_lane_quota_on_reservation_drop() {
|
||||
let runtime = runtime(true);
|
||||
let mut reservation = runtime.session.reserve_websocket_lane(7).unwrap();
|
||||
reservation.bind(1).unwrap();
|
||||
|
||||
runtime.session.close();
|
||||
drop(reservation);
|
||||
|
||||
assert!(runtime.session.state.lock().active_peer_ports.is_empty());
|
||||
let peer_port = runtime
|
||||
.manager
|
||||
.try_acquire_stream(
|
||||
runtime.session.profile_key,
|
||||
runtime.session.profile.max_streams,
|
||||
runtime.session.client_ip,
|
||||
runtime.session.profile.public_addr,
|
||||
)
|
||||
.unwrap();
|
||||
runtime.manager.release_stream(
|
||||
runtime.session.profile_key,
|
||||
runtime.session.client_ip,
|
||||
runtime.session.profile.public_addr,
|
||||
peer_port,
|
||||
);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn closed_session_releases_transferred_rejected_lane_quota() {
|
||||
let runtime = runtime(false);
|
||||
let mut reservation = runtime.session.reserve_websocket_lane(7).unwrap();
|
||||
reservation.bind(1).unwrap();
|
||||
let open = frame::encode(FrameType::Open, 7, &[]);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.session
|
||||
.process_websocket_lane(&mut reservation, 1, &open),
|
||||
Err(ManagerError::Limit),
|
||||
);
|
||||
assert_eq!(
|
||||
reservation.phase,
|
||||
WebSocketLaneReservationPhase::Transferred
|
||||
);
|
||||
|
||||
runtime.session.close();
|
||||
drop(reservation);
|
||||
|
||||
assert!(runtime.session.state.lock().active_peer_ports.is_empty());
|
||||
let peer_port = runtime
|
||||
.manager
|
||||
.try_acquire_stream(
|
||||
runtime.session.profile_key,
|
||||
runtime.session.profile.max_streams,
|
||||
runtime.session.client_ip,
|
||||
runtime.session.profile.public_addr,
|
||||
)
|
||||
.unwrap();
|
||||
runtime.manager.release_stream(
|
||||
runtime.session.profile_key,
|
||||
runtime.session.client_ip,
|
||||
runtime.session.profile.public_addr,
|
||||
peer_port,
|
||||
);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn closed_session_keeps_stream_owned_quota_until_task_completion() {
|
||||
let runtime = runtime(true);
|
||||
let mut reservation = runtime.session.reserve_websocket_lane(7).unwrap();
|
||||
reservation.bind(1).unwrap();
|
||||
let open = frame::encode(FrameType::Open, 7, &[]);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.session
|
||||
.process_websocket_lane(&mut reservation, 1, &open),
|
||||
Ok(true),
|
||||
);
|
||||
assert_eq!(
|
||||
reservation.phase,
|
||||
WebSocketLaneReservationPhase::StreamOwned
|
||||
);
|
||||
|
||||
runtime.session.close();
|
||||
drop(reservation);
|
||||
|
||||
assert!(
|
||||
runtime
|
||||
.manager
|
||||
.try_acquire_stream(
|
||||
runtime.session.profile_key,
|
||||
runtime.session.profile.max_streams,
|
||||
runtime.session.client_ip,
|
||||
runtime.session.profile.public_addr,
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
runtime.session.wait().await;
|
||||
let peer_port = runtime
|
||||
.manager
|
||||
.try_acquire_stream(
|
||||
@@ -129,6 +296,7 @@ async fn rejected_open_retains_stream_quota_until_lane_socket_teardown() {
|
||||
async fn malformed_lane_message_does_not_close_sibling_session_state() {
|
||||
let runtime = runtime(true);
|
||||
let mut reservation = runtime.session.reserve_websocket_lane(7).unwrap();
|
||||
reservation.bind(1).unwrap();
|
||||
let data = frame::encode(FrameType::Data, 7, &[1]);
|
||||
|
||||
assert_eq!(
|
||||
@@ -139,8 +307,122 @@ async fn malformed_lane_message_does_not_close_sibling_session_state() {
|
||||
);
|
||||
assert!(!runtime.session.state.lock().closed);
|
||||
|
||||
runtime.session.close_websocket_lane(7);
|
||||
drop(reservation);
|
||||
runtime.session.close_websocket_lane(reservation);
|
||||
assert!(runtime.session.reserve_websocket_lane(8).is_ok());
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_websocket_poll_does_not_close_reused_lane_instance() {
|
||||
let runtime = runtime(true);
|
||||
let mut stale = runtime.session.reserve_websocket_lane(7).unwrap();
|
||||
stale.bind(1).unwrap();
|
||||
let stale_identity = stale.lane_identity();
|
||||
let replacement = replacement_lane(&runtime, &stale);
|
||||
|
||||
let result = runtime
|
||||
.session
|
||||
.poll_down_websocket_lane(stale_identity, u64::MAX)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(result.lane_closed);
|
||||
assert!(!runtime.session.state.lock().closed);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.session
|
||||
.state
|
||||
.lock()
|
||||
.carrier_lanes
|
||||
.get(&7)
|
||||
.map(|lane| lane.instance),
|
||||
Some(replacement.lane_identity().instance)
|
||||
);
|
||||
drop(stale);
|
||||
runtime.session.close_websocket_lane(replacement);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_close_preserves_replacement_lane_and_tuple() {
|
||||
let runtime = runtime(true);
|
||||
let mut stale = runtime.session.reserve_websocket_lane(7).unwrap();
|
||||
stale.bind(1).unwrap();
|
||||
let replacement = replacement_lane(&runtime, &stale);
|
||||
let replacement_claim = replacement.claim;
|
||||
stale.phase = WebSocketLaneReservationPhase::Transferred;
|
||||
stale.stream = Some(StreamIdentity { id: 7, instance: 1 });
|
||||
|
||||
runtime.session.close_websocket_lane(stale);
|
||||
|
||||
{
|
||||
let state = runtime.session.state.lock();
|
||||
assert_eq!(
|
||||
state.websocket_lane_reservations.get(&7),
|
||||
Some(&replacement_claim)
|
||||
);
|
||||
assert!(
|
||||
state
|
||||
.active_peer_ports
|
||||
.contains(&replacement_claim.peer_port)
|
||||
);
|
||||
assert_eq!(
|
||||
state.carrier_lanes.get(&7).map(|lane| lane.instance),
|
||||
Some(replacement_claim.lane.instance)
|
||||
);
|
||||
}
|
||||
runtime.session.close_websocket_lane(replacement);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_reservation_drop_preserves_replacement_claim() {
|
||||
let runtime = runtime(true);
|
||||
let mut stale = runtime.session.reserve_websocket_lane(7).unwrap();
|
||||
stale.bind(1).unwrap();
|
||||
let replacement = replacement_lane(&runtime, &stale);
|
||||
let replacement_claim = replacement.claim;
|
||||
|
||||
drop(stale);
|
||||
|
||||
{
|
||||
let state = runtime.session.state.lock();
|
||||
assert_eq!(
|
||||
state.websocket_lane_reservations.get(&7),
|
||||
Some(&replacement_claim)
|
||||
);
|
||||
assert!(
|
||||
state
|
||||
.active_peer_ports
|
||||
.contains(&replacement_claim.peer_port)
|
||||
);
|
||||
}
|
||||
runtime.session.close_websocket_lane(replacement);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_transfer_cannot_remove_current_reservation() {
|
||||
let runtime = runtime(true);
|
||||
let mut stale = runtime.session.reserve_websocket_lane(7).unwrap();
|
||||
stale.bind(1).unwrap();
|
||||
let replacement = replacement_lane(&runtime, &stale);
|
||||
let replacement_claim = replacement.claim;
|
||||
|
||||
assert_eq!(
|
||||
stale.transfer_to_stream(StreamIdentity { id: 7, instance: 1 }),
|
||||
Err(ManagerError::Closed)
|
||||
);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.session
|
||||
.state
|
||||
.lock()
|
||||
.websocket_lane_reservations
|
||||
.get(&7),
|
||||
Some(&replacement_claim)
|
||||
);
|
||||
drop(stale);
|
||||
runtime.session.close_websocket_lane(replacement);
|
||||
runtime.shutdown().await;
|
||||
}
|
||||
|
||||
@@ -10,7 +10,9 @@ mod store;
|
||||
mod types;
|
||||
|
||||
pub(crate) use exchange::HttpTraceExchange;
|
||||
pub(crate) use store::{StoredTraceRecord, WebTraceStore, epoch_millis as store_epoch_millis};
|
||||
pub(crate) use store::{
|
||||
StoredTraceRecord, TraceClearOutcome, WebTraceStore, epoch_millis as store_epoch_millis,
|
||||
};
|
||||
pub(crate) use types::{
|
||||
TraceBodySnapshot, TraceBodyState, TraceDirection, TraceFrame, TraceHeader, TraceIdentity,
|
||||
TraceLifecycleEvent, TraceLifecycleRecord, TraceRecord, TraceRecordKind, TraceRoute,
|
||||
|
||||
+78
-72
@@ -41,6 +41,10 @@ impl Drop for StoredTraceRecord {
|
||||
pub(crate) struct TraceStoreStatus {
|
||||
/// Current debug policy.
|
||||
pub(crate) policy: Arc<WebDebugConfig>,
|
||||
/// Runtime generation that last authored the effective policy.
|
||||
pub(crate) policy_generation: u64,
|
||||
/// Epoch fencing in-flight records across policy changes and clears.
|
||||
pub(crate) epoch: u64,
|
||||
/// Retained record count.
|
||||
pub(crate) records: usize,
|
||||
/// Configured record capacity.
|
||||
@@ -61,11 +65,22 @@ pub(crate) struct TraceStoreStatus {
|
||||
pub(crate) latest_seq: Option<u64>,
|
||||
}
|
||||
|
||||
/// Result of one constant-time logical trace clear.
|
||||
pub(crate) struct TraceClearOutcome {
|
||||
/// Records detached from the ring.
|
||||
pub(crate) records_cleared: usize,
|
||||
/// Bytes still retained by in-flight snapshots after detached records drop.
|
||||
pub(crate) leased_bytes: usize,
|
||||
/// New epoch rejecting commits started before the clear.
|
||||
pub(crate) epoch: u64,
|
||||
}
|
||||
|
||||
/// Process-owned bounded WEB debug trace store.
|
||||
pub(crate) struct WebTraceStore {
|
||||
policy: ArcSwap<WebDebugConfig>,
|
||||
policy_update: Mutex<()>,
|
||||
enabled: AtomicBool,
|
||||
policy_generation: AtomicU64,
|
||||
epoch: AtomicU64,
|
||||
records_capacity: usize,
|
||||
bytes_capacity: usize,
|
||||
@@ -88,6 +103,7 @@ impl WebTraceStore {
|
||||
enabled: AtomicBool::new(policy.enabled),
|
||||
policy: ArcSwap::from_pointee(policy),
|
||||
policy_update: Mutex::new(()),
|
||||
policy_generation: AtomicU64::new(0),
|
||||
epoch: AtomicU64::new(1),
|
||||
records_capacity: limits.debug_records_capacity,
|
||||
bytes_capacity: limits.debug_bytes_global,
|
||||
@@ -106,11 +122,16 @@ impl WebTraceStore {
|
||||
})
|
||||
}
|
||||
|
||||
/// Applies one hot policy and clears incompatible retained records.
|
||||
pub(crate) fn apply_policy(&self, policy: &WebDebugConfig) {
|
||||
/// Applies one generation-authored policy and rejects stale generation writers.
|
||||
pub(crate) fn apply_policy(&self, generation: u64, policy: &WebDebugConfig) {
|
||||
let _policy_update = self.policy_update.lock();
|
||||
let current_generation = self.policy_generation.load(Ordering::Acquire);
|
||||
if generation < current_generation {
|
||||
return;
|
||||
}
|
||||
let current = self.policy.load_full();
|
||||
if current.as_ref() == policy {
|
||||
self.policy_generation.store(generation, Ordering::Release);
|
||||
return;
|
||||
}
|
||||
let capture_changed = current.enabled != policy.enabled
|
||||
@@ -122,10 +143,40 @@ impl WebTraceStore {
|
||||
|| current.body_prefix_bytes != policy.body_prefix_bytes
|
||||
|| current.decoy_body_prefix_bytes != policy.decoy_body_prefix_bytes;
|
||||
self.policy.store(Arc::new(policy.clone()));
|
||||
self.policy_generation.store(generation, Ordering::Release);
|
||||
self.enabled.store(policy.enabled, Ordering::Release);
|
||||
if capture_changed {
|
||||
self.epoch.fetch_add(1, Ordering::AcqRel);
|
||||
self.ring.lock().records.clear();
|
||||
let detached = {
|
||||
let mut ring = self.ring.lock();
|
||||
std::mem::replace(
|
||||
&mut ring.records,
|
||||
VecDeque::with_capacity(self.records_capacity),
|
||||
)
|
||||
};
|
||||
drop(_policy_update);
|
||||
drop(detached);
|
||||
}
|
||||
}
|
||||
|
||||
/// Clears retained records while fencing all in-flight pre-clear commits.
|
||||
pub(crate) fn clear(&self) -> TraceClearOutcome {
|
||||
let _policy_update = self.policy_update.lock();
|
||||
let epoch = self.epoch.fetch_add(1, Ordering::AcqRel).saturating_add(1);
|
||||
let detached = {
|
||||
let mut ring = self.ring.lock();
|
||||
std::mem::replace(
|
||||
&mut ring.records,
|
||||
VecDeque::with_capacity(self.records_capacity),
|
||||
)
|
||||
};
|
||||
let records_cleared = detached.len();
|
||||
drop(_policy_update);
|
||||
drop(detached);
|
||||
TraceClearOutcome {
|
||||
records_cleared,
|
||||
leased_bytes: self.used_bytes.load(Ordering::Acquire),
|
||||
epoch,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,9 +348,12 @@ impl WebTraceStore {
|
||||
|
||||
/// Returns current bounds, counters, and retained sequence range.
|
||||
pub(crate) fn status(&self) -> TraceStoreStatus {
|
||||
let _policy_update = self.policy_update.lock();
|
||||
let ring = self.ring.lock();
|
||||
TraceStoreStatus {
|
||||
policy: self.policy.load_full(),
|
||||
policy_generation: self.policy_generation.load(Ordering::Acquire),
|
||||
epoch: self.epoch.load(Ordering::Acquire),
|
||||
records: ring.records.len(),
|
||||
records_capacity: self.records_capacity,
|
||||
used_bytes: self.used_bytes.load(Ordering::Acquire),
|
||||
@@ -312,6 +366,26 @@ impl WebTraceStore {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a non-blocking status snapshot or `None` on trace-store contention.
|
||||
pub(crate) fn try_status(&self) -> Option<TraceStoreStatus> {
|
||||
let _policy_update = self.policy_update.try_lock()?;
|
||||
let ring = self.ring.try_lock()?;
|
||||
Some(TraceStoreStatus {
|
||||
policy: self.policy.load_full(),
|
||||
policy_generation: self.policy_generation.load(Ordering::Acquire),
|
||||
epoch: self.epoch.load(Ordering::Acquire),
|
||||
records: ring.records.len(),
|
||||
records_capacity: self.records_capacity,
|
||||
used_bytes: self.used_bytes.load(Ordering::Acquire),
|
||||
bytes_capacity: self.bytes_capacity,
|
||||
contention_drops: self.contention_drops.load(Ordering::Relaxed),
|
||||
evictions: self.evictions.load(Ordering::Relaxed),
|
||||
byte_truncations: self.byte_truncations.load(Ordering::Relaxed),
|
||||
earliest_seq: ring.records.front().map(|record| record.record.seq),
|
||||
latest_seq: ring.records.back().map(|record| record.record.seq),
|
||||
})
|
||||
}
|
||||
|
||||
/// Reserves one of two bounded concurrent status-page render slots.
|
||||
pub(crate) fn try_render_permit(&self) -> Option<OwnedSemaphorePermit> {
|
||||
Arc::clone(&self.renders).try_acquire_owned().ok()
|
||||
@@ -427,72 +501,4 @@ pub(crate) fn epoch_millis() -> u64 {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn store(records_capacity: usize, bytes_capacity: usize) -> Arc<WebTraceStore> {
|
||||
let policy = WebDebugConfig {
|
||||
enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
let limits = WebLimitsConfig {
|
||||
debug_records_capacity: records_capacity,
|
||||
debug_bytes_global: bytes_capacity,
|
||||
..Default::default()
|
||||
};
|
||||
WebTraceStore::new(policy, &limits)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ring_evicts_oldest_records_and_snapshot_leases_survive_clear() {
|
||||
let store = store(2, 4 * BASE_RECORD_RESERVATION);
|
||||
for _ in 0..3 {
|
||||
store.record_lifecycle(
|
||||
None,
|
||||
Some("192.0.2.10".parse().unwrap()),
|
||||
TraceIdentity::default(),
|
||||
TraceLifecycleEvent::BridgeIssued,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
}
|
||||
|
||||
let snapshot = store.snapshot_matching(|_| true);
|
||||
assert_eq!(
|
||||
snapshot
|
||||
.iter()
|
||||
.map(|record| record.record.seq)
|
||||
.collect::<Vec<_>>(),
|
||||
vec![3, 2]
|
||||
);
|
||||
assert_eq!(store.status().evictions, 1);
|
||||
assert_eq!(store.status().used_bytes, 2 * BASE_RECORD_RESERVATION);
|
||||
|
||||
let policy = WebDebugConfig::default();
|
||||
store.apply_policy(&policy);
|
||||
assert_eq!(store.status().records, 0);
|
||||
assert_eq!(store.status().used_bytes, 2 * BASE_RECORD_RESERVATION);
|
||||
drop(snapshot);
|
||||
assert_eq!(store.status().used_bytes, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capture_policy_epoch_rejects_an_inflight_old_policy_record() {
|
||||
let store = store(4, 8 * BASE_RECORD_RESERVATION);
|
||||
let request = hyper::Request::builder().uri("/").body(()).unwrap();
|
||||
let exchange = store
|
||||
.begin_http(&request, "192.0.2.20".parse().unwrap())
|
||||
.unwrap();
|
||||
|
||||
let changed = WebDebugConfig {
|
||||
enabled: true,
|
||||
capture_headers: false,
|
||||
..Default::default()
|
||||
};
|
||||
store.apply_policy(&changed);
|
||||
exchange.commit();
|
||||
|
||||
assert_eq!(store.status().records, 0);
|
||||
assert_eq!(store.status().used_bytes, 0);
|
||||
}
|
||||
}
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
use super::*;
|
||||
|
||||
fn store(records_capacity: usize, bytes_capacity: usize) -> Arc<WebTraceStore> {
|
||||
let policy = WebDebugConfig {
|
||||
enabled: true,
|
||||
..Default::default()
|
||||
};
|
||||
let limits = WebLimitsConfig {
|
||||
debug_records_capacity: records_capacity,
|
||||
debug_bytes_global: bytes_capacity,
|
||||
..Default::default()
|
||||
};
|
||||
WebTraceStore::new(policy, &limits)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ring_evicts_oldest_records_and_snapshot_leases_survive_clear() {
|
||||
let store = store(2, 4 * BASE_RECORD_RESERVATION);
|
||||
for _ in 0..3 {
|
||||
store.record_lifecycle(
|
||||
None,
|
||||
Some("192.0.2.10".parse().unwrap()),
|
||||
TraceIdentity::default(),
|
||||
TraceLifecycleEvent::BridgeIssued,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
}
|
||||
|
||||
let snapshot = store.snapshot_matching(|_| true);
|
||||
assert_eq!(
|
||||
snapshot
|
||||
.iter()
|
||||
.map(|record| record.record.seq)
|
||||
.collect::<Vec<_>>(),
|
||||
vec![3, 2]
|
||||
);
|
||||
assert_eq!(store.status().evictions, 1);
|
||||
assert_eq!(store.status().used_bytes, 2 * BASE_RECORD_RESERVATION);
|
||||
|
||||
let policy = WebDebugConfig::default();
|
||||
store.apply_policy(2, &policy);
|
||||
assert_eq!(store.status().records, 0);
|
||||
assert_eq!(store.status().used_bytes, 2 * BASE_RECORD_RESERVATION);
|
||||
drop(snapshot);
|
||||
assert_eq!(store.status().used_bytes, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capture_policy_epoch_rejects_an_inflight_old_policy_record() {
|
||||
let store = store(4, 8 * BASE_RECORD_RESERVATION);
|
||||
let request = hyper::Request::builder().uri("/").body(()).unwrap();
|
||||
let exchange = store
|
||||
.begin_http(&request, "192.0.2.20".parse().unwrap())
|
||||
.unwrap();
|
||||
|
||||
let changed = WebDebugConfig {
|
||||
enabled: true,
|
||||
capture_headers: false,
|
||||
..Default::default()
|
||||
};
|
||||
store.apply_policy(2, &changed);
|
||||
exchange.commit();
|
||||
|
||||
assert_eq!(store.status().records, 0);
|
||||
assert_eq!(store.status().used_bytes, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_generation_cannot_restore_an_old_policy() {
|
||||
let store = store(4, 8 * BASE_RECORD_RESERVATION);
|
||||
let current = WebDebugConfig {
|
||||
enabled: true,
|
||||
capture_headers: false,
|
||||
..Default::default()
|
||||
};
|
||||
store.apply_policy(3, ¤t);
|
||||
|
||||
store.apply_policy(2, &WebDebugConfig::default());
|
||||
|
||||
let status = store.status();
|
||||
assert_eq!(status.policy_generation, 3);
|
||||
assert_eq!(status.policy.as_ref(), ¤t);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_clear_fences_inflight_commits_and_preserves_snapshot_leases() {
|
||||
let store = store(4, 8 * BASE_RECORD_RESERVATION);
|
||||
store.record_lifecycle(
|
||||
None,
|
||||
Some("192.0.2.30".parse().unwrap()),
|
||||
TraceIdentity::default(),
|
||||
TraceLifecycleEvent::BridgeIssued,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let snapshot = store.snapshot_matching(|_| true);
|
||||
let request = hyper::Request::builder().uri("/").body(()).unwrap();
|
||||
let exchange = store
|
||||
.begin_http(&request, "192.0.2.30".parse().unwrap())
|
||||
.unwrap();
|
||||
|
||||
let cleared = store.clear();
|
||||
exchange.commit();
|
||||
|
||||
assert_eq!(cleared.records_cleared, 1);
|
||||
assert_eq!(store.status().records, 0);
|
||||
assert_eq!(store.status().used_bytes, BASE_RECORD_RESERVATION);
|
||||
drop(snapshot);
|
||||
assert_eq!(store.status().used_bytes, 0);
|
||||
}
|
||||
Reference in New Issue
Block a user