mirror of
https://github.com/telemt/telemt.git
synced 2026-10-29 20:17:16 +03:00
Compare commits
21 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d3de9865cf | |||
| e2f1c3f3a9 | |||
| dcce897d8b | |||
| 823c6cbc7f | |||
| c4555e25f3 | |||
| d15d50e92a | |||
| 557ffe9ee0 | |||
| 84d5e98c11 | |||
| 642f166c98 | |||
| d65efcc479 | |||
| ca5ebd1308 | |||
| e0bf6fa3a0 | |||
| fca6506449 | |||
| 94f4f7d540 | |||
| c10677ed97 | |||
| 86912e34b2 | |||
| e5bfeafa2f | |||
| 7635d1fdef | |||
| 698fd315ea | |||
| 6fa8da3c43 | |||
| 53f131e34d |
Generated
+1
-1
@@ -2780,7 +2780,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "telemt"
|
name = "telemt"
|
||||||
version = "3.5.9"
|
version = "3.5.13"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes",
|
"aes",
|
||||||
"anyhow",
|
"anyhow",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "telemt"
|
name = "telemt"
|
||||||
version = "3.5.9"
|
version = "3.5.13"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
rust-version = "1.88"
|
rust-version = "1.88"
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
[English](WEB_PROXY.en.md) | [Русский](WEB_PROXY.ru.md) | [Deutsch](WEB_PROXY.de.md)
|
[English](WEB_PROXY.en.md) | [Русский](WEB_PROXY.ru.md) | [Deutsch](WEB_PROXY.de.md)
|
||||||
|
|
||||||
Der WEB-Modus transportiert gewöhnliche MTProxy-Streams über begrenzte HTTPS- oder WebSocket-Carrier, die mit dem Proxy-Typ `WEB` von Telegram Desktop kompatibel sind. Telemt terminiert TLS nicht selbst: NGINX oder HAProxy verwaltet das öffentliche Zertifikat und leitet unverschlüsseltes HTTP/1.1 an einen privaten Telemt-Listener weiter.
|
Der WEB-Modus transportiert gewöhnliche MTProxy-Streams über begrenzte HTTPS- oder WebSocket-Carrier, die mit dem Proxy-Typ `WEB` von Telegram Desktop kompatibel sind. Telemt terminiert TLS nicht selbst: NGINX, HAProxy oder Caddy verwaltet das öffentliche Zertifikat und leitet unverschlüsseltes HTTP/1.1 an einen privaten Telemt-Listener weiter.
|
||||||
|
|
||||||
> [!IMPORTANT]
|
> [!IMPORTANT]
|
||||||
>
|
>
|
||||||
@@ -14,7 +14,7 @@ Der WEB-Modus transportiert gewöhnliche MTProxy-Streams über begrenzte HTTPS-
|
|||||||
Telegram Desktop
|
Telegram Desktop
|
||||||
| HTTPS oder WSS :443
|
| HTTPS oder WSS :443
|
||||||
v
|
v
|
||||||
NGINX oder HAProxy (TLS-Terminierung, kanonischer Host und eine X-Forwarded-For-Adresse)
|
NGINX, HAProxy oder Caddy (TLS-Terminierung, kanonischer Host und eine X-Forwarded-For-Adresse)
|
||||||
| unverschlüsseltes HTTP/1.1 in einem privaten Netz
|
| unverschlüsseltes HTTP/1.1 in einem privaten Netz
|
||||||
v
|
v
|
||||||
Telemt-WEB-Listener
|
Telemt-WEB-Listener
|
||||||
@@ -270,6 +270,66 @@ backend telemt_web
|
|||||||
|
|
||||||
Im Frontend oder im Abschnitt `defaults` muss für das standardmäßige WebSocket-Liveness-Intervall auch `timeout client 65s` oder länger gesetzt sein. Für `https-lanes` muss das öffentliche HAProxy-ALPN `h2`, für WebSocket-Upgrade außerdem `http/1.1` enthalten. Bewahren Sie `Connection`, `Upgrade` und `Sec-WebSocket-*` unverändert; Pfad, Raw Query, Body sowie die Carrier-Header `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor` und `X-Lane-ID` dürfen nicht umgeschrieben werden. Fügen Sie für Prefix-only-Cohosting `acl telemt_web_path path_beg /telegram/web/` hinzu und verlangen Sie in `use_backend` sowohl Host- als auch Pfad-ACL; entfernen Sie das Präfix nicht.
|
Im Frontend oder im Abschnitt `defaults` muss für das standardmäßige WebSocket-Liveness-Intervall auch `timeout client 65s` oder länger gesetzt sein. Für `https-lanes` muss das öffentliche HAProxy-ALPN `h2`, für WebSocket-Upgrade außerdem `http/1.1` enthalten. Bewahren Sie `Connection`, `Upgrade` und `Sec-WebSocket-*` unverändert; Pfad, Raw Query, Body sowie die Carrier-Header `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor` und `X-Lane-ID` dürfen nicht umgeschrieben werden. Fügen Sie für Prefix-only-Cohosting `acl telemt_web_path path_beg /telegram/web/` hinzu und verlangen Sie in `use_backend` sowohl Host- als auch Pfad-ACL; entfernen Sie das Präfix nicht.
|
||||||
|
|
||||||
|
## TLS-Terminierung mit Caddy
|
||||||
|
|
||||||
|
```caddyfile
|
||||||
|
{
|
||||||
|
servers {
|
||||||
|
protocols h1 h2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
proxy.example.com {
|
||||||
|
reverse_proxy 127.0.0.1:18080 {
|
||||||
|
header_up X-Forwarded-For {remote_host}
|
||||||
|
flush_interval -1
|
||||||
|
stream_close_delay 5m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Caddy bezieht und erneuert das öffentliche Zertifikat für die Site-Adresse automatisch. Den ursprünglichen `Host` sowie die Header `Connection`, `Upgrade` und `Sec-WebSocket-*` leitet Caddy unverändert weiter, WebSocket-Upgrade wird ohne zusätzliche Direktiven proxied; fügen Sie kein `header_up Host` hinzu. Öffentliches HTTP/2 (für `https-lanes`) und HTTP/1.1 (für WebSocket-Upgrade) sind standardmäßig aktiv, der private Abschnitt Caddy → Telemt bleibt HTTP/1.1. Die globale Option `protocols h1 h2` hält HTTP/3 aus dem WEB-Pfad heraus, weil der WEB-Vertrag nur HTTP/1.1 und HTTP/2 abdeckt. Sie gilt für alle Sites dieses Caddy-Servers.
|
||||||
|
|
||||||
|
`header_up X-Forwarded-For {remote_host}` überschreibt den Header mit genau einer Adresse. Ohne diese Zeile hängt Caddy bei Peers aus `trusted_proxies` an einen eingehenden `X-Forwarded-For` an, Telemt akzeptiert jedoch nur eine parsebare IP-Adresse. Steht Caddy selbst hinter einem L4-Balancer mit `listener_wrappers { proxy_protocol }`, ist `{remote_host}` die Client-Adresse aus dem PROXY-Header. `flush_interval -1` deaktiviert die Antwortpufferung für Long Polls und Downlinks. Request-Bodies streamt Caddy standardmäßig; aktivieren Sie kein `request_buffers`.
|
||||||
|
|
||||||
|
`reverse_proxy` hat in Caddy standardmäßig weder Read-/Write-Timeouts noch ein Body-Limit, daher passieren der 25-Sekunden-Long-Poll und das WebSocket-Liveness-Intervall unverändert. Sind serverweite `timeouts` oder `request_body max_size` gesetzt, müssen sie über 65 Sekunden bzw. mindestens bei `web.limits.max_body_bytes` liegen. Mit einem einzigen Upstream wiederholt Caddy fehlgeschlagene Anfragen nicht; fügen Sie weder `lb_retries` noch `lb_try_duration` hinzu. Bei einem Konfigurations-Reload schließt Caddy upgegradete Verbindungen standardmäßig sofort. `stream_close_delay` hält bestehende WebSocket-Lanes nach einem Reload für die angegebene Zeit offen.
|
||||||
|
|
||||||
|
Leiten Sie für Prefix-only-Cohosting mit `base_path = "telegram/web"` den Teilbaum über `handle` und nicht über `handle_path` weiter, denn `handle_path` entfernt das Präfix:
|
||||||
|
|
||||||
|
```caddyfile
|
||||||
|
proxy.example.com {
|
||||||
|
handle /telegram/web/* {
|
||||||
|
reverse_proxy 127.0.0.1:18080 {
|
||||||
|
header_up X-Forwarded-For {remote_host}
|
||||||
|
flush_interval -1
|
||||||
|
stream_close_delay 5m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
handle {
|
||||||
|
# gewöhnliche Site
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Das Routing von Caddy fügt keine Weiterleitung mit angehängtem Slash hinzu: `/telegram/web` ohne abschließenden Slash landet im Handler der gewöhnlichen Site. Stellen Sie sicher, dass auch dieser Handler nicht weiterleitet; `file_server` leitet etwa eine Verzeichnisanfrage auf die Variante mit Slash um.
|
||||||
|
|
||||||
|
Ohne `log`-Direktive schreibt Caddy kein Access-Log. Wird eines benötigt, schwärzen Sie Bridge-Capability und Bearer-Credentials:
|
||||||
|
|
||||||
|
```caddyfile
|
||||||
|
log {
|
||||||
|
format filter {
|
||||||
|
request>uri query {
|
||||||
|
replace bridge REDACTED
|
||||||
|
}
|
||||||
|
request>headers>Authorization delete
|
||||||
|
request>headers>Sec-Websocket-Protocol delete
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Tragen Sie in `web_trusted_proxy_cidrs` die Adresse ein, von der aus Caddy Telemt erreicht, für die obige Konfiguration also `["127.0.0.1/32"]`.
|
||||||
|
|
||||||
## Lebenszyklus und Reload-Verhalten
|
## Lebenszyklus und Reload-Verhalten
|
||||||
|
|
||||||
| Konfiguration | Runtime-Verhalten |
|
| Konfiguration | Runtime-Verhalten |
|
||||||
@@ -431,7 +491,7 @@ Der vollständige Vertrag für Requests, Revisionen, Fehler und alle Benutzer-En
|
|||||||
| `/telegram/web` leitet auf `/telegram/web/` um | Fügen Sie für den Pfad ohne Schrägstrich einen exakten Non-WEB-Handler hinzu. Nur der mit Schrägstrich abgeschlossene konfigurierte Teilbaum gehört zum WEB-Vertrag von Telemt. |
|
| `/telegram/web` leitet auf `/telegram/web/` um | Fügen Sie für den Pfad ohne Schrägstrich einen exakten Non-WEB-Handler hinzu. Nur der mit Schrägstrich abgeschlossene konfigurierte Teilbaum gehört zum WEB-Vertrag von Telemt. |
|
||||||
| Ein konkurrierender `https-lanes`-Downlink erreicht den Decoy mit `404` | Prüfen Sie, dass er mit `X-Down-Cursor: 0` beginnt, bewahren Sie `X-Lane-ID` und setzen Sie `lane_open_wait_secs` über den beobachteten Abstand zwischen Downlink und `OPEN`. Fortgeschrittene Cursor fehlender Lanes schlagen absichtlich fail-closed fehl. |
|
| Ein konkurrierender `https-lanes`-Downlink erreicht den Decoy mit `404` | Prüfen Sie, dass er mit `X-Down-Cursor: 0` beginnt, bewahren Sie `X-Lane-ID` und setzen Sie `lane_open_wait_secs` über den beobachteten Abstand zwischen Downlink und `OPEN`. Fortgeschrittene Cursor fehlender Lanes schlagen absichtlich fail-closed fehl. |
|
||||||
| Auto-Negotiation wechselt weiter, nachdem Daten bereits akzeptiert wurden | Das ist ungültig. Prüfen Sie das authentifizierte `X-Carrier-State`-Replay und das Carrier-Commit-Lifecycle-Ereignis; `committed` oder `healthy` ist terminal und erfordert eine neue Sitzung. |
|
| Auto-Negotiation wechselt weiter, nachdem Daten bereits akzeptiert wurden | Das ist ungültig. Prüfen Sie das authentifizierte `X-Carrier-State`-Replay und das Carrier-Commit-Lifecycle-Ereignis; `committed` oder `healthy` ist terminal und erfordert eine neue Sitzung. |
|
||||||
| Long Polls werden nach einem festen Intervall getrennt | Setzen Sie Client-, Server-, Sende- und Lese-Timeouts von NGINX/HAProxy über `web.timeouts.long_poll_secs`. |
|
| Long Polls werden nach einem festen Intervall getrennt | Setzen Sie Client-, Server-, Sende- und Lese-Timeouts von NGINX/HAProxy/Caddy über `web.timeouts.long_poll_secs`. |
|
||||||
| WebSocket-Upgrade erreicht statt `101` den Decoy | Bewahren Sie HTTP/1.1 `Connection: Upgrade`, `Upgrade: websocket`, das einzelne exakte `Sec-WebSocket-Protocol` und den kanonischen bodylosen Request an der konfigurierten Basis plus `/api/v1/ws`. Prüfen Sie außerdem Carrier-/Session-Kompatibilität und die Prozess-Verbindungsreserve. |
|
| WebSocket-Upgrade erreicht statt `101` den Decoy | Bewahren Sie HTTP/1.1 `Connection: Upgrade`, `Upgrade: websocket`, das einzelne exakte `Sec-WebSocket-Protocol` und den kanonischen bodylosen Request an der konfigurierten Basis plus `/api/v1/ws`. Prüfen Sie außerdem Carrier-/Session-Kompatibilität und die Prozess-Verbindungsreserve. |
|
||||||
| Ein `websocket-lanes`-Stream wurde geschlossen, Geschwister bleiben aber verbunden | Dies ist die beabsichtigte Fehlergrenze. Prüfen Sie die Message-/Frame-Zeilen dieser Lane in `/web-status`; fehlerhafte oder lane-fremde Frames, Write-Timeouts und Backend-Close schließen nur die betroffene Lane. |
|
| Ein `websocket-lanes`-Stream wurde geschlossen, Geschwister bleiben aber verbunden | Dies ist die beabsichtigte Fehlergrenze. Prüfen Sie die Message-/Frame-Zeilen dieser Lane in `/web-status`; fehlerhafte oder lane-fremde Frames, Write-Timeouts und Backend-Close schließen nur die betroffene Lane. |
|
||||||
| `/web-status` ist leer | Prüfen Sie, dass `[web.debug].enabled = true` gesetzt ist, wenden Sie die Konfiguration an, wählen Sie ein Fenster innerhalb von `max_window_secs` und erzeugen Sie nach der Policy-Änderung neuen WEB-Datenverkehr. |
|
| `/web-status` ist leer | Prüfen Sie, dass `[web.debug].enabled = true` gesetzt ist, wenden Sie die Konfiguration an, wählen Sie ein Fenster innerhalb von `max_window_secs` und erzeugen Sie nach der Policy-Änderung neuen WEB-Datenverkehr. |
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
[English](WEB_PROXY.en.md) | [Русский](WEB_PROXY.ru.md) | [Deutsch](WEB_PROXY.de.md)
|
[English](WEB_PROXY.en.md) | [Русский](WEB_PROXY.ru.md) | [Deutsch](WEB_PROXY.de.md)
|
||||||
|
|
||||||
WEB mode carries ordinary MTProxy streams through bounded HTTPS or WebSocket carriers compatible with Telegram Desktop's `WEB` proxy type. Telemt does not terminate TLS: NGINX or HAProxy owns the public certificate and forwards plain HTTP/1.1 to a private Telemt listener.
|
WEB mode carries ordinary MTProxy streams through bounded HTTPS or WebSocket carriers compatible with Telegram Desktop's `WEB` proxy type. Telemt does not terminate TLS: NGINX, HAProxy, or Caddy owns the public certificate and forwards plain HTTP/1.1 to a private Telemt listener.
|
||||||
|
|
||||||
> [!IMPORTANT]
|
> [!IMPORTANT]
|
||||||
>
|
>
|
||||||
@@ -14,7 +14,7 @@ WEB mode carries ordinary MTProxy streams through bounded HTTPS or WebSocket car
|
|||||||
Telegram Desktop
|
Telegram Desktop
|
||||||
| HTTPS or WSS :443
|
| HTTPS or WSS :443
|
||||||
v
|
v
|
||||||
NGINX or HAProxy (TLS termination, canonical Host and one X-Forwarded-For address)
|
NGINX, HAProxy, or Caddy (TLS termination, canonical Host and one X-Forwarded-For address)
|
||||||
| plain HTTP/1.1 on a private network
|
| plain HTTP/1.1 on a private network
|
||||||
v
|
v
|
||||||
Telemt WEB listener
|
Telemt WEB listener
|
||||||
@@ -270,6 +270,66 @@ backend telemt_web
|
|||||||
|
|
||||||
The frontend or `defaults` section must also set `timeout client 65s` or longer for the default WebSocket liveness interval. HAProxy's public ALPN must include `h2` for `https-lanes` and `http/1.1` for WebSocket Upgrade. Preserve `Connection`, `Upgrade`, and `Sec-WebSocket-*`; do not rewrite the path, raw query, body, or the `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor`, and `X-Lane-ID` carrier headers. For prefix-only cohosting, add `acl telemt_web_path path_beg /telegram/web/` and require both the host and path ACLs on `use_backend`; do not remove the prefix.
|
The frontend or `defaults` section must also set `timeout client 65s` or longer for the default WebSocket liveness interval. HAProxy's public ALPN must include `h2` for `https-lanes` and `http/1.1` for WebSocket Upgrade. Preserve `Connection`, `Upgrade`, and `Sec-WebSocket-*`; do not rewrite the path, raw query, body, or the `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor`, and `X-Lane-ID` carrier headers. For prefix-only cohosting, add `acl telemt_web_path path_beg /telegram/web/` and require both the host and path ACLs on `use_backend`; do not remove the prefix.
|
||||||
|
|
||||||
|
## Caddy TLS termination
|
||||||
|
|
||||||
|
```caddyfile
|
||||||
|
{
|
||||||
|
servers {
|
||||||
|
protocols h1 h2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
proxy.example.com {
|
||||||
|
reverse_proxy 127.0.0.1:18080 {
|
||||||
|
header_up X-Forwarded-For {remote_host}
|
||||||
|
flush_interval -1
|
||||||
|
stream_close_delay 5m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Caddy obtains and renews the public certificate for the site address automatically. It forwards the original `Host` and the `Connection`, `Upgrade`, and `Sec-WebSocket-*` headers unchanged and proxies WebSocket Upgrade without extra directives; do not add `header_up Host`. Public HTTP/2 (needed for `https-lanes`) and HTTP/1.1 (needed for WebSocket Upgrade) are both enabled by default, while the private Caddy-to-Telemt hop stays HTTP/1.1. The global `protocols h1 h2` option keeps HTTP/3 out of the WEB path, because the WEB contract only covers HTTP/1.1 and HTTP/2. It applies to every site of that Caddy server.
|
||||||
|
|
||||||
|
`header_up X-Forwarded-For {remote_host}` overwrites the header with one address. Without it, Caddy appends to an incoming `X-Forwarded-For` from peers listed in `trusted_proxies`, and Telemt accepts only one parseable IP address. If Caddy itself sits behind an L4 balancer with `listener_wrappers { proxy_protocol }`, `{remote_host}` is the client address taken from the PROXY header. `flush_interval -1` disables response buffering for long polls and downlinks. Request bodies are streamed by default; do not enable `request_buffers`.
|
||||||
|
|
||||||
|
Caddy's `reverse_proxy` has no read or write timeout and no request-body limit by default, so the 25-second long poll and the WebSocket liveness interval pass unchanged. If server-wide `timeouts` or `request_body max_size` are configured, keep them above 65 seconds and at least `web.limits.max_body_bytes` respectively. With a single upstream Caddy does not retry failed requests; do not add `lb_retries` or `lb_try_duration`. On a configuration reload Caddy closes upgraded connections immediately by default. `stream_close_delay` keeps established WebSocket lanes open across a reload for the given time.
|
||||||
|
|
||||||
|
For prefix-only cohosting with `base_path = "telegram/web"`, route the subtree with `handle`, not `handle_path`, because `handle_path` strips the prefix:
|
||||||
|
|
||||||
|
```caddyfile
|
||||||
|
proxy.example.com {
|
||||||
|
handle /telegram/web/* {
|
||||||
|
reverse_proxy 127.0.0.1:18080 {
|
||||||
|
header_up X-Forwarded-For {remote_host}
|
||||||
|
flush_interval -1
|
||||||
|
stream_close_delay 5m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
handle {
|
||||||
|
# ordinary site
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Caddy's routing does not add a slash-appending redirect: `/telegram/web` without the trailing slash falls through to the ordinary site handler. Make sure that handler does not redirect it either; for example, `file_server` redirects a directory request to its slash-terminated form.
|
||||||
|
|
||||||
|
Caddy writes no access log unless a `log` directive is present. If one is needed, redact the bridge capability and bearer credentials:
|
||||||
|
|
||||||
|
```caddyfile
|
||||||
|
log {
|
||||||
|
format filter {
|
||||||
|
request>uri query {
|
||||||
|
replace bridge REDACTED
|
||||||
|
}
|
||||||
|
request>headers>Authorization delete
|
||||||
|
request>headers>Sec-Websocket-Protocol delete
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Set `web_trusted_proxy_cidrs` to the address Caddy uses to connect to Telemt, for example `["127.0.0.1/32"]` for the configuration above.
|
||||||
|
|
||||||
## Lifecycle and reload behavior
|
## Lifecycle and reload behavior
|
||||||
|
|
||||||
| Configuration | Runtime behavior |
|
| Configuration | Runtime behavior |
|
||||||
@@ -431,7 +491,7 @@ See the complete [Control API contract](../Architecture/API/API.md) for request
|
|||||||
| `/telegram/web` redirects to `/telegram/web/` | Add an exact non-WEB handler for the no-slash path. Only the slash-terminated configured subtree belongs to Telemt's WEB contract. |
|
| `/telegram/web` redirects to `/telegram/web/` | Add an exact non-WEB handler for the no-slash path. Only the slash-terminated configured subtree belongs to Telemt's WEB contract. |
|
||||||
| A racing `https-lanes` downlink reaches the decoy with `404` | Confirm it starts at `X-Down-Cursor: 0`, preserve `X-Lane-ID`, and set `lane_open_wait_secs` above the observed down-before-`OPEN` skew. Advanced cursors for missing lanes intentionally fail closed. |
|
| A racing `https-lanes` downlink reaches the decoy with `404` | Confirm it starts at `X-Down-Cursor: 0`, preserve `X-Lane-ID`, and set `lane_open_wait_secs` above the observed down-before-`OPEN` skew. Advanced cursors for missing lanes intentionally fail closed. |
|
||||||
| Auto-negotiation advances after traffic was already accepted | This is not valid behavior. Inspect the authenticated `X-Carrier-State` replay and the carrier commit lifecycle row; a committed or healthy response is terminal and requires a new session. |
|
| Auto-negotiation advances after traffic was already accepted | This is not valid behavior. Inspect the authenticated `X-Carrier-State` replay and the carrier commit lifecycle row; a committed or healthy response is terminal and requires a new session. |
|
||||||
| Long polls disconnect near a fixed interval | Raise NGINX/HAProxy client, server, send, and read timeouts above `web.timeouts.long_poll_secs`. |
|
| Long polls disconnect near a fixed interval | Raise NGINX/HAProxy/Caddy client, server, send, and read timeouts above `web.timeouts.long_poll_secs`. |
|
||||||
| WebSocket Upgrade reaches the decoy instead of returning `101` | Preserve HTTP/1.1 `Connection: Upgrade`, `Upgrade: websocket`, the single exact `Sec-WebSocket-Protocol`, and the canonical bodyless request at the configured base plus `/api/v1/ws`. Also check carrier/session compatibility and the process connection reserve. |
|
| WebSocket Upgrade reaches the decoy instead of returning `101` | Preserve HTTP/1.1 `Connection: Upgrade`, `Upgrade: websocket`, the single exact `Sec-WebSocket-Protocol`, and the canonical bodyless request at the configured base plus `/api/v1/ws`. Also check carrier/session compatibility and the process connection reserve. |
|
||||||
| One `websocket-lanes` stream closes while siblings stay connected | This is the intended failure boundary. Inspect that lane's message/frame rows in `/web-status`; malformed, cross-lane, write-timeout, and backend-close paths terminate only the affected lane. |
|
| One `websocket-lanes` stream closes while siblings stay connected | This is the intended failure boundary. Inspect that lane's message/frame rows in `/web-status`; malformed, cross-lane, write-timeout, and backend-close paths terminate only the affected lane. |
|
||||||
| `/web-status` is empty | Confirm `[web.debug].enabled = true`, apply the configuration, select a window within `max_window_secs`, and generate new WEB traffic after the policy change. |
|
| `/web-status` is empty | Confirm `[web.debug].enabled = true`, apply the configuration, select a window within `max_window_secs`, and generate new WEB traffic after the policy change. |
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
[English](WEB_PROXY.en.md) | [Русский](WEB_PROXY.ru.md) | [Deutsch](WEB_PROXY.de.md)
|
[English](WEB_PROXY.en.md) | [Русский](WEB_PROXY.ru.md) | [Deutsch](WEB_PROXY.de.md)
|
||||||
|
|
||||||
WEB-режим переносит обычные MTProxy-потоки через bounded HTTPS или WebSocket carriers, совместимые с типом прокси `WEB` в Telegram Desktop. Telemt не терминирует TLS: публичный сертификат обслуживает NGINX или HAProxy, который передаёт обычный HTTP/1.1 на приватный listener Telemt.
|
WEB-режим переносит обычные MTProxy-потоки через bounded HTTPS или WebSocket carriers, совместимые с типом прокси `WEB` в Telegram Desktop. Telemt не терминирует TLS: публичный сертификат обслуживает NGINX, HAProxy или Caddy, который передаёт обычный HTTP/1.1 на приватный listener Telemt.
|
||||||
|
|
||||||
> [!IMPORTANT]
|
> [!IMPORTANT]
|
||||||
>
|
>
|
||||||
@@ -14,7 +14,7 @@ WEB-режим переносит обычные MTProxy-потоки через
|
|||||||
Telegram Desktop
|
Telegram Desktop
|
||||||
| HTTPS или WSS :443
|
| HTTPS или WSS :443
|
||||||
v
|
v
|
||||||
NGINX или HAProxy (TLS termination, канонический Host и один адрес X-Forwarded-For)
|
NGINX, HAProxy или Caddy (TLS termination, канонический Host и один адрес X-Forwarded-For)
|
||||||
| обычный HTTP/1.1 в приватной сети
|
| обычный HTTP/1.1 в приватной сети
|
||||||
v
|
v
|
||||||
WEB-listener Telemt
|
WEB-listener Telemt
|
||||||
@@ -270,6 +270,66 @@ backend telemt_web
|
|||||||
|
|
||||||
Во frontend или секции `defaults` также задайте `timeout client 65s` или больше для default WebSocket liveness interval. Для `https-lanes` публичный ALPN HAProxy должен содержать `h2`, а для WebSocket Upgrade — `http/1.1`. Сохраняйте `Connection`, `Upgrade` и `Sec-WebSocket-*`; не переписывайте path, raw query, body и carrier headers `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor`, `X-Lane-ID`. Для prefix-only cohosting добавьте `acl telemt_web_path path_beg /telegram/web/` и потребуйте одновременно host- и path-ACL в `use_backend`; не удаляйте prefix.
|
Во frontend или секции `defaults` также задайте `timeout client 65s` или больше для default WebSocket liveness interval. Для `https-lanes` публичный ALPN HAProxy должен содержать `h2`, а для WebSocket Upgrade — `http/1.1`. Сохраняйте `Connection`, `Upgrade` и `Sec-WebSocket-*`; не переписывайте path, raw query, body и carrier headers `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor`, `X-Lane-ID`. Для prefix-only cohosting добавьте `acl telemt_web_path path_beg /telegram/web/` и потребуйте одновременно host- и path-ACL в `use_backend`; не удаляйте prefix.
|
||||||
|
|
||||||
|
## Терминация TLS на Caddy
|
||||||
|
|
||||||
|
```caddyfile
|
||||||
|
{
|
||||||
|
servers {
|
||||||
|
protocols h1 h2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
proxy.example.com {
|
||||||
|
reverse_proxy 127.0.0.1:18080 {
|
||||||
|
header_up X-Forwarded-For {remote_host}
|
||||||
|
flush_interval -1
|
||||||
|
stream_close_delay 5m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Caddy сам получает и продлевает публичный сертификат для адреса сайта. Он передаёт исходный `Host` и заголовки `Connection`, `Upgrade` и `Sec-WebSocket-*` без изменений и проксирует WebSocket Upgrade без дополнительных директив; не добавляйте `header_up Host`. Публичные HTTP/2 (нужен для `https-lanes`) и HTTP/1.1 (нужен для WebSocket Upgrade) включены по умолчанию, а приватный участок Caddy → Telemt остаётся HTTP/1.1. Глобальная опция `protocols h1 h2` убирает HTTP/3 с пути WEB: контракт WEB описывает только HTTP/1.1 и HTTP/2. Она действует на все сайты этого сервера Caddy.
|
||||||
|
|
||||||
|
`header_up X-Forwarded-For {remote_host}` перезаписывает заголовок одним адресом. Без неё Caddy дописывает адрес к входящему `X-Forwarded-For` от пиров из `trusted_proxies`, а Telemt принимает только один разбираемый IP-адрес. Если сам Caddy стоит за L4-балансировщиком с `listener_wrappers { proxy_protocol }`, `{remote_host}` — адрес клиента из заголовка PROXY. `flush_interval -1` отключает буферизацию ответа для long polls и downlink. Тела запросов Caddy по умолчанию передаёт потоком; не включайте `request_buffers`.
|
||||||
|
|
||||||
|
У `reverse_proxy` в Caddy по умолчанию нет read/write timeout и лимита тела запроса, поэтому 25-секундный long poll и WebSocket liveness interval проходят без изменений. Если заданы серверные `timeouts` или `request_body max_size`, держите их выше 65 секунд и не меньше `web.limits.max_body_bytes` соответственно. С одним upstream Caddy не повторяет неудачные запросы; не добавляйте `lb_retries` и `lb_try_duration`. При reload конфигурации Caddy по умолчанию сразу закрывает upgraded-соединения. `stream_close_delay` держит установленные WebSocket lanes открытыми на заданное время после reload.
|
||||||
|
|
||||||
|
Для prefix-only cohosting с `base_path = "telegram/web"` направляйте поддерево через `handle`, а не `handle_path`: `handle_path` срезает prefix.
|
||||||
|
|
||||||
|
```caddyfile
|
||||||
|
proxy.example.com {
|
||||||
|
handle /telegram/web/* {
|
||||||
|
reverse_proxy 127.0.0.1:18080 {
|
||||||
|
header_up X-Forwarded-For {remote_host}
|
||||||
|
flush_interval -1
|
||||||
|
stream_close_delay 5m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
handle {
|
||||||
|
# обычный сайт
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Маршрутизация Caddy не добавляет редирект со слэшем: `/telegram/web` без завершающего слэша уходит в обработчик обычного сайта. Убедитесь, что и этот обработчик не редиректит его; например, `file_server` перенаправляет запрос каталога на вариант со слэшем.
|
||||||
|
|
||||||
|
Caddy не пишет access log без директивы `log`. Если журнал нужен, скройте bridge capability и bearer credentials:
|
||||||
|
|
||||||
|
```caddyfile
|
||||||
|
log {
|
||||||
|
format filter {
|
||||||
|
request>uri query {
|
||||||
|
replace bridge REDACTED
|
||||||
|
}
|
||||||
|
request>headers>Authorization delete
|
||||||
|
request>headers>Sec-Websocket-Protocol delete
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
В `web_trusted_proxy_cidrs` укажите адрес, с которого Caddy подключается к Telemt, например `["127.0.0.1/32"]` для конфигурации выше.
|
||||||
|
|
||||||
## Lifecycle и reload
|
## Lifecycle и reload
|
||||||
|
|
||||||
| Конфигурация | Поведение runtime |
|
| Конфигурация | Поведение runtime |
|
||||||
@@ -431,7 +491,7 @@ curl -sS -X POST http://127.0.0.1:9091/v1/users/web-user/rotate-secret \
|
|||||||
| `/telegram/web` перенаправляет на `/telegram/web/` | Добавьте точный non-WEB handler для path без слеша. В WEB-контракт Telemt входит только настроенное поддерево с завершающим слешем. |
|
| `/telegram/web` перенаправляет на `/telegram/web/` | Добавьте точный non-WEB handler для path без слеша. В WEB-контракт Telemt входит только настроенное поддерево с завершающим слешем. |
|
||||||
| Downlink `https-lanes`, участвующий в гонке, попадает в decoy с `404` | Убедитесь, что он начинается с `X-Down-Cursor: 0`, сохраняйте `X-Lane-ID` и задайте `lane_open_wait_secs` выше наблюдаемого разрыва down-before-`OPEN`. Продвинутый cursor отсутствующей lane намеренно закрывается fail-closed. |
|
| Downlink `https-lanes`, участвующий в гонке, попадает в decoy с `404` | Убедитесь, что он начинается с `X-Down-Cursor: 0`, сохраняйте `X-Lane-ID` и задайте `lane_open_wait_secs` выше наблюдаемого разрыва down-before-`OPEN`. Продвинутый cursor отсутствующей lane намеренно закрывается fail-closed. |
|
||||||
| Auto-negotiation переходит дальше после уже принятого трафика | Такое поведение некорректно. Проверьте аутентифицированный replay `X-Carrier-State` и lifecycle row commit carrier; ответ `committed` или `healthy` terminal и требует новой сессии. |
|
| Auto-negotiation переходит дальше после уже принятого трафика | Такое поведение некорректно. Проверьте аутентифицированный replay `X-Carrier-State` и lifecycle row commit carrier; ответ `committed` или `healthy` terminal и требует новой сессии. |
|
||||||
| Long polls разрываются через фиксированный интервал | Поднимите client, server, send и read timeouts NGINX/HAProxy выше `web.timeouts.long_poll_secs`. |
|
| Long polls разрываются через фиксированный интервал | Поднимите client, server, send и read timeouts NGINX/HAProxy/Caddy выше `web.timeouts.long_poll_secs`. |
|
||||||
| WebSocket Upgrade попадает в decoy вместо `101` | Сохраните HTTP/1.1 `Connection: Upgrade`, `Upgrade: websocket`, единственный точный `Sec-WebSocket-Protocol` и канонический bodyless request по настроенному base плюс `/api/v1/ws`. Также проверьте соответствие carrier/session и process connection reserve. |
|
| WebSocket Upgrade попадает в decoy вместо `101` | Сохраните HTTP/1.1 `Connection: Upgrade`, `Upgrade: websocket`, единственный точный `Sec-WebSocket-Protocol` и канонический bodyless request по настроенному base плюс `/api/v1/ws`. Также проверьте соответствие carrier/session и process connection reserve. |
|
||||||
| Один stream `websocket-lanes` закрылся, а siblings остались подключены | Это штатная failure boundary. Проверьте message/frame rows этой lane в `/web-status`; malformed, cross-lane, write-timeout и backend-close закрывают только затронутую lane. |
|
| Один stream `websocket-lanes` закрылся, а siblings остались подключены | Это штатная failure boundary. Проверьте message/frame rows этой lane в `/web-status`; malformed, cross-lane, write-timeout и backend-close закрывают только затронутую lane. |
|
||||||
| `/web-status` пуст | Убедитесь, что `[web.debug].enabled = true`, примените конфигурацию, выберите окно в пределах `max_window_secs` и создайте новый WEB-трафик после изменения policy. |
|
| `/web-status` пуст | Убедитесь, что `[web.debug].enabled = true`, примените конфигурацию, выберите окно в пределах `max_window_secs` и создайте новый WEB-трафик после изменения policy. |
|
||||||
|
|||||||
@@ -453,3 +453,7 @@ mod base_path_tests;
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[path = "config_edit/tests.rs"]
|
#[path = "config_edit/tests.rs"]
|
||||||
mod tests;
|
mod tests;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "config_edit/conveyor_tests.rs"]
|
||||||
|
mod conveyor_tests;
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn conveyor_api_is_boolean_hot_and_rejects_invalid_patch_atomically() {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let path = directory.path().join("config.toml");
|
||||||
|
std::fs::write(&path, "[web]\nconveyor = true\n").unwrap();
|
||||||
|
for enabled in [false, true] {
|
||||||
|
let active = ProxyConfig::load(&path).unwrap();
|
||||||
|
let patch = serde_json::json!({"web": {"conveyor": enabled}});
|
||||||
|
let mut response = apply_patch_to_path(&path, &patch, None).await.unwrap();
|
||||||
|
let desired = ProxyConfig::load(&path).unwrap();
|
||||||
|
let resolved = reconcile_runtime_effect(&mut response, &active, &desired).unwrap();
|
||||||
|
assert_eq!(resolved.effective.web.conveyor, enabled);
|
||||||
|
assert!(response.runtime_reload_required);
|
||||||
|
assert!(!response.process_restart_required);
|
||||||
|
let (managed, _) = read_managed_config(&path).await.unwrap();
|
||||||
|
assert_eq!(managed["web"]["conveyor"].as_bool(), Some(enabled));
|
||||||
|
let previous = std::fs::read(&path).unwrap();
|
||||||
|
let invalid = serde_json::json!({"web": {"conveyor": "false"}});
|
||||||
|
let error = apply_patch_to_path(&path, &invalid, None)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert_eq!(error.status, hyper::StatusCode::BAD_REQUEST);
|
||||||
|
assert_eq!(std::fs::read(&path).unwrap(), previous);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,54 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn carrier_method_api_defaults_and_patches_are_hot() {
|
||||||
|
let (path, _directory) = temp_config("[web]\nenabled = false\n");
|
||||||
|
let (value, _) = read_managed_config(&path).await.unwrap();
|
||||||
|
assert_eq!(value["web"]["carrier_method"].as_str(), Some("post"));
|
||||||
|
for token in ["put", "post"] {
|
||||||
|
let active = ProxyConfig::load(&path).unwrap();
|
||||||
|
let patch = serde_json::json!({"web": {"carrier_method": token}});
|
||||||
|
let mut response = apply_patch_to_path(&path, &patch, None).await.unwrap();
|
||||||
|
let desired = ProxyConfig::load(&path).unwrap();
|
||||||
|
reconcile_runtime_effect(&mut response, &active, &desired).unwrap();
|
||||||
|
assert!(!response.restart_required);
|
||||||
|
assert!(response.runtime_reload_required);
|
||||||
|
assert!(!response.process_restart_required);
|
||||||
|
assert!(response.deferred_process_fields.is_empty());
|
||||||
|
let (value, revision) = read_managed_config(&path).await.unwrap();
|
||||||
|
assert_eq!(value["web"]["carrier_method"].as_str(), Some(token));
|
||||||
|
assert_eq!(revision, response.revision);
|
||||||
|
let written = tokio::fs::read_to_string(&path).await.unwrap();
|
||||||
|
assert!(written.contains(&format!("carrier_method = \"{token}\"")));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn carrier_method_api_rejects_invalid_values_without_writing() {
|
||||||
|
let (path, _directory) = temp_config("[web]\ncarrier_method = \"put\"\n");
|
||||||
|
let original = tokio::fs::read(&path).await.unwrap();
|
||||||
|
let revision = crate::api::config_store::current_revision(&path)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
for value in [
|
||||||
|
serde_json::json!("PUT"),
|
||||||
|
serde_json::json!("patch"),
|
||||||
|
serde_json::json!(true),
|
||||||
|
serde_json::json!(42),
|
||||||
|
] {
|
||||||
|
let patch = serde_json::json!({"web": {"carrier_method": value}});
|
||||||
|
let error = apply_patch_to_path(&path, &patch, None).await.unwrap_err();
|
||||||
|
assert_eq!(error.status, hyper::StatusCode::BAD_REQUEST);
|
||||||
|
assert_eq!(tokio::fs::read(&path).await.unwrap(), original);
|
||||||
|
assert_eq!(
|
||||||
|
crate::api::config_store::current_revision(&path)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
revision
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn json_object_converts_to_toml_table() {
|
fn json_object_converts_to_toml_table() {
|
||||||
let j: Json = serde_json::json!({"censorship": {"tls_domain": "a.com"}, "default_dc": 2});
|
let j: Json = serde_json::json!({"censorship": {"tls_domain": "a.com"}, "default_dc": 2});
|
||||||
@@ -127,7 +176,8 @@ async fn patch_web_debug_is_hot_and_limits_are_process_deferred() {
|
|||||||
assert!(desired.web.debug.sideband);
|
assert!(desired.web.debug.sideband);
|
||||||
|
|
||||||
let limits_patch: Json = serde_json::json!({
|
let limits_patch: Json = serde_json::json!({
|
||||||
"web": {"limits": {"max_http_connections": 2049}}
|
// Extra HTTP heads must fit alongside the default-on conveyor metadata reservation.
|
||||||
|
"web": {"limits": {"max_http_connections": 2049, "memory_envelope_bytes": 1610612736u64}}
|
||||||
});
|
});
|
||||||
let limits = apply_patch_to_path(&path, &limits_patch, None)
|
let limits = apply_patch_to_path(&path, &limits_patch, None)
|
||||||
.await
|
.await
|
||||||
|
|||||||
+19
-6
@@ -2,9 +2,9 @@
|
|||||||
//!
|
//!
|
||||||
//! Subcommands:
|
//! Subcommands:
|
||||||
//! - `start [OPTIONS] [config.toml]` - Start the daemon
|
//! - `start [OPTIONS] [config.toml]` - Start the daemon
|
||||||
//! - `stop [--pid-file PATH]` - Stop a running daemon
|
//! - `stop [--pid-file PATH] [--strict-runtime-paths]` - Stop a running daemon
|
||||||
//! - `reload [--pid-file PATH]` - Reload configuration (SIGHUP)
|
//! - `reload [--pid-file PATH] [--strict-runtime-paths]` - Reload configuration (SIGHUP)
|
||||||
//! - `status [--pid-file PATH]` - Check daemon status
|
//! - `status [--pid-file PATH] [--strict-runtime-paths]` - Check daemon status
|
||||||
//! - `run [OPTIONS] [config.toml]` - Run in foreground (default behavior)
|
//! - `run [OPTIONS] [config.toml]` - Run in foreground (default behavior)
|
||||||
//! - `healthcheck [OPTIONS] [config.toml]` - Run control-plane health probe
|
//! - `healthcheck [OPTIONS] [config.toml]` - Run control-plane health probe
|
||||||
|
|
||||||
@@ -137,6 +137,10 @@ pub fn parse_command(args: &[String]) -> ParsedCommand {
|
|||||||
while i < args.len() {
|
while i < args.len() {
|
||||||
match args[i].as_str() {
|
match args[i].as_str() {
|
||||||
"start" | "stop" | "reload" | "status" | "run" | "healthcheck" => {}
|
"start" | "stop" | "reload" | "status" | "run" | "healthcheck" => {}
|
||||||
|
#[cfg(unix)]
|
||||||
|
"--strict-runtime-paths" => {
|
||||||
|
cmd.daemon_opts.strict_runtime_paths = true;
|
||||||
|
}
|
||||||
"--mode" => {
|
"--mode" => {
|
||||||
i += 1;
|
i += 1;
|
||||||
if i < args.len() {
|
if i < args.len() {
|
||||||
@@ -199,9 +203,18 @@ pub fn parse_command(args: &[String]) -> ParsedCommand {
|
|||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
pub fn execute_subcommand(cmd: &ParsedCommand) -> Option<i32> {
|
pub fn execute_subcommand(cmd: &ParsedCommand) -> Option<i32> {
|
||||||
match cmd.subcommand {
|
match cmd.subcommand {
|
||||||
Subcommand::Stop => Some(daemon_commands::stop(&cmd.pid_file)),
|
Subcommand::Stop => Some(daemon_commands::stop(
|
||||||
Subcommand::Reload => Some(daemon_commands::reload(&cmd.pid_file)),
|
&cmd.pid_file,
|
||||||
Subcommand::Status => Some(daemon_commands::status(&cmd.pid_file)),
|
cmd.daemon_opts.strict_runtime_paths,
|
||||||
|
)),
|
||||||
|
Subcommand::Reload => Some(daemon_commands::reload(
|
||||||
|
&cmd.pid_file,
|
||||||
|
cmd.daemon_opts.strict_runtime_paths,
|
||||||
|
)),
|
||||||
|
Subcommand::Status => Some(daemon_commands::status(
|
||||||
|
&cmd.pid_file,
|
||||||
|
cmd.daemon_opts.strict_runtime_paths,
|
||||||
|
)),
|
||||||
Subcommand::Healthcheck => {
|
Subcommand::Healthcheck => {
|
||||||
if let Some(invalid_mode) = cmd.healthcheck_mode_invalid.as_ref() {
|
if let Some(invalid_mode) = cmd.healthcheck_mode_invalid.as_ref() {
|
||||||
if invalid_mode.is_empty() {
|
if invalid_mode.is_empty() {
|
||||||
|
|||||||
+165
-8
@@ -15,6 +15,9 @@ pub fn parse_daemon_args(args: &[String]) -> DaemonOptions {
|
|||||||
"--foreground" | "-f" => {
|
"--foreground" | "-f" => {
|
||||||
opts.foreground = true;
|
opts.foreground = true;
|
||||||
}
|
}
|
||||||
|
"--strict-runtime-paths" => {
|
||||||
|
opts.strict_runtime_paths = true;
|
||||||
|
}
|
||||||
"--pid-file" => {
|
"--pid-file" => {
|
||||||
i += 1;
|
i += 1;
|
||||||
if i < args.len() {
|
if i < args.len() {
|
||||||
@@ -60,19 +63,21 @@ pub fn parse_daemon_args(args: &[String]) -> DaemonOptions {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Sends SIGTERM and waits briefly for graceful PID-file cleanup.
|
/// Sends SIGTERM and waits briefly for graceful PID-file cleanup.
|
||||||
pub(super) fn stop(pid_file: &Path) -> i32 {
|
pub(super) fn stop(pid_file: &Path, strict_runtime_paths: bool) -> i32 {
|
||||||
use nix::sys::signal::Signal;
|
use nix::sys::signal::Signal;
|
||||||
|
|
||||||
println!("Stopping telemt daemon...");
|
println!("Stopping telemt daemon...");
|
||||||
|
|
||||||
match daemon::signal_pid_file(pid_file, Signal::SIGTERM) {
|
match daemon::signal_pid_file(pid_file, Signal::SIGTERM, strict_runtime_paths) {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
println!("Stop signal sent successfully");
|
println!("Stop signal sent successfully");
|
||||||
|
|
||||||
// Wait for process to exit for up to ten seconds.
|
// Wait for process to exit for up to ten seconds.
|
||||||
for _ in 0..20 {
|
for _ in 0..20 {
|
||||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||||
if let daemon::DaemonStatus::NotRunning = daemon::check_status(pid_file) {
|
if let daemon::DaemonStatus::NotRunning =
|
||||||
|
daemon::check_status(pid_file, strict_runtime_paths)
|
||||||
|
{
|
||||||
println!("Daemon stopped");
|
println!("Daemon stopped");
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
@@ -88,12 +93,12 @@ pub(super) fn stop(pid_file: &Path) -> i32 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Sends SIGHUP to trigger configuration reload.
|
/// Sends SIGHUP to trigger configuration reload.
|
||||||
pub(super) fn reload(pid_file: &Path) -> i32 {
|
pub(super) fn reload(pid_file: &Path, strict_runtime_paths: bool) -> i32 {
|
||||||
use nix::sys::signal::Signal;
|
use nix::sys::signal::Signal;
|
||||||
|
|
||||||
println!("Reloading telemt configuration...");
|
println!("Reloading telemt configuration...");
|
||||||
|
|
||||||
match daemon::signal_pid_file(pid_file, Signal::SIGHUP) {
|
match daemon::signal_pid_file(pid_file, Signal::SIGHUP, strict_runtime_paths) {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
println!("Reload signal sent successfully");
|
println!("Reload signal sent successfully");
|
||||||
0
|
0
|
||||||
@@ -106,8 +111,8 @@ pub(super) fn reload(pid_file: &Path) -> i32 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Reports daemon status without mutating PID lifecycle state.
|
/// Reports daemon status without mutating PID lifecycle state.
|
||||||
pub(super) fn status(pid_file: &Path) -> i32 {
|
pub(super) fn status(pid_file: &Path, strict_runtime_paths: bool) -> i32 {
|
||||||
match daemon::check_status(pid_file) {
|
match daemon::check_status(pid_file, strict_runtime_paths) {
|
||||||
daemon::DaemonStatus::Running(pid) => {
|
daemon::DaemonStatus::Running(pid) => {
|
||||||
println!("telemt is running (pid {})", pid);
|
println!("telemt is running (pid {})", pid);
|
||||||
0
|
0
|
||||||
@@ -126,16 +131,168 @@ pub(super) fn status(pid_file: &Path) -> i32 {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use std::fs;
|
use std::fs;
|
||||||
|
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||||
|
use std::process::{Child, Command, Stdio};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
const CONTROL_PID: &str = "TELEMT_RUNTIME_PATH_TEST_PID";
|
||||||
|
const CONTROL_READY: &str = "TELEMT_RUNTIME_PATH_TEST_READY";
|
||||||
|
const CONTROL_RELOADED: &str = "TELEMT_RUNTIME_PATH_TEST_RELOADED";
|
||||||
|
|
||||||
|
struct ControlChild(Child);
|
||||||
|
|
||||||
|
impl Drop for ControlChild {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = self.0.kill();
|
||||||
|
let _ = self.0.wait();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn wait_for_file(path: &Path) -> bool {
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(5);
|
||||||
|
while Instant::now() < deadline {
|
||||||
|
if path.exists() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
std::thread::sleep(Duration::from_millis(10));
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_path_policy_defaults_to_compatibility_for_all_commands() {
|
||||||
|
for command in ["run", "start", "stop", "reload", "status"] {
|
||||||
|
let args = vec![command.to_string(), "config.toml".to_string()];
|
||||||
|
assert!(
|
||||||
|
!crate::cli::parse_command(&args)
|
||||||
|
.daemon_opts
|
||||||
|
.strict_runtime_paths
|
||||||
|
);
|
||||||
|
assert!(!parse_daemon_args(&args).strict_runtime_paths);
|
||||||
|
|
||||||
|
for position in [1, args.len()] {
|
||||||
|
let mut strict_args = args.clone();
|
||||||
|
strict_args.insert(position, "--strict-runtime-paths".to_string());
|
||||||
|
let parsed = crate::cli::parse_command(&strict_args);
|
||||||
|
assert!(parsed.daemon_opts.strict_runtime_paths);
|
||||||
|
assert!(parse_daemon_args(&strict_args).strict_runtime_paths);
|
||||||
|
assert_eq!(parsed.config_path, "config.toml");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let args = vec![
|
||||||
|
"--strict-runtime-paths".to_string(),
|
||||||
|
"config.toml".to_string(),
|
||||||
|
];
|
||||||
|
let parsed = crate::cli::parse_command(&args);
|
||||||
|
assert_eq!(parsed.subcommand, crate::cli::Subcommand::Run);
|
||||||
|
assert!(parsed.daemon_opts.strict_runtime_paths);
|
||||||
|
assert_eq!(parsed.config_path, "config.toml");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn daemon_control_subprocess() {
|
||||||
|
let Some(pid_path) = std::env::var_os(CONTROL_PID) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let ready = PathBuf::from(std::env::var_os(CONTROL_READY).unwrap());
|
||||||
|
let reloaded = PathBuf::from(std::env::var_os(CONTROL_RELOADED).unwrap());
|
||||||
|
let mut terminate =
|
||||||
|
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).unwrap();
|
||||||
|
let mut reload =
|
||||||
|
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::hangup()).unwrap();
|
||||||
|
let mut owner = daemon::PidFile::new(PathBuf::from(pid_path), false);
|
||||||
|
owner.acquire().unwrap();
|
||||||
|
fs::write(&ready, b"ready").unwrap();
|
||||||
|
let deadline = tokio::time::sleep(Duration::from_secs(15));
|
||||||
|
tokio::pin!(deadline);
|
||||||
|
loop {
|
||||||
|
tokio::select! {
|
||||||
|
_ = terminate.recv() => break,
|
||||||
|
_ = reload.recv() => fs::write(&reloaded, b"reloaded").unwrap(),
|
||||||
|
_ = &mut deadline => panic!("daemon control subprocess timed out"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
owner.release().unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn control_commands_follow_runtime_path_policy() {
|
||||||
|
for (mode, linked_parent, trusted_parent) in [
|
||||||
|
(0o777, false, false),
|
||||||
|
(0o777, true, false),
|
||||||
|
(0o755, false, true),
|
||||||
|
] {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let real = root.path().join("run");
|
||||||
|
let linked = root.path().join("linked");
|
||||||
|
fs::create_dir(&real).unwrap();
|
||||||
|
fs::set_permissions(&real, fs::Permissions::from_mode(mode)).unwrap();
|
||||||
|
symlink(&real, &linked).unwrap();
|
||||||
|
let pid_path = if linked_parent { &linked } else { &real }.join("telemt.pid");
|
||||||
|
let ready = root.path().join("ready");
|
||||||
|
let reloaded = root.path().join("reloaded");
|
||||||
|
let mut child = ControlChild(
|
||||||
|
Command::new(std::env::current_exe().unwrap())
|
||||||
|
.args([
|
||||||
|
"--exact",
|
||||||
|
"cli::daemon_commands::tests::daemon_control_subprocess",
|
||||||
|
"--nocapture",
|
||||||
|
])
|
||||||
|
.env(CONTROL_PID, &pid_path)
|
||||||
|
.env(CONTROL_READY, &ready)
|
||||||
|
.env(CONTROL_RELOADED, &reloaded)
|
||||||
|
.stdout(Stdio::null())
|
||||||
|
.stderr(Stdio::null())
|
||||||
|
.spawn()
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
wait_for_file(&ready),
|
||||||
|
"daemon control subprocess did not become ready"
|
||||||
|
);
|
||||||
|
let command = |name: &str, strict: bool| {
|
||||||
|
let mut args = vec![
|
||||||
|
name.to_string(),
|
||||||
|
"--pid-file".to_string(),
|
||||||
|
pid_path.to_str().unwrap().to_string(),
|
||||||
|
];
|
||||||
|
if strict {
|
||||||
|
args.push("--strict-runtime-paths".to_string());
|
||||||
|
}
|
||||||
|
crate::cli::execute_subcommand(&crate::cli::parse_command(&args))
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(command("status", false), Some(0));
|
||||||
|
if trusted_parent {
|
||||||
|
assert_eq!(command("status", true), Some(0));
|
||||||
|
assert_eq!(command("reload", true), Some(0));
|
||||||
|
assert!(wait_for_file(&reloaded));
|
||||||
|
assert_eq!(command("stop", true), Some(0));
|
||||||
|
} else {
|
||||||
|
assert_eq!(command("status", true), Some(1));
|
||||||
|
assert_eq!(command("reload", true), Some(1));
|
||||||
|
assert!(!reloaded.exists());
|
||||||
|
assert_eq!(command("stop", true), Some(1));
|
||||||
|
assert_eq!(command("status", false), Some(0));
|
||||||
|
assert_eq!(command("reload", false), Some(0));
|
||||||
|
assert!(wait_for_file(&reloaded));
|
||||||
|
assert_eq!(command("stop", false), Some(0));
|
||||||
|
}
|
||||||
|
assert!(child.0.wait().unwrap().success());
|
||||||
|
assert!(!pid_path.exists());
|
||||||
|
assert_eq!(command("status", false), Some(1));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn status_does_not_remove_stale_pid_file() {
|
fn status_does_not_remove_stale_pid_file() {
|
||||||
let directory = tempfile::tempdir().unwrap();
|
let directory = tempfile::tempdir().unwrap();
|
||||||
let pid_file = directory.path().join("telemt.pid");
|
let pid_file = directory.path().join("telemt.pid");
|
||||||
fs::write(&pid_file, b"2000000000\n").unwrap();
|
fs::write(&pid_file, b"2000000000\n").unwrap();
|
||||||
|
|
||||||
assert_eq!(status(&pid_file), 1);
|
assert_eq!(status(&pid_file, false), 1);
|
||||||
assert!(pid_file.exists());
|
assert!(pid_file.exists());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -59,7 +59,7 @@ const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_BUDGET_PER_CORE: u16 = 16;
|
|||||||
const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS: u64 = 1000;
|
const DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS: u64 = 1000;
|
||||||
const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30;
|
const DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS: u64 = 30;
|
||||||
const DEFAULT_ACCEPT_PERMIT_TIMEOUT_MS: u64 = 250;
|
const DEFAULT_ACCEPT_PERMIT_TIMEOUT_MS: u64 = 250;
|
||||||
const DEFAULT_CONNTRACK_CONTROL_ENABLED: bool = true;
|
const DEFAULT_CONNTRACK_CONTROL_ENABLED: bool = false;
|
||||||
const DEFAULT_CONNTRACK_PRESSURE_HIGH_WATERMARK_PCT: u8 = 85;
|
const DEFAULT_CONNTRACK_PRESSURE_HIGH_WATERMARK_PCT: u8 = 85;
|
||||||
const DEFAULT_CONNTRACK_PRESSURE_LOW_WATERMARK_PCT: u8 = 70;
|
const DEFAULT_CONNTRACK_PRESSURE_LOW_WATERMARK_PCT: u8 = 70;
|
||||||
const DEFAULT_CONNTRACK_DELETE_BUDGET_PER_SEC: u64 = 4096;
|
const DEFAULT_CONNTRACK_DELETE_BUDGET_PER_SEC: u64 = 4096;
|
||||||
@@ -244,6 +244,7 @@ pub(crate) fn default_accept_permit_timeout_ms() -> u64 {
|
|||||||
DEFAULT_ACCEPT_PERMIT_TIMEOUT_MS
|
DEFAULT_ACCEPT_PERMIT_TIMEOUT_MS
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Keeps privileged conntrack control disabled unless explicitly requested.
|
||||||
pub(crate) fn default_conntrack_control_enabled() -> bool {
|
pub(crate) fn default_conntrack_control_enabled() -> bool {
|
||||||
DEFAULT_CONNTRACK_CONTROL_ENABLED
|
DEFAULT_CONNTRACK_CONTROL_ENABLED
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,7 @@
|
|||||||
//! | `network` | `dns_overrides` | Applied immediately |
|
//! | `network` | `dns_overrides` | Applied immediately |
|
||||||
//! | `access` | All user/quota fields | Effective immediately |
|
//! | `access` | All user/quota fields | Effective immediately |
|
||||||
//! | `web` | Carrier, timing, and debug policy | Applied to newly issued sessions |
|
//! | `web` | Carrier, timing, and debug policy | Applied to newly issued sessions |
|
||||||
|
//! | `web` | `carrier_method` | Applied to newly rendered bridge pages |
|
||||||
//! Fields that require re-binding sockets (`server.listeners`, legacy
|
//! Fields that require re-binding sockets (`server.listeners`, legacy
|
||||||
//! `server.port`, `censorship.*`, `network.*`, `use_middle_proxy`) are **not**
|
//! `server.port`, `censorship.*`, `network.*`, `use_middle_proxy`) are **not**
|
||||||
//! applied; a warning is emitted. SYN limiter rules are process-owned and are
|
//! applied; a warning is emitted. SYN limiter rules are process-owned and are
|
||||||
@@ -67,3 +68,11 @@ use watcher::{ReloadState, reload_config};
|
|||||||
mod base_path_tests;
|
mod base_path_tests;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests;
|
mod tests;
|
||||||
|
// Carrier method reloads preserve page-owned requests and process-owned limits.
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "hot_reload/carrier_method_tests.rs"]
|
||||||
|
mod carrier_method_tests;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "hot_reload/conveyor_tests.rs"]
|
||||||
|
mod conveyor_tests;
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
use crate::config::WebCarrierMethod;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn carrier_method_reload_is_hot_and_preserves_process_limits() {
|
||||||
|
let mut active = ProxyConfig::default();
|
||||||
|
for method in [WebCarrierMethod::Put, WebCarrierMethod::Post] {
|
||||||
|
let mut desired = active.clone();
|
||||||
|
desired.web.carrier_method = method;
|
||||||
|
assert_eq!(classify_config_changes(&active, &desired).changed, ["web"]);
|
||||||
|
assert!(!classify_config_changes(&active, &desired).restart_required);
|
||||||
|
desired.web.limits.max_http_connections += 1;
|
||||||
|
let applied = overlay_hot_fields(&active, &desired);
|
||||||
|
assert_eq!(applied.web.carrier_method, method);
|
||||||
|
assert_eq!(
|
||||||
|
applied.web.limits.max_http_connections,
|
||||||
|
active.web.limits.max_http_connections
|
||||||
|
);
|
||||||
|
active = applied;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn carrier_method_reload_publishes_both_directions_and_keeps_last_good_value() {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let path = directory.path().join("config.toml");
|
||||||
|
let source = |method| format!("[web]\ncarrier_method = \"{method}\"\n");
|
||||||
|
std::fs::write(&path, source("post")).unwrap();
|
||||||
|
let initial = Arc::new(ProxyConfig::load(&path).unwrap());
|
||||||
|
let initial_hash = ProxyConfig::load_with_metadata(&path)
|
||||||
|
.unwrap()
|
||||||
|
.rendered_hash;
|
||||||
|
let (config_tx, _config_rx) = watch::channel(Arc::clone(&initial));
|
||||||
|
let (log_tx, _log_rx) = watch::channel(initial.general.log_level.clone());
|
||||||
|
let mut reload_state = ReloadState::new(Some(initial_hash));
|
||||||
|
for (token, method) in [
|
||||||
|
("put", WebCarrierMethod::Put),
|
||||||
|
("post", WebCarrierMethod::Post),
|
||||||
|
] {
|
||||||
|
std::fs::write(&path, source(token)).unwrap();
|
||||||
|
reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state).unwrap();
|
||||||
|
let applied = config_tx.borrow().clone();
|
||||||
|
assert_eq!(applied.web.carrier_method, method);
|
||||||
|
std::fs::write(&path, source("PATCH")).unwrap();
|
||||||
|
reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state);
|
||||||
|
let unchanged = config_tx.borrow().clone();
|
||||||
|
assert!(Arc::ptr_eq(&unchanged, &applied));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn conveyor_reload_is_hot_in_both_directions_and_preserves_process_limits() {
|
||||||
|
let mut active = ProxyConfig::default();
|
||||||
|
for enabled in [false, true] {
|
||||||
|
let mut desired = active.clone();
|
||||||
|
desired.web.conveyor = enabled;
|
||||||
|
assert_eq!(classify_config_changes(&active, &desired).changed, ["web"]);
|
||||||
|
assert!(!classify_config_changes(&active, &desired).restart_required);
|
||||||
|
desired.web.limits.max_body_readers += 1;
|
||||||
|
let applied = overlay_hot_fields(&active, &desired);
|
||||||
|
assert_eq!(applied.web.conveyor, enabled);
|
||||||
|
assert_eq!(
|
||||||
|
applied.web.limits.max_body_readers,
|
||||||
|
active.web.limits.max_body_readers
|
||||||
|
);
|
||||||
|
active = applied;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -263,6 +263,8 @@ const LISTENER_CONFIG_KEYS: &[&str] = &[
|
|||||||
const WEB_CONFIG_KEYS: &[&str] = &[
|
const WEB_CONFIG_KEYS: &[&str] = &[
|
||||||
"enabled",
|
"enabled",
|
||||||
"carrier",
|
"carrier",
|
||||||
|
"carrier_method",
|
||||||
|
"conveyor",
|
||||||
"carriers",
|
"carriers",
|
||||||
"carrier_learning",
|
"carrier_learning",
|
||||||
"carrier_negotiation_aggressiveness",
|
"carrier_negotiation_aggressiveness",
|
||||||
|
|||||||
@@ -4,7 +4,8 @@ const WEB_DEBUG_RENDERERS: usize = 2;
|
|||||||
const WEB_DEBUG_STATUS_PAGE_BYTES: usize = 8 * 1024 * 1024;
|
const WEB_DEBUG_STATUS_PAGE_BYTES: usize = 8 * 1024 * 1024;
|
||||||
const WEB_DEBUG_GROUP_SCRATCH_BYTES: usize = 4 * 1024 * 1024;
|
const WEB_DEBUG_GROUP_SCRATCH_BYTES: usize = 4 * 1024 * 1024;
|
||||||
const WEB_CARRIER_LEARNING_ENTRY_BYTES: usize = 512;
|
const WEB_CARRIER_LEARNING_ENTRY_BYTES: usize = 512;
|
||||||
const WEB_LANE_STATE_BYTES: usize = 512;
|
// Includes the lane plus bounded conveyor slots, Notify ownership, and both hash tables.
|
||||||
|
const WEB_LANE_STATE_BYTES: usize = 1024;
|
||||||
const WEB_OVERLOAD_CONNECTION_BYTES: usize = 4 * 1024;
|
const WEB_OVERLOAD_CONNECTION_BYTES: usize = 4 * 1024;
|
||||||
// Each profile capability is stored in its vhost and in the global containment table.
|
// Each profile capability is stored in its vhost and in the global containment table.
|
||||||
const WEB_CAPABILITY_INDEX_ENTRY_BYTES: usize = 64;
|
const WEB_CAPABILITY_INDEX_ENTRY_BYTES: usize = 64;
|
||||||
|
|||||||
@@ -1,5 +1,71 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn conntrack_control_is_opt_in_by_default() {
|
||||||
|
assert!(!default_conntrack_control_enabled());
|
||||||
|
assert!(
|
||||||
|
!ProxyConfig::default()
|
||||||
|
.server
|
||||||
|
.conntrack_control
|
||||||
|
.inline_conntrack_control
|
||||||
|
);
|
||||||
|
let cfg = load_config_from_temp_toml(
|
||||||
|
r#"
|
||||||
|
[censorship]
|
||||||
|
tls_domain = "example.com"
|
||||||
|
|
||||||
|
[access.users]
|
||||||
|
user = "00000000000000000000000000000000"
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
assert!(!cfg.server.conntrack_control.inline_conntrack_control);
|
||||||
|
assert!(
|
||||||
|
!cfg.server
|
||||||
|
.conntrack_control
|
||||||
|
.inline_conntrack_control_explicit
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&cfg.server.conntrack_control).unwrap()["inline_conntrack_control"],
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn conntrack_control_preserves_explicit_enablement() {
|
||||||
|
for enabled in [false, true] {
|
||||||
|
let cfg = load_config_from_temp_toml(&format!(
|
||||||
|
r#"
|
||||||
|
[server.conntrack_control]
|
||||||
|
inline_conntrack_control = {enabled}
|
||||||
|
backend = "nftables"
|
||||||
|
|
||||||
|
[censorship]
|
||||||
|
tls_domain = "example.com"
|
||||||
|
|
||||||
|
[access.users]
|
||||||
|
user = "00000000000000000000000000000000"
|
||||||
|
"#,
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
cfg.server.conntrack_control.inline_conntrack_control,
|
||||||
|
enabled
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
cfg.server
|
||||||
|
.conntrack_control
|
||||||
|
.inline_conntrack_control_explicit
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
cfg.server.conntrack_control.backend,
|
||||||
|
ConntrackBackend::Nftables
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&cfg.server.conntrack_control).unwrap()["inline_conntrack_control"],
|
||||||
|
enabled,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn conntrack_pressure_high_watermark_out_of_range_is_rejected() {
|
fn conntrack_pressure_high_watermark_out_of_range_is_rejected() {
|
||||||
let toml = r#"
|
let toml = r#"
|
||||||
|
|||||||
@@ -3,6 +3,13 @@ use super::*;
|
|||||||
#[path = "web_tests/base_path_tests.rs"]
|
#[path = "web_tests/base_path_tests.rs"]
|
||||||
mod base_path_tests;
|
mod base_path_tests;
|
||||||
|
|
||||||
|
// Method selection must not alter capability identity or accepted config literals.
|
||||||
|
#[path = "web_tests/carrier_method_tests.rs"]
|
||||||
|
mod carrier_method_tests;
|
||||||
|
|
||||||
|
#[path = "web_tests/conveyor_tests.rs"]
|
||||||
|
mod conveyor_tests;
|
||||||
|
|
||||||
const WEB_CONFIG: &str = r#"
|
const WEB_CONFIG: &str = r#"
|
||||||
[access.users]
|
[access.users]
|
||||||
alice = "000102030405060708090a0b0c0d0e0f"
|
alice = "000102030405060708090a0b0c0d0e0f"
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn carrier_method_defaults_and_roundtrips_without_changing_capabilities() {
|
||||||
|
let default = load_config_from_temp_toml(WEB_CONFIG);
|
||||||
|
assert_eq!(default.web.carrier_method, WebCarrierMethod::Post);
|
||||||
|
let capabilities = default.web.runtime.as_ref().unwrap().capabilities.clone();
|
||||||
|
for (token, method) in [
|
||||||
|
("post", WebCarrierMethod::Post),
|
||||||
|
("put", WebCarrierMethod::Put),
|
||||||
|
] {
|
||||||
|
let configured = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
&format!("carrier = \"https-lanes\"\ncarrier_method = \"{token}\""),
|
||||||
|
);
|
||||||
|
let source = format!("[general]\nconfig_strict = true\n{configured}");
|
||||||
|
let config = load_config_from_temp_toml(&source);
|
||||||
|
assert_eq!(config.web.carrier_method, method);
|
||||||
|
assert_eq!(
|
||||||
|
config.web.runtime.as_ref().unwrap().capabilities,
|
||||||
|
capabilities
|
||||||
|
);
|
||||||
|
let json = serde_json::to_value(&config.web).unwrap();
|
||||||
|
assert_eq!(json["carrier_method"], token);
|
||||||
|
let decoded: WebConfig = serde_json::from_value(json).unwrap();
|
||||||
|
assert_eq!(decoded.carrier_method, method);
|
||||||
|
let serialized = toml::to_string(&config.web).unwrap();
|
||||||
|
let decoded: WebConfig = toml::from_str(&serialized).unwrap();
|
||||||
|
assert_eq!(decoded.carrier_method, method);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn carrier_method_rejects_unknown_tokens_types_and_aliases() {
|
||||||
|
for value in ["\"POST\"", "\"PUT\"", "\"patch\"", "true", "42", "[]"] {
|
||||||
|
let source = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
&format!("carrier = \"https-lanes\"\ncarrier_method = {value}"),
|
||||||
|
);
|
||||||
|
assert!(load_config_error_from_temp_toml(&source).contains("carrier_method"));
|
||||||
|
}
|
||||||
|
let configured = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
"carrier = \"https-lanes\"\nhttp_method = \"put\"",
|
||||||
|
);
|
||||||
|
let source = format!("[general]\nconfig_strict = true\n{configured}");
|
||||||
|
assert!(load_config_error_from_temp_toml(&source).contains("http_method"));
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn conveyor_defaults_on_and_roundtrips_without_changing_capabilities() {
|
||||||
|
let default = load_config_from_temp_toml(WEB_CONFIG);
|
||||||
|
assert!(default.web.conveyor);
|
||||||
|
assert!(ProxyConfig::default().web.conveyor);
|
||||||
|
let capabilities = default.web.runtime.as_ref().unwrap().capabilities.clone();
|
||||||
|
for enabled in [false, true] {
|
||||||
|
let configured = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
&format!("carrier = \"https-lanes\"\nconveyor = {enabled}"),
|
||||||
|
);
|
||||||
|
let config =
|
||||||
|
load_config_from_temp_toml(&format!("[general]\nconfig_strict = true\n{configured}",));
|
||||||
|
assert_eq!(config.web.conveyor, enabled);
|
||||||
|
assert_eq!(
|
||||||
|
config.web.runtime.as_ref().unwrap().capabilities,
|
||||||
|
capabilities
|
||||||
|
);
|
||||||
|
let json = serde_json::to_value(&config.web).unwrap();
|
||||||
|
assert_eq!(json["conveyor"], enabled);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::from_value::<WebConfig>(json).unwrap().conveyor,
|
||||||
|
enabled
|
||||||
|
);
|
||||||
|
let toml = toml::to_string(&config.web).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
toml::from_str::<WebConfig>(&toml).unwrap().conveyor,
|
||||||
|
enabled
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn conveyor_rejects_non_boolean_values() {
|
||||||
|
for value in ["\"true\"", "1", "0", "[]", "{}"] {
|
||||||
|
let configured = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
&format!("carrier = \"https-lanes\"\nconveyor = {value}"),
|
||||||
|
);
|
||||||
|
assert!(load_config_error_from_temp_toml(&configured).contains("conveyor"));
|
||||||
|
}
|
||||||
|
}
|
||||||
+1
-1
@@ -63,7 +63,7 @@ pub(crate) use web::{
|
|||||||
};
|
};
|
||||||
pub(crate) use web_carrier::WEB_CARRIER_LEARNING_MIN_ENTRIES;
|
pub(crate) use web_carrier::WEB_CARRIER_LEARNING_MIN_ENTRIES;
|
||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
pub use web_carrier::{WebCarrier, WebCarriers};
|
pub use web_carrier::{WebCarrier, WebCarrierMethod, WebCarriers};
|
||||||
pub(crate) use web_debug::web_debug_fits_limits;
|
pub(crate) use web_debug::web_debug_fits_limits;
|
||||||
pub use web_debug::{WebDebugBodyCapture, WebDebugConfig};
|
pub use web_debug::{WebDebugBodyCapture, WebDebugConfig};
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ use std::sync::Arc;
|
|||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use super::web_carrier::{WebCarrier, WebCarriers};
|
use super::web_carrier::{WebCarrier, WebCarrierMethod, WebCarriers};
|
||||||
use super::web_debug::WebDebugConfig;
|
use super::web_debug::WebDebugConfig;
|
||||||
|
|
||||||
// Serialized WEB defaults remain separate from the runtime data model.
|
// Serialized WEB defaults remain separate from the runtime data model.
|
||||||
@@ -413,6 +413,12 @@ pub struct WebConfig {
|
|||||||
/// Sole carrier when negotiation is disabled and final fallback when enabled.
|
/// Sole carrier when negotiation is disabled and final fallback when enabled.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub carrier: WebCarrier,
|
pub carrier: WebCarrier,
|
||||||
|
/// Page-owned HTTP method for HTTPS uplink and downlink, including retries.
|
||||||
|
#[serde(default)]
|
||||||
|
pub carrier_method: WebCarrierMethod,
|
||||||
|
/// Enables negotiated bounded HTTP uplink pipelining for new WEB sessions.
|
||||||
|
#[serde(default = "default_web_conveyor")]
|
||||||
|
pub conveyor: bool,
|
||||||
/// Ordered carriers considered by server-side negotiation before the fallback carrier.
|
/// Ordered carriers considered by server-side negotiation before the fallback carrier.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub carriers: WebCarriers,
|
pub carriers: WebCarriers,
|
||||||
@@ -471,6 +477,8 @@ impl Default for WebConfig {
|
|||||||
Self {
|
Self {
|
||||||
enabled: false,
|
enabled: false,
|
||||||
carrier: WebCarrier::default(),
|
carrier: WebCarrier::default(),
|
||||||
|
carrier_method: WebCarrierMethod::default(),
|
||||||
|
conveyor: default_web_conveyor(),
|
||||||
carriers: WebCarriers::default(),
|
carriers: WebCarriers::default(),
|
||||||
carrier_learning: default_web_carrier_learning(),
|
carrier_learning: default_web_carrier_learning(),
|
||||||
carrier_negotiation_aggressiveness: WebCarrierNegotiationAggressiveness::default(),
|
carrier_negotiation_aggressiveness: WebCarrierNegotiationAggressiveness::default(),
|
||||||
|
|||||||
@@ -2,6 +2,11 @@ pub(super) fn default_web_static_index() -> String {
|
|||||||
"index.html".to_string()
|
"index.html".to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Enables explicitly negotiated HTTP pipelining unless the operator disables it.
|
||||||
|
pub(super) fn default_web_conveyor() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
macro_rules! usize_default {
|
macro_rules! usize_default {
|
||||||
($name:ident, $value:expr) => {
|
($name:ident, $value:expr) => {
|
||||||
pub(super) fn $name() -> usize {
|
pub(super) fn $name() -> usize {
|
||||||
|
|||||||
@@ -3,6 +3,27 @@ use serde::{Deserialize, Serialize};
|
|||||||
/// Minimum restart-owned entries required for one complete learning sample.
|
/// Minimum restart-owned entries required for one complete learning sample.
|
||||||
pub(crate) const WEB_CARRIER_LEARNING_MIN_ENTRIES: usize = 3;
|
pub(crate) const WEB_CARRIER_LEARNING_MIN_ENTRIES: usize = 3;
|
||||||
|
|
||||||
|
/// HTTP method emitted by newly rendered HTTPS carrier bridges.
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
|
pub enum WebCarrierMethod {
|
||||||
|
/// Preserve the original bridge request method.
|
||||||
|
#[default]
|
||||||
|
Post,
|
||||||
|
/// Allow idempotent-method connection reuse in Linux WebKitGTK.
|
||||||
|
Put,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebCarrierMethod {
|
||||||
|
/// Returns the HTTP token embedded in the page-owned carrier policy.
|
||||||
|
pub(crate) const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Post => "POST",
|
||||||
|
Self::Put => "PUT",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Carrier selected for one newly issued WEB relay session.
|
/// Carrier selected for one newly issued WEB relay session.
|
||||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||||
#[serde(rename_all = "kebab-case")]
|
#[serde(rename_all = "kebab-case")]
|
||||||
|
|||||||
@@ -26,15 +26,21 @@ const RETRY_DELAYS: [Duration; 6] = [
|
|||||||
Duration::from_secs(30),
|
Duration::from_secs(30),
|
||||||
];
|
];
|
||||||
|
|
||||||
|
/// Reports whether a generation's desired firewall policy was confirmed.
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
pub(super) enum ReconcileOutcome {
|
pub(super) enum ReconcileOutcome {
|
||||||
|
/// The applied policy matches the accepted generation's desired policy.
|
||||||
Applied,
|
Applied,
|
||||||
|
/// The attempt failed without confirming the desired policy.
|
||||||
Failed,
|
Failed,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Associates a reconciliation result with its accepted runtime generation.
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
pub(super) struct ReconcileStatus {
|
pub(super) struct ReconcileStatus {
|
||||||
|
/// Runtime generation whose desired policy was attempted.
|
||||||
pub(super) generation: u64,
|
pub(super) generation: u64,
|
||||||
|
/// Result of the latest completed attempt for this generation.
|
||||||
pub(super) outcome: ReconcileOutcome,
|
pub(super) outcome: ReconcileOutcome,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -51,25 +57,14 @@ pub(crate) struct FirewallAuthority {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl FirewallAuthority {
|
impl FirewallAuthority {
|
||||||
/// Starts the single process-owned firewall reconciler.
|
/// Starts reconciliation only when enabled and CAP_NET_ADMIN is available.
|
||||||
pub(crate) fn spawn(control_plane: &ProcessControlPlane) -> Result<Self, String> {
|
pub(crate) fn spawn(
|
||||||
let (desired_tx, desired_rx) = watch::channel(None);
|
control_plane: &ProcessControlPlane,
|
||||||
let (status_tx, status_rx) = watch::channel(None);
|
config: &ProxyConfig,
|
||||||
let terminal = CancellationToken::new();
|
) -> Result<Option<Self>, String> {
|
||||||
let closed = Arc::new(AtomicBool::new(false));
|
let Some((authority, actor)) = Self::prepare(config, SystemCommandRunner) else {
|
||||||
let completed_flag = Arc::new(AtomicBool::new(false));
|
return Ok(None);
|
||||||
let cleanup_succeeded = Arc::new(AtomicBool::new(false));
|
};
|
||||||
let completed = Arc::new(Notify::new());
|
|
||||||
let actor = FirewallReconciler::new(
|
|
||||||
SystemCommandRunner,
|
|
||||||
desired_rx,
|
|
||||||
status_tx,
|
|
||||||
terminal.clone(),
|
|
||||||
Arc::clone(&closed),
|
|
||||||
Arc::clone(&completed_flag),
|
|
||||||
Arc::clone(&cleanup_succeeded),
|
|
||||||
Arc::clone(&completed),
|
|
||||||
);
|
|
||||||
control_plane
|
control_plane
|
||||||
.spawn_cooperative(move |process_cancellation| async move {
|
.spawn_cooperative(move |process_cancellation| async move {
|
||||||
actor.run(process_cancellation).await;
|
actor.run(process_cancellation).await;
|
||||||
@@ -78,7 +73,37 @@ impl FirewallAuthority {
|
|||||||
"process control-plane admission closed before conntrack firewall startup"
|
"process control-plane admission closed before conntrack firewall startup"
|
||||||
.to_string()
|
.to_string()
|
||||||
})?;
|
})?;
|
||||||
Ok(Self {
|
Ok(Some(authority))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Admission precedes allocation so an opted-out process never owns firewall cleanup.
|
||||||
|
// Keeping construction separate permits fake runners without changing async Send bounds.
|
||||||
|
fn prepare<R>(config: &ProxyConfig, runner: R) -> Option<(Self, FirewallReconciler<R>)>
|
||||||
|
where
|
||||||
|
R: FirewallCommandRunner + 'static,
|
||||||
|
{
|
||||||
|
if !config.server.conntrack_control.inline_conntrack_control || !runner.has_cap_net_admin()
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let (desired_tx, desired_rx) = watch::channel(None);
|
||||||
|
let (status_tx, status_rx) = watch::channel(None);
|
||||||
|
let terminal = CancellationToken::new();
|
||||||
|
let closed = Arc::new(AtomicBool::new(false));
|
||||||
|
let completed_flag = Arc::new(AtomicBool::new(false));
|
||||||
|
let cleanup_succeeded = Arc::new(AtomicBool::new(false));
|
||||||
|
let completed = Arc::new(Notify::new());
|
||||||
|
let actor = FirewallReconciler::new(
|
||||||
|
runner,
|
||||||
|
desired_rx,
|
||||||
|
status_tx,
|
||||||
|
terminal.clone(),
|
||||||
|
Arc::clone(&closed),
|
||||||
|
Arc::clone(&completed_flag),
|
||||||
|
Arc::clone(&cleanup_succeeded),
|
||||||
|
Arc::clone(&completed),
|
||||||
|
);
|
||||||
|
let authority = Self {
|
||||||
desired_tx,
|
desired_tx,
|
||||||
status_rx,
|
status_rx,
|
||||||
terminal,
|
terminal,
|
||||||
@@ -86,7 +111,8 @@ impl FirewallAuthority {
|
|||||||
completed_flag,
|
completed_flag,
|
||||||
cleanup_succeeded,
|
cleanup_succeeded,
|
||||||
completed,
|
completed,
|
||||||
})
|
};
|
||||||
|
Some((authority, actor))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Publishes policy only after its runtime generation becomes active.
|
/// Publishes policy only after its runtime generation becomes active.
|
||||||
@@ -169,6 +195,7 @@ impl Drop for CompletionGuard {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Serializes confirmed-state transitions and retains cleanup ownership until shutdown.
|
||||||
pub(super) struct FirewallReconciler<R> {
|
pub(super) struct FirewallReconciler<R> {
|
||||||
runner: R,
|
runner: R,
|
||||||
desired_rx: watch::Receiver<Option<DesiredState>>,
|
desired_rx: watch::Receiver<Option<DesiredState>>,
|
||||||
@@ -186,6 +213,7 @@ impl<R> FirewallReconciler<R>
|
|||||||
where
|
where
|
||||||
R: FirewallCommandRunner + 'static,
|
R: FirewallCommandRunner + 'static,
|
||||||
{
|
{
|
||||||
|
/// Constructs an admitted actor with unknown state pending startup recovery.
|
||||||
pub(super) fn new(
|
pub(super) fn new(
|
||||||
runner: R,
|
runner: R,
|
||||||
desired_rx: watch::Receiver<Option<DesiredState>>,
|
desired_rx: watch::Receiver<Option<DesiredState>>,
|
||||||
@@ -214,6 +242,7 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reconciles accepted generations and completes bounded cleanup on cancellation.
|
||||||
pub(super) async fn run(mut self, process_cancellation: CancellationToken) {
|
pub(super) async fn run(mut self, process_cancellation: CancellationToken) {
|
||||||
let mut current = None;
|
let mut current = None;
|
||||||
let mut retry_index = 0usize;
|
let mut retry_index = 0usize;
|
||||||
@@ -337,6 +366,7 @@ where
|
|||||||
let _completion = &self.completion;
|
let _completion = &self.completion;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Accepts fenced publications without replacing telemetry with stale desired state.
|
||||||
pub(super) fn take_latest_desired(&mut self) -> Option<DesiredState> {
|
pub(super) fn take_latest_desired(&mut self) -> Option<DesiredState> {
|
||||||
let next = self.desired_rx.borrow_and_update().clone()?;
|
let next = self.desired_rx.borrow_and_update().clone()?;
|
||||||
if next.generation < self.last_generation {
|
if next.generation < self.last_generation {
|
||||||
@@ -369,3 +399,8 @@ where
|
|||||||
Some(next)
|
Some(next)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Exercises startup admission and cleanup ownership with fake helper processes.
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "tests/startup_admission.rs"]
|
||||||
|
mod startup_admission_tests;
|
||||||
|
|||||||
@@ -10,14 +10,19 @@ use crate::util::trusted_command::{resolve_trusted_helper, trusted_helper_comman
|
|||||||
|
|
||||||
const COMMAND_TIMEOUT: Duration = Duration::from_secs(30);
|
const COMMAND_TIMEOUT: Duration = Duration::from_secs(30);
|
||||||
|
|
||||||
|
/// A privileged helper invocation with arguments kept separate from shell syntax.
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
pub(super) struct CommandSpec {
|
pub(super) struct CommandSpec {
|
||||||
|
/// Logical helper name resolved by the trusted executable policy.
|
||||||
pub(super) binary: &'static str,
|
pub(super) binary: &'static str,
|
||||||
|
/// Arguments passed directly to the helper process.
|
||||||
pub(super) args: Vec<String>,
|
pub(super) args: Vec<String>,
|
||||||
|
/// Optional restore script supplied on standard input.
|
||||||
pub(super) stdin: Option<String>,
|
pub(super) stdin: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CommandSpec {
|
impl CommandSpec {
|
||||||
|
/// Creates a helper invocation without an input script.
|
||||||
pub(super) fn new(binary: &'static str, args: impl IntoIterator<Item = &'static str>) -> Self {
|
pub(super) fn new(binary: &'static str, args: impl IntoIterator<Item = &'static str>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
binary,
|
binary,
|
||||||
@@ -26,6 +31,7 @@ impl CommandSpec {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Creates a helper invocation that receives a restore script.
|
||||||
pub(super) fn with_stdin(
|
pub(super) fn with_stdin(
|
||||||
binary: &'static str,
|
binary: &'static str,
|
||||||
args: impl IntoIterator<Item = &'static str>,
|
args: impl IntoIterator<Item = &'static str>,
|
||||||
@@ -39,22 +45,32 @@ impl CommandSpec {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Distinguishes idempotent absence from transaction and execution failures.
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
pub(super) enum CommandErrorKind {
|
pub(super) enum CommandErrorKind {
|
||||||
|
/// The trusted helper executable is unavailable.
|
||||||
Missing,
|
Missing,
|
||||||
|
/// The requested firewall object or rule is absent.
|
||||||
NotFound,
|
NotFound,
|
||||||
|
/// A terminal or process cancellation interrupted the invocation.
|
||||||
Cancelled,
|
Cancelled,
|
||||||
|
/// The helper exceeded its execution deadline.
|
||||||
Timeout,
|
Timeout,
|
||||||
|
/// A failure that must not be treated as successful cleanup.
|
||||||
Failed,
|
Failed,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A classified helper failure retaining its diagnostic for reconciliation.
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
pub(super) struct CommandError {
|
pub(super) struct CommandError {
|
||||||
|
/// Recovery semantics associated with the failure.
|
||||||
pub(super) kind: CommandErrorKind,
|
pub(super) kind: CommandErrorKind,
|
||||||
|
/// Original helper diagnostic or an execution failure description.
|
||||||
pub(super) message: String,
|
pub(super) message: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CommandError {
|
impl CommandError {
|
||||||
|
/// Creates the cancellation failure used by interruptible transactions.
|
||||||
pub(super) fn cancelled() -> Self {
|
pub(super) fn cancelled() -> Self {
|
||||||
Self {
|
Self {
|
||||||
kind: CommandErrorKind::Cancelled,
|
kind: CommandErrorKind::Cancelled,
|
||||||
@@ -62,6 +78,7 @@ impl CommandError {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Creates a failure that prevents idempotent cleanup from claiming success.
|
||||||
pub(super) fn failed(message: impl Into<String>) -> Self {
|
pub(super) fn failed(message: impl Into<String>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
kind: CommandErrorKind::Failed,
|
kind: CommandErrorKind::Failed,
|
||||||
@@ -76,14 +93,19 @@ impl std::fmt::Display for CommandError {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Executes firewall commands through a production or deterministic test runner.
|
||||||
pub(super) trait FirewallCommandRunner: Send + Sync {
|
pub(super) trait FirewallCommandRunner: Send + Sync {
|
||||||
|
/// Reports whether a helper can be resolved under the runner's trust policy.
|
||||||
fn available(&self, binary: &str) -> bool;
|
fn available(&self, binary: &str) -> bool;
|
||||||
|
|
||||||
|
/// Reports whether the process has the capability required to alter firewall rules.
|
||||||
fn has_cap_net_admin(&self) -> bool;
|
fn has_cap_net_admin(&self) -> bool;
|
||||||
|
|
||||||
|
/// Executes one invocation while preserving classified failure semantics.
|
||||||
async fn run(&self, spec: CommandSpec) -> Result<(), CommandError>;
|
async fn run(&self, spec: CommandSpec) -> Result<(), CommandError>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Runs trusted system helpers with bounded execution and captured diagnostics.
|
||||||
#[derive(Clone, Copy, Default)]
|
#[derive(Clone, Copy, Default)]
|
||||||
pub(super) struct SystemCommandRunner;
|
pub(super) struct SystemCommandRunner;
|
||||||
|
|
||||||
@@ -187,18 +209,95 @@ impl FirewallCommandRunner for SystemCommandRunner {
|
|||||||
} else {
|
} else {
|
||||||
stderr
|
stderr
|
||||||
};
|
};
|
||||||
let kind = if is_not_found_error(&message) {
|
let kind = classify_command_error(binary, &spec.args, &message);
|
||||||
CommandErrorKind::NotFound
|
|
||||||
} else {
|
|
||||||
CommandErrorKind::Failed
|
|
||||||
};
|
|
||||||
Err(CommandError { kind, message })
|
Err(CommandError { kind, message })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Recognizes the existing legacy iptables and native nftables absence formats.
|
||||||
pub(super) fn is_not_found_error(message: &str) -> bool {
|
pub(super) fn is_not_found_error(message: &str) -> bool {
|
||||||
message.contains("No chain/target/match by that name")
|
message.contains("No chain/target/match by that name")
|
||||||
|| message.contains("Bad rule (does a matching rule exist in that chain?)")
|
|| message.contains("Bad rule (does a matching rule exist in that chain?)")
|
||||||
|| message.contains("Could not process rule: No such file or directory")
|
|| message.contains("Could not process rule: No such file or directory")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Bounds additional iptables-nft absence diagnostics to owned cleanup and checks.
|
||||||
|
pub(super) fn classify_command_error(
|
||||||
|
binary: &str,
|
||||||
|
args: &[String],
|
||||||
|
message: &str,
|
||||||
|
) -> CommandErrorKind {
|
||||||
|
if is_not_found_error(message) || is_missing_owned_iptables_chain(binary, args, message) {
|
||||||
|
CommandErrorKind::NotFound
|
||||||
|
} else {
|
||||||
|
CommandErrorKind::Failed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_missing_owned_iptables_chain(binary: &str, args: &[String], message: &str) -> bool {
|
||||||
|
if !matches!(binary, "iptables" | "ip6tables") {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// An absent jump target is harmless for deletion, but not for installation.
|
||||||
|
let chain = match args {
|
||||||
|
[flag, table, operation, source, jump, target]
|
||||||
|
if flag == "-t"
|
||||||
|
&& table == "raw"
|
||||||
|
&& matches!(operation.as_str(), "-C" | "-D")
|
||||||
|
&& source == "PREROUTING"
|
||||||
|
&& jump == "-j"
|
||||||
|
&& target == "TELEMT_NOTRACK" =>
|
||||||
|
{
|
||||||
|
target.as_str()
|
||||||
|
}
|
||||||
|
[flag, table, operation, target]
|
||||||
|
if flag == "-t"
|
||||||
|
&& table == "raw"
|
||||||
|
&& matches!(operation.as_str(), "-F" | "-X")
|
||||||
|
&& matches!(
|
||||||
|
target.as_str(),
|
||||||
|
"TELEMT_NOTRACK" | "TELEMT_NT_A" | "TELEMT_NT_B"
|
||||||
|
) =>
|
||||||
|
{
|
||||||
|
target.as_str()
|
||||||
|
}
|
||||||
|
_ => return false,
|
||||||
|
};
|
||||||
|
let mut lines = message.lines();
|
||||||
|
let Some(first) = lines.next() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let diagnostic = if first.starts_with("Chain '") {
|
||||||
|
first
|
||||||
|
} else {
|
||||||
|
let Some((version, diagnostic)) = first
|
||||||
|
.strip_prefix(binary)
|
||||||
|
.and_then(|line| line.strip_prefix(" v"))
|
||||||
|
.and_then(|line| line.split_once(" (nf_tables): "))
|
||||||
|
else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if version.is_empty() || version.bytes().any(|byte| byte.is_ascii_whitespace()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
diagnostic
|
||||||
|
};
|
||||||
|
if diagnostic
|
||||||
|
.strip_prefix("Chain '")
|
||||||
|
.and_then(|line| line.strip_suffix("' does not exist"))
|
||||||
|
!= Some(chain)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// Reject mixed diagnostics instead of hiding another failure after an absence message.
|
||||||
|
let Some(help) = lines.next() else {
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
help.strip_prefix("Try `")
|
||||||
|
.and_then(|line| line.strip_prefix(binary))
|
||||||
|
.and_then(|line| line.strip_prefix(" -h' or '"))
|
||||||
|
.and_then(|line| line.strip_prefix(binary))
|
||||||
|
== Some(" --help' for more information.")
|
||||||
|
&& lines.next().is_none()
|
||||||
|
}
|
||||||
|
|||||||
@@ -20,6 +20,10 @@ use super::transaction::{InterruptibleRunner, reconcile_once};
|
|||||||
#[path = "tests/model_tests.rs"]
|
#[path = "tests/model_tests.rs"]
|
||||||
mod model_tests;
|
mod model_tests;
|
||||||
|
|
||||||
|
// Replays helper diagnostics through command classification and recovery.
|
||||||
|
#[path = "tests/recovery_errors.rs"]
|
||||||
|
mod recovery_errors;
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
struct FailureRule {
|
struct FailureRule {
|
||||||
binary: &'static str,
|
binary: &'static str,
|
||||||
|
|||||||
@@ -0,0 +1,512 @@
|
|||||||
|
use super::super::command::classify_command_error;
|
||||||
|
use super::super::transaction::recover_to_empty;
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
const OWNED_CHAINS: [&str; 3] = ["TELEMT_NOTRACK", "TELEMT_NT_A", "TELEMT_NT_B"];
|
||||||
|
type ChainKey = (&'static str, String, String);
|
||||||
|
type ChainRules = BTreeMap<ChainKey, BTreeSet<String>>;
|
||||||
|
|
||||||
|
fn missing_chain(binary: &str, chain: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"{binary} v1.8.10 (nf_tables): Chain '{chain}' does not exist\n\
|
||||||
|
Try `{binary} -h' or '{binary} --help' for more information."
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn classified_error(spec: &CommandSpec, message: &str) -> CommandError {
|
||||||
|
CommandError {
|
||||||
|
kind: classify_command_error(spec.binary, &spec.args, message),
|
||||||
|
message: message.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct FixtureState {
|
||||||
|
calls: Vec<CommandSpec>,
|
||||||
|
chains: ChainRules,
|
||||||
|
nft_tables: BTreeSet<String>,
|
||||||
|
failure: Option<&'static str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct FixtureRunner {
|
||||||
|
state: Arc<Mutex<FixtureState>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FixtureRunner {
|
||||||
|
fn new(failure: Option<&'static str>) -> Self {
|
||||||
|
let mut state = FixtureState {
|
||||||
|
failure,
|
||||||
|
..FixtureState::default()
|
||||||
|
};
|
||||||
|
for binary in ["iptables", "ip6tables"] {
|
||||||
|
for (table, chain, rules) in [
|
||||||
|
("raw", "PREROUTING", vec!["-j FOREIGN_RAW"]),
|
||||||
|
("raw", "FOREIGN_RAW", vec!["-j ACCEPT"]),
|
||||||
|
("filter", "INPUT", vec!["-j MTPR_SYNFIX", "-j TMT_SYN_TEST"]),
|
||||||
|
("filter", "MTPR_SYNFIX", vec!["-p tcp --syn -j DROP"]),
|
||||||
|
("filter", "TMT_SYN_TEST", vec!["-p tcp --syn -j DROP"]),
|
||||||
|
] {
|
||||||
|
state.chains.insert(
|
||||||
|
(binary, table.to_string(), chain.to_string()),
|
||||||
|
rules.into_iter().map(str::to_string).collect(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
state.nft_tables.extend([
|
||||||
|
"mtpr_synfix".to_string(),
|
||||||
|
"telemt_synlimit_test".to_string(),
|
||||||
|
"foreign_table".to_string(),
|
||||||
|
]);
|
||||||
|
Self {
|
||||||
|
state: Arc::new(Mutex::new(state)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn calls(&self) -> Vec<CommandSpec> {
|
||||||
|
self.state.lock().unwrap().calls.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn foreign_snapshot(&self) -> (ChainRules, BTreeSet<String>) {
|
||||||
|
let state = self.state.lock().unwrap();
|
||||||
|
let mut chains = state.chains.clone();
|
||||||
|
chains.retain(|(_, table, chain), _| {
|
||||||
|
table != "raw" || !OWNED_CHAINS.contains(&chain.as_str())
|
||||||
|
});
|
||||||
|
for ((_, table, chain), rules) in &mut chains {
|
||||||
|
if table == "raw" && chain == "PREROUTING" {
|
||||||
|
rules.remove("-j TELEMT_NOTRACK");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(chains, state.nft_tables.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_owned_policy(&self, enabled: bool) {
|
||||||
|
let state = self.state.lock().unwrap();
|
||||||
|
for (binary, ip) in [("iptables", "192.0.2.10"), ("ip6tables", "2001:db8::10")] {
|
||||||
|
for chain in OWNED_CHAINS {
|
||||||
|
let key = (binary, "raw".to_string(), chain.to_string());
|
||||||
|
if !enabled {
|
||||||
|
assert!(!state.chains.contains_key(&key));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let expected = match chain {
|
||||||
|
"TELEMT_NOTRACK" => BTreeSet::from(["-j TELEMT_NT_A".to_string()]),
|
||||||
|
"TELEMT_NT_A" => {
|
||||||
|
BTreeSet::from([format!("-p tcp --dport 443 -d {ip} -j CT --notrack")])
|
||||||
|
}
|
||||||
|
"TELEMT_NT_B" => BTreeSet::new(),
|
||||||
|
_ => unreachable!(),
|
||||||
|
};
|
||||||
|
assert_eq!(state.chains.get(&key), Some(&expected));
|
||||||
|
}
|
||||||
|
let prerouting = (binary, "raw".to_string(), "PREROUTING".to_string());
|
||||||
|
assert_eq!(
|
||||||
|
state.chains[&prerouting].contains("-j TELEMT_NOTRACK"),
|
||||||
|
enabled
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FirewallCommandRunner for FixtureRunner {
|
||||||
|
fn available(&self, _binary: &str) -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
fn has_cap_net_admin(&self) -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run(&self, spec: CommandSpec) -> Result<(), CommandError> {
|
||||||
|
let mut state = self.state.lock().unwrap();
|
||||||
|
state.calls.push(spec.clone());
|
||||||
|
if spec.binary == "nft" {
|
||||||
|
assert_eq!(&spec.args[..3], ["delete", "table", "inet"]);
|
||||||
|
return if state.nft_tables.remove(&spec.args[3]) {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(classified_error(
|
||||||
|
&spec,
|
||||||
|
"Error: Could not process rule: No such file or directory",
|
||||||
|
))
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if let Some(script) = &spec.stdin {
|
||||||
|
let binary = match spec.binary {
|
||||||
|
"iptables-restore" => "iptables",
|
||||||
|
"ip6tables-restore" => "ip6tables",
|
||||||
|
_ => panic!("unexpected restore helper"),
|
||||||
|
};
|
||||||
|
assert_eq!(spec.args, ["--noflush"]);
|
||||||
|
for line in script.lines().filter(|line| line.starts_with('-')) {
|
||||||
|
let fields = line.split_whitespace().collect::<Vec<_>>();
|
||||||
|
let key = (binary, "raw".to_string(), fields[1].to_string());
|
||||||
|
let rules = state.chains.get_mut(&key).expect("restore target exists");
|
||||||
|
match fields[0] {
|
||||||
|
"-F" => rules.clear(),
|
||||||
|
"-A" => {
|
||||||
|
rules.insert(fields[2..].join(" "));
|
||||||
|
}
|
||||||
|
_ => panic!("unexpected restore operation"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
assert!(matches!(spec.binary, "iptables" | "ip6tables"));
|
||||||
|
assert_eq!(spec.args[0], "-t");
|
||||||
|
let operation = spec.args[2].as_str();
|
||||||
|
let key = (spec.binary, spec.args[1].clone(), spec.args[3].clone());
|
||||||
|
match operation {
|
||||||
|
"-N" => {
|
||||||
|
if state.chains.contains_key(&key) {
|
||||||
|
Err(classified_error(&spec, "iptables: Chain already exists."))
|
||||||
|
} else {
|
||||||
|
state.chains.insert(key, BTreeSet::new());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"-C" | "-D" => {
|
||||||
|
if operation == "-D"
|
||||||
|
&& let Some(message) = state.failure.take()
|
||||||
|
{
|
||||||
|
return Err(classified_error(&spec, message));
|
||||||
|
}
|
||||||
|
let target = (spec.binary, spec.args[1].clone(), spec.args[5].clone());
|
||||||
|
if !state.chains.contains_key(&target) {
|
||||||
|
return Err(classified_error(
|
||||||
|
&spec,
|
||||||
|
&missing_chain(spec.binary, &spec.args[5]),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let rules = state.chains.get_mut(&key).expect("builtin chain exists");
|
||||||
|
let rule = spec.args[4..].join(" ");
|
||||||
|
if !rules.contains(&rule) {
|
||||||
|
return Err(classified_error(
|
||||||
|
&spec,
|
||||||
|
"Bad rule (does a matching rule exist in that chain?).",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if operation == "-D" {
|
||||||
|
rules.remove(&rule);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
"-I" => {
|
||||||
|
let target = (spec.binary, spec.args[1].clone(), spec.args[6].clone());
|
||||||
|
if !state.chains.contains_key(&target) {
|
||||||
|
return Err(classified_error(
|
||||||
|
&spec,
|
||||||
|
&missing_chain(spec.binary, &spec.args[6]),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
state
|
||||||
|
.chains
|
||||||
|
.get_mut(&key)
|
||||||
|
.unwrap()
|
||||||
|
.insert(spec.args[5..].join(" "));
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
"-F" | "-X" => {
|
||||||
|
let Some(rules) = state.chains.get_mut(&key) else {
|
||||||
|
return Err(classified_error(
|
||||||
|
&spec,
|
||||||
|
"iptables: No chain/target/match by that name.",
|
||||||
|
));
|
||||||
|
};
|
||||||
|
if operation == "-F" {
|
||||||
|
rules.clear();
|
||||||
|
} else {
|
||||||
|
assert!(rules.is_empty());
|
||||||
|
state.chains.remove(&key);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
_ => panic!("unexpected iptables operation"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn quoted_missing_chains_are_scoped_to_owned_commands() {
|
||||||
|
for binary in ["iptables", "ip6tables"] {
|
||||||
|
for operation in ["-C", "-D"] {
|
||||||
|
let spec = CommandSpec::new(
|
||||||
|
binary,
|
||||||
|
["-t", "raw", operation, "PREROUTING", "-j", "TELEMT_NOTRACK"],
|
||||||
|
);
|
||||||
|
let message = missing_chain(binary, "TELEMT_NOTRACK");
|
||||||
|
let error = classified_error(&spec, &message);
|
||||||
|
assert_eq!(error.kind, CommandErrorKind::NotFound, "{spec:?}");
|
||||||
|
assert_eq!(error.message, message);
|
||||||
|
}
|
||||||
|
for operation in ["-F", "-X"] {
|
||||||
|
for chain in OWNED_CHAINS {
|
||||||
|
let spec = CommandSpec::new(binary, ["-t", "raw", operation, chain]);
|
||||||
|
assert_eq!(
|
||||||
|
classified_error(&spec, &missing_chain(binary, chain)).kind,
|
||||||
|
CommandErrorKind::NotFound,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let spec = CommandSpec::new(binary, ["-t", "raw", "-F", "TELEMT_NOTRACK"]);
|
||||||
|
for message in [
|
||||||
|
"Chain 'TELEMT_NOTRACK' does not exist".to_string(),
|
||||||
|
format!("{binary} v1.8.11 (nf_tables): Chain 'TELEMT_NOTRACK' does not exist"),
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
classified_error(&spec, &message).kind,
|
||||||
|
CommandErrorKind::NotFound
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unrelated_commands_and_diagnostics_remain_failures() {
|
||||||
|
let message = missing_chain("iptables", "TELEMT_NOTRACK");
|
||||||
|
for spec in [
|
||||||
|
CommandSpec::new("iptables", ["-t", "raw", "-N", "TELEMT_NOTRACK"]),
|
||||||
|
CommandSpec::new(
|
||||||
|
"iptables",
|
||||||
|
["-t", "raw", "-A", "PREROUTING", "-j", "TELEMT_NOTRACK"],
|
||||||
|
),
|
||||||
|
CommandSpec::new(
|
||||||
|
"iptables",
|
||||||
|
["-t", "raw", "-I", "PREROUTING", "1", "-j", "TELEMT_NOTRACK"],
|
||||||
|
),
|
||||||
|
CommandSpec::new("iptables", ["-t", "filter", "-F", "TELEMT_NOTRACK"]),
|
||||||
|
CommandSpec::new("iptables", ["-t", "raw", "-F", "TELEMT_NT_A"]),
|
||||||
|
CommandSpec::new(
|
||||||
|
"iptables",
|
||||||
|
["-t", "raw", "-D", "PREROUTING", "-j", "TELEMT_NT_A"],
|
||||||
|
),
|
||||||
|
CommandSpec::new(
|
||||||
|
"iptables",
|
||||||
|
["-t", "raw", "-D", "INPUT", "-j", "TELEMT_NOTRACK"],
|
||||||
|
),
|
||||||
|
CommandSpec::new(
|
||||||
|
"iptables",
|
||||||
|
["-t", "raw", "-D", "PREROUTING", "-g", "TELEMT_NOTRACK"],
|
||||||
|
),
|
||||||
|
CommandSpec::new("iptables", ["-F", "TELEMT_NOTRACK"]),
|
||||||
|
CommandSpec::new(
|
||||||
|
"iptables",
|
||||||
|
["-t", "raw", "-F", "TELEMT_NOTRACK", "--invalid"],
|
||||||
|
),
|
||||||
|
CommandSpec::new("iptables-restore", ["--noflush"]),
|
||||||
|
CommandSpec::new("ip6tables-restore", ["--noflush"]),
|
||||||
|
CommandSpec::new("nft", ["delete", "table", "inet", "telemt_conntrack"]),
|
||||||
|
CommandSpec::new("conntrack", ["-D"]),
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
classified_error(&spec, &message).kind,
|
||||||
|
CommandErrorKind::Failed,
|
||||||
|
"{spec:?}",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for chain in [
|
||||||
|
"TELEMT_NT_C",
|
||||||
|
"TELEMT_NOTRACK_EXTRA",
|
||||||
|
"MTPR_SYNFIX",
|
||||||
|
"PREROUTING",
|
||||||
|
] {
|
||||||
|
let spec = CommandSpec::new("iptables", ["-t", "raw", "-F", chain]);
|
||||||
|
assert_eq!(
|
||||||
|
classified_error(&spec, &missing_chain("iptables", chain)).kind,
|
||||||
|
CommandErrorKind::Failed,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let spec = CommandSpec::new(
|
||||||
|
"iptables",
|
||||||
|
["-t", "raw", "-D", "PREROUTING", "-j", "TELEMT_NOTRACK"],
|
||||||
|
);
|
||||||
|
for message in [
|
||||||
|
"Permission denied",
|
||||||
|
"can't initialize iptables table `raw': Table does not exist",
|
||||||
|
"Another app is currently holding the xtables lock",
|
||||||
|
"Chain is not empty",
|
||||||
|
"Directory not empty",
|
||||||
|
"Can't delete chain with references left",
|
||||||
|
"unknown option --invalid",
|
||||||
|
"command timed out",
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
classified_error(&spec, message).kind,
|
||||||
|
CommandErrorKind::Failed
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for message in [
|
||||||
|
format!("{message}\nPermission denied"),
|
||||||
|
format!("Permission denied\n{message}"),
|
||||||
|
"Chain 'TELEMT_NOTRACK' does not exist\nPermission denied".to_string(),
|
||||||
|
"Chain 'TELEMT_NOTRACK' does not exist\n\
|
||||||
|
Try `ip6tables -h' or 'ip6tables --help' for more information."
|
||||||
|
.to_string(),
|
||||||
|
"iptables v1.8.10 (legacy): Chain 'TELEMT_NOTRACK' does not exist".to_string(),
|
||||||
|
"iptables v (nf_tables): Chain 'TELEMT_NOTRACK' does not exist".to_string(),
|
||||||
|
missing_chain("ip6tables", "TELEMT_NOTRACK"),
|
||||||
|
missing_chain("iptables", "TELEMT_NT_A"),
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
classified_error(&spec, &message).kind,
|
||||||
|
CommandErrorKind::Failed
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for binary in ["nft", "conntrack", "iptables-restore", "ip6tables-restore"] {
|
||||||
|
let spec = CommandSpec::new(binary, ["-t", "raw", "-F", "TELEMT_NOTRACK"]);
|
||||||
|
assert_eq!(
|
||||||
|
classified_error(&spec, "Chain 'TELEMT_NOTRACK' does not exist").kind,
|
||||||
|
CommandErrorKind::Failed,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unknown_recovery_converges_without_touching_foreign_rules() {
|
||||||
|
for policy in [DesiredPolicy::Empty, dual_stack_policy(443)] {
|
||||||
|
let runner = FixtureRunner::new(None);
|
||||||
|
let foreign = runner.foreign_snapshot();
|
||||||
|
let mut applied = AppliedState::Unknown;
|
||||||
|
let requested = desired(1, policy);
|
||||||
|
reconcile_once(&runner, &runner, &mut applied, &requested)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
&applied,
|
||||||
|
AppliedState::Known(plan) if plan.matches_policy(&requested.policy)
|
||||||
|
));
|
||||||
|
runner.assert_owned_policy(matches!(requested.policy, DesiredPolicy::Rules { .. }));
|
||||||
|
assert_eq!(runner.foreign_snapshot(), foreign);
|
||||||
|
for binary in ["iptables", "ip6tables"] {
|
||||||
|
let calls = runner.calls();
|
||||||
|
assert_eq!(
|
||||||
|
calls
|
||||||
|
.iter()
|
||||||
|
.filter(|call| call.binary == binary && call.args[2] == "-D")
|
||||||
|
.count(),
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
calls
|
||||||
|
.iter()
|
||||||
|
.filter(|call| {
|
||||||
|
call.binary == binary && matches!(call.args[2].as_str(), "-F" | "-X")
|
||||||
|
})
|
||||||
|
.count(),
|
||||||
|
6,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let calls = runner.calls().len();
|
||||||
|
reconcile_once(
|
||||||
|
&runner,
|
||||||
|
&runner,
|
||||||
|
&mut applied,
|
||||||
|
&desired(2, requested.policy.clone()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(runner.calls().len(), calls);
|
||||||
|
recover_to_empty(&runner).await.unwrap();
|
||||||
|
recover_to_empty(&runner).await.unwrap();
|
||||||
|
runner.assert_owned_policy(false);
|
||||||
|
assert_eq!(runner.foreign_snapshot(), foreign);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn genuine_recovery_failures_leave_state_unknown_and_do_not_install() {
|
||||||
|
for message in [
|
||||||
|
"Permission denied",
|
||||||
|
"can't initialize iptables table `raw': Table does not exist",
|
||||||
|
"Another app is currently holding the xtables lock",
|
||||||
|
"unknown firewall failure",
|
||||||
|
] {
|
||||||
|
let runner = FixtureRunner::new(Some(message));
|
||||||
|
let mut applied = AppliedState::Unknown;
|
||||||
|
let error = reconcile_once(
|
||||||
|
&runner,
|
||||||
|
&runner,
|
||||||
|
&mut applied,
|
||||||
|
&desired(1, dual_stack_policy(443)),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.message.contains(message));
|
||||||
|
assert_eq!(applied, AppliedState::Unknown);
|
||||||
|
assert!(runner.calls().iter().all(|call| call.stdin.is_none()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn actor_converges_then_idles_and_preserves_foreign_rules_on_shutdown() {
|
||||||
|
for failure in [None, Some("Permission denied")] {
|
||||||
|
let runner = FixtureRunner::new(failure);
|
||||||
|
let observed = runner.clone();
|
||||||
|
let foreign = observed.foreign_snapshot();
|
||||||
|
let (desired_tx, desired_rx) = watch::channel(None);
|
||||||
|
let (status_tx, mut status_rx) = watch::channel(None);
|
||||||
|
let terminal = CancellationToken::new();
|
||||||
|
let closed = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||||
|
let completed = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||||
|
let cleanup = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||||
|
let reconciler = FirewallReconciler::new(
|
||||||
|
runner,
|
||||||
|
desired_rx,
|
||||||
|
status_tx,
|
||||||
|
terminal.clone(),
|
||||||
|
closed.clone(),
|
||||||
|
completed.clone(),
|
||||||
|
cleanup.clone(),
|
||||||
|
Arc::new(Notify::new()),
|
||||||
|
);
|
||||||
|
let task = tokio::spawn(reconciler.run(CancellationToken::new()));
|
||||||
|
let requested = desired(1, dual_stack_policy(443));
|
||||||
|
let stats = requested.stats.clone();
|
||||||
|
desired_tx.send_replace(Some(requested));
|
||||||
|
status_rx.changed().await.unwrap();
|
||||||
|
if failure.is_some() {
|
||||||
|
assert_eq!(
|
||||||
|
status_rx.borrow().as_ref().unwrap().outcome,
|
||||||
|
ReconcileOutcome::Failed,
|
||||||
|
);
|
||||||
|
assert!(!stats.get_conntrack_rule_apply_ok());
|
||||||
|
assert_eq!(stats.get_conntrack_rule_reconcile_error_total(), 1);
|
||||||
|
let calls = observed.calls().len();
|
||||||
|
tokio::time::advance(Duration::from_millis(999)).await;
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
assert_eq!(observed.calls().len(), calls);
|
||||||
|
tokio::time::advance(Duration::from_millis(1)).await;
|
||||||
|
status_rx.changed().await.unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
status_rx.borrow().as_ref().unwrap().outcome,
|
||||||
|
ReconcileOutcome::Applied,
|
||||||
|
);
|
||||||
|
assert_eq!(status_rx.borrow().as_ref().unwrap().generation, 1);
|
||||||
|
assert!(stats.get_conntrack_rule_apply_ok());
|
||||||
|
assert_eq!(stats.get_conntrack_rule_reconcile_success_total(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
stats.get_conntrack_rule_reconcile_error_total(),
|
||||||
|
u64::from(failure.is_some()),
|
||||||
|
);
|
||||||
|
let calls = observed.calls().len();
|
||||||
|
tokio::time::advance(Duration::from_secs(90)).await;
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
assert_eq!(observed.calls().len(), calls);
|
||||||
|
assert!(!status_rx.has_changed().unwrap());
|
||||||
|
observed.assert_owned_policy(true);
|
||||||
|
assert_eq!(observed.foreign_snapshot(), foreign);
|
||||||
|
terminal.cancel();
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), task)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(closed.load(Ordering::Acquire));
|
||||||
|
assert!(completed.load(Ordering::Acquire));
|
||||||
|
assert!(cleanup.load(Ordering::Acquire));
|
||||||
|
assert!(!stats.get_conntrack_rule_apply_ok());
|
||||||
|
observed.assert_owned_policy(false);
|
||||||
|
assert_eq!(observed.foreign_snapshot(), foreign);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,267 @@
|
|||||||
|
use std::sync::Mutex;
|
||||||
|
use std::sync::atomic::AtomicUsize;
|
||||||
|
|
||||||
|
use crate::config::{ConntrackBackend, ConntrackMode};
|
||||||
|
|
||||||
|
use super::super::command::{CommandErrorKind, CommandSpec, classify_command_error};
|
||||||
|
use super::super::transaction::reconcile_once;
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
const NETLINK_ERROR: &str =
|
||||||
|
"src/mnl.c:68: Unable to initialize Netlink socket: Address family not supported by protocol";
|
||||||
|
const PERMISSION_ERROR: &str = "iptables v1.8.11 (nf_tables): Could not fetch rule set generation id: Permission denied (you must be root)";
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct StartupRunner {
|
||||||
|
has_cap_net_admin: bool,
|
||||||
|
cap_probes: Arc<AtomicUsize>,
|
||||||
|
calls: Arc<Mutex<Vec<CommandSpec>>>,
|
||||||
|
failure: Option<&'static str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StartupRunner {
|
||||||
|
fn new(has_cap_net_admin: bool, failure: Option<&'static str>) -> Self {
|
||||||
|
Self {
|
||||||
|
has_cap_net_admin,
|
||||||
|
cap_probes: Arc::new(AtomicUsize::new(0)),
|
||||||
|
calls: Arc::new(Mutex::new(Vec::new())),
|
||||||
|
failure,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn calls(&self) -> Vec<CommandSpec> {
|
||||||
|
self.calls.lock().unwrap().clone()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FirewallCommandRunner for StartupRunner {
|
||||||
|
fn available(&self, _binary: &str) -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
fn has_cap_net_admin(&self) -> bool {
|
||||||
|
self.cap_probes.fetch_add(1, Ordering::Relaxed);
|
||||||
|
self.has_cap_net_admin
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run(&self, spec: CommandSpec) -> Result<(), CommandError> {
|
||||||
|
self.calls.lock().unwrap().push(spec.clone());
|
||||||
|
if let Some(message) = self.failure {
|
||||||
|
return Err(CommandError {
|
||||||
|
kind: classify_command_error(spec.binary, &spec.args, message),
|
||||||
|
message: message.to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (spec.binary == "nft" && spec.args.first().map(String::as_str) == Some("delete"))
|
||||||
|
|| (matches!(spec.binary, "iptables" | "ip6tables")
|
||||||
|
&& matches!(
|
||||||
|
spec.args.get(2).map(String::as_str),
|
||||||
|
Some("-C" | "-D" | "-F" | "-X")
|
||||||
|
))
|
||||||
|
{
|
||||||
|
return Err(CommandError {
|
||||||
|
kind: CommandErrorKind::NotFound,
|
||||||
|
message: "injected absent owned object".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn start_with_runner(
|
||||||
|
config: &ProxyConfig,
|
||||||
|
runner: StartupRunner,
|
||||||
|
scope: &ProcessControlPlane,
|
||||||
|
) -> Option<FirewallAuthority> {
|
||||||
|
let (authority, actor) = FirewallAuthority::prepare(config, runner)?;
|
||||||
|
assert!(
|
||||||
|
scope
|
||||||
|
.spawn_cooperative(move |cancellation| actor.run(cancellation))
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
Some(authority)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn disabled_startup_never_acquires_cleanup_ownership() {
|
||||||
|
for explicit in [false, true] {
|
||||||
|
for has_cap in [true, false] {
|
||||||
|
for backend in [
|
||||||
|
ConntrackBackend::Auto,
|
||||||
|
ConntrackBackend::Nftables,
|
||||||
|
ConntrackBackend::Iptables,
|
||||||
|
] {
|
||||||
|
for mode in [
|
||||||
|
ConntrackMode::Tracked,
|
||||||
|
ConntrackMode::Notrack,
|
||||||
|
ConntrackMode::Hybrid,
|
||||||
|
] {
|
||||||
|
for message in [NETLINK_ERROR, PERMISSION_ERROR] {
|
||||||
|
let mut config = ProxyConfig::default();
|
||||||
|
if explicit {
|
||||||
|
config.server.conntrack_control.inline_conntrack_control = false;
|
||||||
|
config
|
||||||
|
.server
|
||||||
|
.conntrack_control
|
||||||
|
.inline_conntrack_control_explicit = true;
|
||||||
|
}
|
||||||
|
config.server.conntrack_control.backend = backend;
|
||||||
|
config.server.conntrack_control.mode = mode;
|
||||||
|
let runner = StartupRunner::new(has_cap, Some(message));
|
||||||
|
let scope = ProcessControlPlane::new();
|
||||||
|
let authority = start_with_runner(&config, runner.clone(), &scope);
|
||||||
|
assert!(
|
||||||
|
authority.is_none(),
|
||||||
|
"{explicit} {has_cap} {backend:?} {mode:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(runner.cap_probes.load(Ordering::Relaxed), 0);
|
||||||
|
tokio::time::advance(Duration::from_secs(90)).await;
|
||||||
|
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||||
|
assert!(runner.calls().is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn enabled_startup_without_capability_creates_no_firewall_work() {
|
||||||
|
let mut config = ProxyConfig::default();
|
||||||
|
config.server.conntrack_control.inline_conntrack_control = true;
|
||||||
|
for mode in [ConntrackMode::Tracked, ConntrackMode::Notrack] {
|
||||||
|
config.server.conntrack_control.mode = mode;
|
||||||
|
let runner = StartupRunner::new(false, Some(PERMISSION_ERROR));
|
||||||
|
let scope = ProcessControlPlane::new();
|
||||||
|
assert!(start_with_runner(&config, runner.clone(), &scope).is_none());
|
||||||
|
assert_eq!(runner.cap_probes.load(Ordering::Relaxed), 1);
|
||||||
|
tokio::time::advance(Duration::from_secs(90)).await;
|
||||||
|
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||||
|
assert!(runner.calls().is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn admitted_startup_preserves_recovery_idle_and_shutdown_cleanup() {
|
||||||
|
for mode in [ConntrackMode::Tracked, ConntrackMode::Notrack] {
|
||||||
|
let mut config = ProxyConfig::default();
|
||||||
|
config.server.conntrack_control.inline_conntrack_control = true;
|
||||||
|
config.server.conntrack_control.mode = mode;
|
||||||
|
config.server.conntrack_control.backend = ConntrackBackend::Nftables;
|
||||||
|
let runner = StartupRunner::new(true, None);
|
||||||
|
let scope = ProcessControlPlane::new();
|
||||||
|
let authority = start_with_runner(&config, runner.clone(), &scope).unwrap();
|
||||||
|
let stats = Arc::new(Stats::new());
|
||||||
|
assert!(
|
||||||
|
authority
|
||||||
|
.publish_initial(1, Arc::new(config), stats.clone())
|
||||||
|
.await
|
||||||
|
);
|
||||||
|
assert!(stats.get_conntrack_rule_apply_ok());
|
||||||
|
for binary in ["nft", "iptables", "ip6tables"] {
|
||||||
|
assert!(runner.calls().iter().any(|call| call.binary == binary));
|
||||||
|
}
|
||||||
|
let calls = runner.calls().len();
|
||||||
|
tokio::time::advance(Duration::from_secs(90)).await;
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
assert_eq!(runner.calls().len(), calls);
|
||||||
|
assert!(authority.shutdown_and_clear().await);
|
||||||
|
assert!(!stats.get_conntrack_rule_apply_ok());
|
||||||
|
assert!(runner.calls().len() > calls);
|
||||||
|
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn admitted_process_cancellation_preserves_cleanup() {
|
||||||
|
let mut config = ProxyConfig::default();
|
||||||
|
config.server.conntrack_control.inline_conntrack_control = true;
|
||||||
|
let runner = StartupRunner::new(true, None);
|
||||||
|
let scope = ProcessControlPlane::new();
|
||||||
|
let authority = start_with_runner(&config, runner.clone(), &scope).unwrap();
|
||||||
|
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||||
|
assert!(authority.shutdown_and_clear().await);
|
||||||
|
assert!(!runner.calls().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn netlink_and_permission_errors_are_not_classified_as_absence() {
|
||||||
|
for spec in [
|
||||||
|
CommandSpec::new("nft", ["delete", "table", "inet", "telemt_conntrack"]),
|
||||||
|
CommandSpec::new("nft", ["-f", "-"]),
|
||||||
|
CommandSpec::new(
|
||||||
|
"iptables",
|
||||||
|
["-t", "raw", "-D", "PREROUTING", "-j", "TELEMT_NOTRACK"],
|
||||||
|
),
|
||||||
|
CommandSpec::new("iptables", ["-t", "raw", "-F", "TELEMT_NT_A"]),
|
||||||
|
CommandSpec::new("ip6tables", ["-t", "raw", "-X", "TELEMT_NT_B"]),
|
||||||
|
CommandSpec::new("iptables-restore", ["--noflush"]),
|
||||||
|
CommandSpec::new("ip6tables-restore", ["--noflush"]),
|
||||||
|
] {
|
||||||
|
for message in [
|
||||||
|
NETLINK_ERROR,
|
||||||
|
PERMISSION_ERROR,
|
||||||
|
"iptables: Failed to initialize nft: Address family not supported by protocol",
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
classify_command_error(spec.binary, &spec.args, message),
|
||||||
|
CommandErrorKind::Failed,
|
||||||
|
"{spec:?} {message}",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn admitted_genuine_failures_remain_unknown_and_retry() {
|
||||||
|
for message in [NETLINK_ERROR, PERMISSION_ERROR] {
|
||||||
|
let mut config = ProxyConfig::default();
|
||||||
|
config.server.conntrack_control.inline_conntrack_control = true;
|
||||||
|
config.server.conntrack_control.mode = ConntrackMode::Notrack;
|
||||||
|
config.server.conntrack_control.backend = ConntrackBackend::Nftables;
|
||||||
|
let runner = StartupRunner::new(true, Some(message));
|
||||||
|
let mut applied = AppliedState::Unknown;
|
||||||
|
let requested = DesiredState {
|
||||||
|
generation: 1,
|
||||||
|
policy: DesiredPolicy::from_config(&config),
|
||||||
|
stats: Arc::new(Stats::new()),
|
||||||
|
};
|
||||||
|
let error = reconcile_once(&runner, &runner, &mut applied, &requested)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.message.contains(message));
|
||||||
|
assert_eq!(applied, AppliedState::Unknown);
|
||||||
|
assert!(runner.calls().iter().all(|call| call.stdin.is_none()));
|
||||||
|
|
||||||
|
let scope = ProcessControlPlane::new();
|
||||||
|
let authority = start_with_runner(&config, runner.clone(), &scope).unwrap();
|
||||||
|
let stats = Arc::new(Stats::new());
|
||||||
|
assert!(
|
||||||
|
!authority
|
||||||
|
.publish_initial(1, Arc::new(config), stats.clone())
|
||||||
|
.await
|
||||||
|
);
|
||||||
|
assert!(!stats.get_conntrack_rule_apply_ok());
|
||||||
|
assert_eq!(stats.get_conntrack_rule_reconcile_error_total(), 1);
|
||||||
|
let mut status = authority.status_rx.clone();
|
||||||
|
assert_eq!(
|
||||||
|
status.borrow_and_update().as_ref().unwrap().outcome,
|
||||||
|
ReconcileOutcome::Failed,
|
||||||
|
);
|
||||||
|
let calls = runner.calls().len();
|
||||||
|
tokio::time::advance(Duration::from_millis(999)).await;
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
assert_eq!(runner.calls().len(), calls);
|
||||||
|
tokio::time::advance(Duration::from_millis(1)).await;
|
||||||
|
status.changed().await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
status.borrow().as_ref().unwrap().outcome,
|
||||||
|
ReconcileOutcome::Failed,
|
||||||
|
);
|
||||||
|
assert_eq!(stats.get_conntrack_rule_reconcile_error_total(), 2);
|
||||||
|
assert!(!authority.shutdown_and_clear().await);
|
||||||
|
assert!(authority.completed_flag.load(Ordering::Acquire));
|
||||||
|
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||||
|
assert!(runner.calls().iter().all(|call| call.stdin.is_none()));
|
||||||
|
}
|
||||||
|
}
|
||||||
+8
-3
@@ -9,7 +9,7 @@ use std::os::unix::fs::OpenOptionsExt;
|
|||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use nix::errno::Errno;
|
use nix::errno::Errno;
|
||||||
use nix::unistd::{self, ForkResult, Gid, Uid, chdir, close, fork, getpid, setsid};
|
use nix::unistd::{self, ForkResult, Gid, Uid, chdir, fork, getpid, setsid};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
// PID file ownership and process-control helpers.
|
// PID file ownership and process-control helpers.
|
||||||
@@ -29,6 +29,8 @@ pub struct DaemonOptions {
|
|||||||
pub daemonize: bool,
|
pub daemonize: bool,
|
||||||
/// Path to PID file.
|
/// Path to PID file.
|
||||||
pub pid_file: Option<PathBuf>,
|
pub pid_file: Option<PathBuf>,
|
||||||
|
/// Require trusted, symlink-free PID and log parents. Disabled by default for compatibility.
|
||||||
|
pub strict_runtime_paths: bool,
|
||||||
/// User to run as after binding sockets.
|
/// User to run as after binding sockets.
|
||||||
pub user: Option<String>,
|
pub user: Option<String>,
|
||||||
/// Group to run as after binding sockets.
|
/// Group to run as after binding sockets.
|
||||||
@@ -174,8 +176,10 @@ fn redirect_stdio_to_devnull() -> Result<(), DaemonError> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if devnull_fd > 2 {
|
// Keep stdio descriptors open; other source descriptors are closed once by File's Drop.
|
||||||
let _ = close(devnull_fd);
|
// Transfer ownership only after all dup2 calls succeed so errors retain RAII cleanup.
|
||||||
|
if devnull_fd <= 2 {
|
||||||
|
let _ = std::os::unix::io::IntoRawFd::into_raw_fd(devnull);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -332,6 +336,7 @@ mod tests {
|
|||||||
fn test_daemon_options_default() {
|
fn test_daemon_options_default() {
|
||||||
let opts = DaemonOptions::default();
|
let opts = DaemonOptions::default();
|
||||||
assert!(!opts.daemonize);
|
assert!(!opts.daemonize);
|
||||||
|
assert!(!opts.strict_runtime_paths);
|
||||||
assert!(!opts.should_daemonize());
|
assert!(!opts.should_daemonize());
|
||||||
assert_eq!(opts.pid_file_path(), Path::new(DEFAULT_PID_FILE));
|
assert_eq!(opts.pid_file_path(), Path::new(DEFAULT_PID_FILE));
|
||||||
}
|
}
|
||||||
|
|||||||
+34
-21
@@ -17,6 +17,7 @@ use crate::util::secure_fs::AnchoredPath;
|
|||||||
/// PID file manager backed by a persistent sibling lock file.
|
/// PID file manager backed by a persistent sibling lock file.
|
||||||
pub struct PidFile {
|
pub struct PidFile {
|
||||||
path: PathBuf,
|
path: PathBuf,
|
||||||
|
strict_runtime_paths: bool,
|
||||||
lock_path: PathBuf,
|
lock_path: PathBuf,
|
||||||
pid_file: Option<File>,
|
pid_file: Option<File>,
|
||||||
pid_identity: Option<FileIdentity>,
|
pid_identity: Option<FileIdentity>,
|
||||||
@@ -40,12 +41,13 @@ impl FileIdentity {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl PidFile {
|
impl PidFile {
|
||||||
/// Creates a new PID file manager for the given path.
|
/// Creates a PID manager with explicit parent-path policy; `false` allows legacy parents.
|
||||||
pub fn new<P: AsRef<Path>>(path: P) -> Self {
|
pub fn new<P: AsRef<Path>>(path: P, strict_runtime_paths: bool) -> Self {
|
||||||
let path = normalize_pid_path(path.as_ref());
|
let path = normalize_pid_path(path.as_ref());
|
||||||
let lock_path = sibling_lock_path(&path);
|
let lock_path = sibling_lock_path(&path);
|
||||||
Self {
|
Self {
|
||||||
path,
|
path,
|
||||||
|
strict_runtime_paths,
|
||||||
lock_path,
|
lock_path,
|
||||||
pid_file: None,
|
pid_file: None,
|
||||||
pid_identity: None,
|
pid_identity: None,
|
||||||
@@ -56,7 +58,7 @@ impl PidFile {
|
|||||||
|
|
||||||
/// Checks whether the PID file names a running process without modifying either file.
|
/// Checks whether the PID file names a running process without modifying either file.
|
||||||
pub fn check_running(&self) -> Result<Option<i32>, DaemonError> {
|
pub fn check_running(&self) -> Result<Option<i32>, DaemonError> {
|
||||||
let Some(pid) = read_pid_file_if_exists(&self.path)? else {
|
let Some(pid) = read_pid_file_if_exists(&self.path, self.strict_runtime_paths)? else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
Ok(is_process_running(pid).then_some(pid))
|
Ok(is_process_running(pid).then_some(pid))
|
||||||
@@ -67,9 +69,10 @@ impl PidFile {
|
|||||||
/// Fails if another owner holds the lock or the existing PID names a running process.
|
/// Fails if another owner holds the lock or the existing PID names a running process.
|
||||||
pub fn acquire(&mut self) -> Result<(), DaemonError> {
|
pub fn acquire(&mut self) -> Result<(), DaemonError> {
|
||||||
let anchor =
|
let anchor =
|
||||||
AnchoredPath::open_trusted_parent_or_create(&self.path, 0o755).map_err(|error| {
|
AnchoredPath::open_runtime_parent(&self.path, Some(0o755), self.strict_runtime_paths)
|
||||||
|
.map_err(|error| {
|
||||||
DaemonError::PidFile(format!(
|
DaemonError::PidFile(format!(
|
||||||
"cannot open trusted parent for {}: {}",
|
"cannot open PID parent for {}: {}",
|
||||||
self.path.display(),
|
self.path.display(),
|
||||||
error
|
error
|
||||||
))
|
))
|
||||||
@@ -245,13 +248,16 @@ fn open_file_at(anchor: &AnchoredPath, name: &OsStr, flags: OFlag, mode: u32) ->
|
|||||||
Ok(File::from(descriptor))
|
Ok(File::from(descriptor))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn read_pid_file_if_exists(path: &Path) -> Result<Option<i32>, DaemonError> {
|
fn read_pid_file_if_exists(
|
||||||
let anchor = match AnchoredPath::open_trusted_parent(path) {
|
path: &Path,
|
||||||
|
strict_runtime_paths: bool,
|
||||||
|
) -> Result<Option<i32>, DaemonError> {
|
||||||
|
let anchor = match AnchoredPath::open_runtime_parent(path, None, strict_runtime_paths) {
|
||||||
Ok(anchor) => anchor,
|
Ok(anchor) => anchor,
|
||||||
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None),
|
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None),
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
return Err(DaemonError::PidFile(format!(
|
return Err(DaemonError::PidFile(format!(
|
||||||
"cannot open trusted parent for {}: {}",
|
"cannot open PID parent for {}: {}",
|
||||||
path.display(),
|
path.display(),
|
||||||
error
|
error
|
||||||
)));
|
)));
|
||||||
@@ -352,11 +358,14 @@ fn validate_regular_single_link(file: &File, path: &Path) -> Result<fs::Metadata
|
|||||||
Ok(metadata)
|
Ok(metadata)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reads a PID from a PID file.
|
/// Reads a PID using the selected parent-path policy; `false` allows legacy parents.
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub fn read_pid_file<P: AsRef<Path>>(path: P) -> Result<i32, DaemonError> {
|
pub fn read_pid_file<P: AsRef<Path>>(
|
||||||
|
path: P,
|
||||||
|
strict_runtime_paths: bool,
|
||||||
|
) -> Result<i32, DaemonError> {
|
||||||
let path = normalize_pid_path(path.as_ref());
|
let path = normalize_pid_path(path.as_ref());
|
||||||
read_pid_file_if_exists(&path)?.ok_or_else(|| {
|
read_pid_file_if_exists(&path, strict_runtime_paths)?.ok_or_else(|| {
|
||||||
DaemonError::PidFile(format!(
|
DaemonError::PidFile(format!(
|
||||||
"cannot read {}: file does not exist",
|
"cannot read {}: file does not exist",
|
||||||
path.display()
|
path.display()
|
||||||
@@ -364,17 +373,18 @@ pub fn read_pid_file<P: AsRef<Path>>(path: P) -> Result<i32, DaemonError> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Sends a signal to the process specified in a PID file.
|
/// Signals a lock-owning process using the same parent-path policy for PID and lock files.
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub fn signal_pid_file<P: AsRef<Path>>(
|
pub fn signal_pid_file<P: AsRef<Path>>(
|
||||||
path: P,
|
path: P,
|
||||||
signal: nix::sys::signal::Signal,
|
signal: nix::sys::signal::Signal,
|
||||||
|
strict_runtime_paths: bool,
|
||||||
) -> Result<(), DaemonError> {
|
) -> Result<(), DaemonError> {
|
||||||
let path = normalize_pid_path(path.as_ref());
|
let path = normalize_pid_path(path.as_ref());
|
||||||
let pid = read_pid_file(&path)?;
|
let pid = read_pid_file(&path, strict_runtime_paths)?;
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
let pidfd = open_pidfd(pid)?;
|
let pidfd = open_pidfd(pid)?;
|
||||||
if !daemon_lock_is_held(&path)? {
|
if !daemon_lock_is_held(&path, strict_runtime_paths)? {
|
||||||
return Err(DaemonError::PidFile(format!(
|
return Err(DaemonError::PidFile(format!(
|
||||||
"refusing to signal unlocked or stale PID file {}",
|
"refusing to signal unlocked or stale PID file {}",
|
||||||
path.display()
|
path.display()
|
||||||
@@ -399,12 +409,15 @@ pub enum DaemonStatus {
|
|||||||
NotRunning,
|
NotRunning,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Checks daemon status without modifying the PID or lock file.
|
/// Checks daemon status read-only, applying the selected policy to both parent lookups.
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub fn check_status<P: AsRef<Path>>(path: P) -> DaemonStatus {
|
pub fn check_status<P: AsRef<Path>>(path: P, strict_runtime_paths: bool) -> DaemonStatus {
|
||||||
let path = normalize_pid_path(path.as_ref());
|
let path = normalize_pid_path(path.as_ref());
|
||||||
match read_pid_file_if_exists(&path) {
|
match read_pid_file_if_exists(&path, strict_runtime_paths) {
|
||||||
Ok(Some(pid)) if daemon_lock_is_held(&path).unwrap_or(false) && is_process_running(pid) => {
|
Ok(Some(pid))
|
||||||
|
if daemon_lock_is_held(&path, strict_runtime_paths).unwrap_or(false)
|
||||||
|
&& is_process_running(pid) =>
|
||||||
|
{
|
||||||
DaemonStatus::Running(pid)
|
DaemonStatus::Running(pid)
|
||||||
}
|
}
|
||||||
Ok(Some(pid)) => DaemonStatus::Stale(pid),
|
Ok(Some(pid)) => DaemonStatus::Stale(pid),
|
||||||
@@ -412,14 +425,14 @@ pub fn check_status<P: AsRef<Path>>(path: P) -> DaemonStatus {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn daemon_lock_is_held(path: &Path) -> Result<bool, DaemonError> {
|
fn daemon_lock_is_held(path: &Path, strict_runtime_paths: bool) -> Result<bool, DaemonError> {
|
||||||
let lock_path = sibling_lock_path(path);
|
let lock_path = sibling_lock_path(path);
|
||||||
let anchor = match AnchoredPath::open_trusted_parent(path) {
|
let anchor = match AnchoredPath::open_runtime_parent(path, None, strict_runtime_paths) {
|
||||||
Ok(anchor) => anchor,
|
Ok(anchor) => anchor,
|
||||||
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(false),
|
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(false),
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
return Err(DaemonError::PidFile(format!(
|
return Err(DaemonError::PidFile(format!(
|
||||||
"cannot open trusted parent for {}: {}",
|
"cannot open PID parent for {}: {}",
|
||||||
path.display(),
|
path.display(),
|
||||||
error
|
error
|
||||||
)));
|
)));
|
||||||
|
|||||||
+132
-23
@@ -38,6 +38,107 @@ fn pid_file_remains_send_and_sync() {
|
|||||||
assert_send_sync::<PidFile>();
|
assert_send_sync::<PidFile>();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn compatibility_pid_lifecycle_accepts_writable_parent_directories() {
|
||||||
|
use std::os::unix::fs::PermissionsExt;
|
||||||
|
|
||||||
|
for mode in [0o770, 0o777, 0o1777] {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
fs::set_permissions(directory.path(), fs::Permissions::from_mode(mode)).unwrap();
|
||||||
|
let pid_path = directory.path().join("telemt.pid");
|
||||||
|
let mut strict_owner = PidFile::new(&pid_path, true);
|
||||||
|
assert!(strict_owner.acquire().is_err());
|
||||||
|
assert!(!pid_path.exists());
|
||||||
|
assert!(!sibling_lock_path(&pid_path).exists());
|
||||||
|
let mut owner = PidFile::new(&pid_path, false);
|
||||||
|
owner.acquire().unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
read_pid_file(&pid_path, false).unwrap(),
|
||||||
|
std::process::id() as i32
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
owner.check_running().unwrap(),
|
||||||
|
Some(std::process::id() as i32)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
check_status(&pid_path, false),
|
||||||
|
DaemonStatus::Running(std::process::id() as i32)
|
||||||
|
);
|
||||||
|
assert!(read_pid_file(&pid_path, true).is_err());
|
||||||
|
assert!(strict_owner.check_running().is_err());
|
||||||
|
assert_eq!(check_status(&pid_path, true), DaemonStatus::NotRunning);
|
||||||
|
|
||||||
|
owner.release().unwrap();
|
||||||
|
assert!(!pid_path.exists());
|
||||||
|
assert!(sibling_lock_path(&pid_path).exists());
|
||||||
|
assert_eq!(check_status(&pid_path, false), DaemonStatus::NotRunning);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn compatibility_pid_lifecycle_follows_symlinked_parents() {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let real = directory.path().join("tmp");
|
||||||
|
let linked = directory.path().join("var");
|
||||||
|
fs::create_dir(&real).unwrap();
|
||||||
|
symlink("tmp", &linked).unwrap();
|
||||||
|
let pid_path = linked.join("run/telemt.pid");
|
||||||
|
let mut strict_owner = PidFile::new(&pid_path, true);
|
||||||
|
assert!(strict_owner.acquire().is_err());
|
||||||
|
assert!(!real.join("run").exists());
|
||||||
|
let mut owner = PidFile::new(&pid_path, false);
|
||||||
|
owner.acquire().unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
read_pid_file(&pid_path, false).unwrap(),
|
||||||
|
std::process::id() as i32
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
check_status(&pid_path, false),
|
||||||
|
DaemonStatus::Running(std::process::id() as i32)
|
||||||
|
);
|
||||||
|
assert!(real.join("run/telemt.pid.lock").exists());
|
||||||
|
assert!(read_pid_file(&pid_path, true).is_err());
|
||||||
|
assert_eq!(check_status(&pid_path, true), DaemonStatus::NotRunning);
|
||||||
|
|
||||||
|
owner.release().unwrap();
|
||||||
|
assert!(!real.join("run/telemt.pid").exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn private_pid_parent_supports_both_policies() {
|
||||||
|
for strict_runtime_paths in [false, true] {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let pid_path = directory.path().join("nested/run/telemt.pid");
|
||||||
|
let mut owner = PidFile::new(&pid_path, strict_runtime_paths);
|
||||||
|
owner.acquire().unwrap();
|
||||||
|
|
||||||
|
for read_strict in [false, true] {
|
||||||
|
assert_eq!(
|
||||||
|
read_pid_file(&pid_path, read_strict).unwrap(),
|
||||||
|
std::process::id() as i32
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
check_status(&pid_path, read_strict),
|
||||||
|
DaemonStatus::Running(std::process::id() as i32)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
fs::metadata(pid_path.parent().unwrap()).unwrap().mode() & 0o777,
|
||||||
|
0o755
|
||||||
|
);
|
||||||
|
let mut contender = PidFile::new(&pid_path, strict_runtime_paths);
|
||||||
|
assert!(matches!(
|
||||||
|
contender.acquire(),
|
||||||
|
Err(DaemonError::AlreadyRunning(_))
|
||||||
|
));
|
||||||
|
owner.release().unwrap();
|
||||||
|
contender.acquire().unwrap();
|
||||||
|
contender.release().unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn system_var_run_alias_keeps_the_default_pid_path_usable() {
|
fn system_var_run_alias_keeps_the_default_pid_path_usable() {
|
||||||
let Ok(metadata) = fs::symlink_metadata("/var/run") else {
|
let Ok(metadata) = fs::symlink_metadata("/var/run") else {
|
||||||
@@ -52,7 +153,7 @@ fn system_var_run_alias_keeps_the_default_pid_path_usable() {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let pid_file = PidFile::new("/var/run/telemt.pid");
|
let pid_file = PidFile::new("/var/run/telemt.pid", false);
|
||||||
|
|
||||||
assert_eq!(pid_file.path(), Path::new("/run/telemt.pid"));
|
assert_eq!(pid_file.path(), Path::new("/run/telemt.pid"));
|
||||||
}
|
}
|
||||||
@@ -64,7 +165,7 @@ fn lock_holder_subprocess() {
|
|||||||
};
|
};
|
||||||
let ready_path = PathBuf::from(std::env::var_os(HELPER_READY_PATH).unwrap());
|
let ready_path = PathBuf::from(std::env::var_os(HELPER_READY_PATH).unwrap());
|
||||||
let stop_path = PathBuf::from(std::env::var_os(HELPER_STOP_PATH).unwrap());
|
let stop_path = PathBuf::from(std::env::var_os(HELPER_STOP_PATH).unwrap());
|
||||||
let mut pid_file = PidFile::new(PathBuf::from(pid_path));
|
let mut pid_file = PidFile::new(PathBuf::from(pid_path), false);
|
||||||
pid_file.acquire().unwrap();
|
pid_file.acquire().unwrap();
|
||||||
fs::write(&ready_path, b"ready").unwrap();
|
fs::write(&ready_path, b"ready").unwrap();
|
||||||
assert!(wait_for_path(&stop_path, Duration::from_secs(10)));
|
assert!(wait_for_path(&stop_path, Duration::from_secs(10)));
|
||||||
@@ -100,7 +201,7 @@ fn persistent_sibling_lock_serializes_processes_after_pid_unlink() {
|
|||||||
let lock_inode = fs::metadata(&lock_path).unwrap().ino();
|
let lock_inode = fs::metadata(&lock_path).unwrap().ino();
|
||||||
fs::remove_file(&pid_path).unwrap();
|
fs::remove_file(&pid_path).unwrap();
|
||||||
|
|
||||||
let mut contender = PidFile::new(&pid_path);
|
let mut contender = PidFile::new(&pid_path, false);
|
||||||
assert!(contender.acquire().is_err());
|
assert!(contender.acquire().is_err());
|
||||||
|
|
||||||
fs::write(&stop_path, b"stop").unwrap();
|
fs::write(&stop_path, b"stop").unwrap();
|
||||||
@@ -123,10 +224,13 @@ fn stale_pid_checks_are_read_only() {
|
|||||||
let directory = tempfile::tempdir().unwrap();
|
let directory = tempfile::tempdir().unwrap();
|
||||||
let pid_path = directory.path().join("telemt.pid");
|
let pid_path = directory.path().join("telemt.pid");
|
||||||
fs::write(&pid_path, b"2000000000\n").unwrap();
|
fs::write(&pid_path, b"2000000000\n").unwrap();
|
||||||
let pid_file = PidFile::new(&pid_path);
|
let pid_file = PidFile::new(&pid_path, false);
|
||||||
|
|
||||||
assert_eq!(pid_file.check_running().unwrap(), None);
|
assert_eq!(pid_file.check_running().unwrap(), None);
|
||||||
assert_eq!(check_status(&pid_path), DaemonStatus::Stale(2_000_000_000));
|
assert_eq!(
|
||||||
|
check_status(&pid_path, false),
|
||||||
|
DaemonStatus::Stale(2_000_000_000)
|
||||||
|
);
|
||||||
assert!(pid_path.exists());
|
assert!(pid_path.exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -136,15 +240,15 @@ fn status_requires_live_lock_ownership() {
|
|||||||
let pid_path = directory.path().join("telemt.pid");
|
let pid_path = directory.path().join("telemt.pid");
|
||||||
fs::write(&pid_path, format!("{}\n", std::process::id())).unwrap();
|
fs::write(&pid_path, format!("{}\n", std::process::id())).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
check_status(&pid_path),
|
check_status(&pid_path, false),
|
||||||
DaemonStatus::Stale(std::process::id() as i32)
|
DaemonStatus::Stale(std::process::id() as i32)
|
||||||
);
|
);
|
||||||
|
|
||||||
fs::remove_file(&pid_path).unwrap();
|
fs::remove_file(&pid_path).unwrap();
|
||||||
let mut owner = PidFile::new(&pid_path);
|
let mut owner = PidFile::new(&pid_path, false);
|
||||||
owner.acquire().unwrap();
|
owner.acquire().unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
check_status(&pid_path),
|
check_status(&pid_path, false),
|
||||||
DaemonStatus::Running(std::process::id() as i32)
|
DaemonStatus::Running(std::process::id() as i32)
|
||||||
);
|
);
|
||||||
owner.release().unwrap();
|
owner.release().unwrap();
|
||||||
@@ -155,7 +259,7 @@ fn unowned_release_does_not_remove_pid_file() {
|
|||||||
let directory = tempfile::tempdir().unwrap();
|
let directory = tempfile::tempdir().unwrap();
|
||||||
let pid_path = directory.path().join("telemt.pid");
|
let pid_path = directory.path().join("telemt.pid");
|
||||||
fs::write(&pid_path, b"2000000000\n").unwrap();
|
fs::write(&pid_path, b"2000000000\n").unwrap();
|
||||||
let mut pid_file = PidFile::new(&pid_path);
|
let mut pid_file = PidFile::new(&pid_path, false);
|
||||||
|
|
||||||
pid_file.release().unwrap();
|
pid_file.release().unwrap();
|
||||||
|
|
||||||
@@ -169,10 +273,11 @@ fn acquire_rejects_pid_symlink_without_truncating_target() {
|
|||||||
let target_path = directory.path().join("target");
|
let target_path = directory.path().join("target");
|
||||||
fs::write(&target_path, b"preserve\n").unwrap();
|
fs::write(&target_path, b"preserve\n").unwrap();
|
||||||
symlink(&target_path, &pid_path).unwrap();
|
symlink(&target_path, &pid_path).unwrap();
|
||||||
let mut pid_file = PidFile::new(&pid_path);
|
for strict_runtime_paths in [false, true] {
|
||||||
|
let mut pid_file = PidFile::new(&pid_path, strict_runtime_paths);
|
||||||
assert!(pid_file.acquire().is_err());
|
assert!(pid_file.acquire().is_err());
|
||||||
assert_eq!(fs::read(&target_path).unwrap(), b"preserve\n");
|
assert_eq!(fs::read(&target_path).unwrap(), b"preserve\n");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -182,10 +287,11 @@ fn acquire_rejects_pid_hard_link_without_truncating_target() {
|
|||||||
let target_path = directory.path().join("target");
|
let target_path = directory.path().join("target");
|
||||||
fs::write(&target_path, b"preserve\n").unwrap();
|
fs::write(&target_path, b"preserve\n").unwrap();
|
||||||
fs::hard_link(&target_path, &pid_path).unwrap();
|
fs::hard_link(&target_path, &pid_path).unwrap();
|
||||||
let mut pid_file = PidFile::new(&pid_path);
|
for strict_runtime_paths in [false, true] {
|
||||||
|
let mut pid_file = PidFile::new(&pid_path, strict_runtime_paths);
|
||||||
assert!(pid_file.acquire().is_err());
|
assert!(pid_file.acquire().is_err());
|
||||||
assert_eq!(fs::read(&target_path).unwrap(), b"preserve\n");
|
assert_eq!(fs::read(&target_path).unwrap(), b"preserve\n");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -196,7 +302,7 @@ fn acquire_rejects_symlinked_parent_without_publishing_outside() {
|
|||||||
fs::create_dir(&real_parent).unwrap();
|
fs::create_dir(&real_parent).unwrap();
|
||||||
symlink(&real_parent, &linked_parent).unwrap();
|
symlink(&real_parent, &linked_parent).unwrap();
|
||||||
let pid_path = linked_parent.join("telemt.pid");
|
let pid_path = linked_parent.join("telemt.pid");
|
||||||
let mut pid_file = PidFile::new(&pid_path);
|
let mut pid_file = PidFile::new(&pid_path, true);
|
||||||
|
|
||||||
assert!(pid_file.acquire().is_err());
|
assert!(pid_file.acquire().is_err());
|
||||||
assert!(!real_parent.join("telemt.pid").exists());
|
assert!(!real_parent.join("telemt.pid").exists());
|
||||||
@@ -210,7 +316,7 @@ fn release_remains_anchored_after_parent_path_replacement() {
|
|||||||
let moved_parent = directory.path().join("moved");
|
let moved_parent = directory.path().join("moved");
|
||||||
fs::create_dir(&active_parent).unwrap();
|
fs::create_dir(&active_parent).unwrap();
|
||||||
let pid_path = active_parent.join("telemt.pid");
|
let pid_path = active_parent.join("telemt.pid");
|
||||||
let mut pid_file = PidFile::new(&pid_path);
|
let mut pid_file = PidFile::new(&pid_path, false);
|
||||||
pid_file.acquire().unwrap();
|
pid_file.acquire().unwrap();
|
||||||
|
|
||||||
fs::rename(&active_parent, &moved_parent).unwrap();
|
fs::rename(&active_parent, &moved_parent).unwrap();
|
||||||
@@ -231,7 +337,7 @@ fn release_does_not_remove_replacement_path() {
|
|||||||
let directory = tempfile::tempdir().unwrap();
|
let directory = tempfile::tempdir().unwrap();
|
||||||
let pid_path = directory.path().join("telemt.pid");
|
let pid_path = directory.path().join("telemt.pid");
|
||||||
let owned_path = directory.path().join("owned.pid");
|
let owned_path = directory.path().join("owned.pid");
|
||||||
let mut pid_file = PidFile::new(&pid_path);
|
let mut pid_file = PidFile::new(&pid_path, false);
|
||||||
pid_file.acquire().unwrap();
|
pid_file.acquire().unwrap();
|
||||||
fs::rename(&pid_path, &owned_path).unwrap();
|
fs::rename(&pid_path, &owned_path).unwrap();
|
||||||
fs::write(&pid_path, b"replacement\n").unwrap();
|
fs::write(&pid_path, b"replacement\n").unwrap();
|
||||||
@@ -253,7 +359,7 @@ fn pid_parser_rejects_process_group_values() {
|
|||||||
|
|
||||||
for value in ["-1\n", "0\n", "1\n"] {
|
for value in ["-1\n", "0\n", "1\n"] {
|
||||||
fs::write(&pid_path, value).unwrap();
|
fs::write(&pid_path, value).unwrap();
|
||||||
assert!(read_pid_file(&pid_path).is_err());
|
assert!(read_pid_file(&pid_path, false).is_err());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -262,7 +368,7 @@ fn pid_file_release_keeps_lock_inode() {
|
|||||||
let directory = tempfile::tempdir().unwrap();
|
let directory = tempfile::tempdir().unwrap();
|
||||||
let pid_path = directory.path().join("telemt.pid");
|
let pid_path = directory.path().join("telemt.pid");
|
||||||
let lock_path = sibling_lock_path(&pid_path);
|
let lock_path = sibling_lock_path(&pid_path);
|
||||||
let mut pid_file = PidFile::new(&pid_path);
|
let mut pid_file = PidFile::new(&pid_path, false);
|
||||||
|
|
||||||
pid_file.acquire().unwrap();
|
pid_file.acquire().unwrap();
|
||||||
assert!(
|
assert!(
|
||||||
@@ -271,7 +377,10 @@ fn pid_file_release_keeps_lock_inode() {
|
|||||||
.into_iter()
|
.into_iter()
|
||||||
.all(|file| file.is_some())
|
.all(|file| file.is_some())
|
||||||
);
|
);
|
||||||
assert_eq!(read_pid_file(&pid_path).unwrap(), std::process::id() as i32);
|
assert_eq!(
|
||||||
|
read_pid_file(&pid_path, false).unwrap(),
|
||||||
|
std::process::id() as i32
|
||||||
|
);
|
||||||
let lock_inode = fs::metadata(&lock_path).unwrap().ino();
|
let lock_inode = fs::metadata(&lock_path).unwrap().ino();
|
||||||
pid_file.release().unwrap();
|
pid_file.release().unwrap();
|
||||||
|
|
||||||
|
|||||||
+7
-11
@@ -77,6 +77,8 @@ pub struct LoggingOptions {
|
|||||||
pub destination: LogDestination,
|
pub destination: LogDestination,
|
||||||
/// Disable ANSI colors.
|
/// Disable ANSI colors.
|
||||||
pub disable_colors: bool,
|
pub disable_colors: bool,
|
||||||
|
/// Require trusted, symlink-free log parents on Unix. Disabled by default for compatibility.
|
||||||
|
pub strict_runtime_paths: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Guard that must be held to keep file logging active.
|
/// Guard that must be held to keep file logging active.
|
||||||
@@ -125,7 +127,6 @@ pub fn init_logging(
|
|||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
LogDestination::Syslog => {
|
LogDestination::Syslog => {
|
||||||
// Use a custom fmt layer that writes to syslog
|
|
||||||
let fmt_layer = fmt::Layer::default()
|
let fmt_layer = fmt::Layer::default()
|
||||||
.with_ansi(false)
|
.with_ansi(false)
|
||||||
.with_target(false)
|
.with_target(false)
|
||||||
@@ -143,7 +144,8 @@ pub fn init_logging(
|
|||||||
|
|
||||||
LogDestination::File { options } => {
|
LogDestination::File { options } => {
|
||||||
let file_appender =
|
let file_appender =
|
||||||
file::BoundedFileAppender::new(options.clone()).expect("Failed to open log file");
|
file::BoundedFileAppender::new(options.clone(), opts.strict_runtime_paths)
|
||||||
|
.expect("Failed to open log file");
|
||||||
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
|
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
|
||||||
|
|
||||||
let fmt_layer = fmt::Layer::default()
|
let fmt_layer = fmt::Layer::default()
|
||||||
@@ -175,14 +177,10 @@ struct SyslogWriter {
|
|||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
impl SyslogMakeWriter {
|
impl SyslogMakeWriter {
|
||||||
fn new() -> Self {
|
fn new() -> Self {
|
||||||
// Open syslog connection on first use
|
|
||||||
static INIT: std::sync::Once = std::sync::Once::new();
|
static INIT: std::sync::Once = std::sync::Once::new();
|
||||||
INIT.call_once(|| {
|
INIT.call_once(|| unsafe {
|
||||||
unsafe {
|
let ident = b"telemt\0".as_ptr() as *const libc::c_char;
|
||||||
// Open syslog with ident "telemt", LOG_PID, LOG_DAEMON facility
|
libc::openlog(ident, libc::LOG_PID | libc::LOG_NDELAY, libc::LOG_DAEMON);
|
||||||
let ident = b"telemt\0".as_ptr() as *const libc::c_char;
|
|
||||||
libc::openlog(ident, libc::LOG_PID | libc::LOG_NDELAY, libc::LOG_DAEMON);
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
Self
|
Self
|
||||||
}
|
}
|
||||||
@@ -202,7 +200,6 @@ fn syslog_priority_for_level(level: &tracing::Level) -> libc::c_int {
|
|||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
impl std::io::Write for SyslogWriter {
|
impl std::io::Write for SyslogWriter {
|
||||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||||
// Convert to C string, stripping newlines
|
|
||||||
let msg = String::from_utf8_lossy(buf);
|
let msg = String::from_utf8_lossy(buf);
|
||||||
let msg = msg.trim_end();
|
let msg = msg.trim_end();
|
||||||
|
|
||||||
@@ -210,7 +207,6 @@ impl std::io::Write for SyslogWriter {
|
|||||||
return Ok(buf.len());
|
return Ok(buf.len());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write to syslog
|
|
||||||
let c_msg = std::ffi::CString::new(msg.as_bytes())
|
let c_msg = std::ffi::CString::new(msg.as_bytes())
|
||||||
.unwrap_or_else(|_| std::ffi::CString::new("(invalid utf8)").unwrap());
|
.unwrap_or_else(|_| std::ffi::CString::new("(invalid utf8)").unwrap());
|
||||||
|
|
||||||
|
|||||||
+11
-3
@@ -44,13 +44,15 @@ pub(crate) struct BoundedFileAppender {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl BoundedFileAppender {
|
impl BoundedFileAppender {
|
||||||
pub(crate) fn new(options: FileLogOptions) -> io::Result<Self> {
|
/// Opens the appender using the process-level Unix parent-path policy.
|
||||||
Self::with_now(options, Box::new(Utc::now))
|
pub(crate) fn new(options: FileLogOptions, strict_runtime_paths: bool) -> io::Result<Self> {
|
||||||
|
Self::with_now(options, Box::new(Utc::now), strict_runtime_paths)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn with_now(
|
fn with_now(
|
||||||
options: FileLogOptions,
|
options: FileLogOptions,
|
||||||
now: Box<dyn Fn() -> DateTime<Utc> + Send + Sync>,
|
now: Box<dyn Fn() -> DateTime<Utc> + Send + Sync>,
|
||||||
|
strict_runtime_paths: bool,
|
||||||
) -> io::Result<Self> {
|
) -> io::Result<Self> {
|
||||||
let path = Path::new(&options.path);
|
let path = Path::new(&options.path);
|
||||||
let dir = path
|
let dir = path
|
||||||
@@ -67,7 +69,13 @@ impl BoundedFileAppender {
|
|||||||
let start = now();
|
let start = now();
|
||||||
let current_path = active_path_for(&dir, &base_name, options.rotation, &start);
|
let current_path = active_path_for(&dir, &base_name, options.rotation, &start);
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
let dir_fd = crate::util::secure_fs::open_trusted_dir_nofollow_or_create(&dir, 0o750)?;
|
let dir_fd = if strict_runtime_paths {
|
||||||
|
crate::util::secure_fs::open_trusted_dir_nofollow_or_create(&dir, 0o750)?
|
||||||
|
} else {
|
||||||
|
crate::util::secure_fs::open_compatible_dir(&dir, Some(0o750))?
|
||||||
|
};
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
let _ = strict_runtime_paths;
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
let (file, current_size) = open_append_file(&dir_fd, ¤t_path)?;
|
let (file, current_size) = open_append_file(&dir_fd, ¤t_path)?;
|
||||||
#[cfg(not(unix))]
|
#[cfg(not(unix))]
|
||||||
|
|||||||
+119
-6
@@ -34,6 +34,115 @@ fn matching_logs(dir: &Path) -> Vec<PathBuf> {
|
|||||||
files
|
files
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn compatibility_appender_accepts_writable_log_directories() {
|
||||||
|
use std::os::unix::fs::PermissionsExt;
|
||||||
|
|
||||||
|
for mode in [0o770, 0o777, 0o1777] {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
fs::set_permissions(dir.path(), fs::Permissions::from_mode(mode)).unwrap();
|
||||||
|
let path = dir.path().join("telemt.log");
|
||||||
|
assert!(
|
||||||
|
BoundedFileAppender::with_now(options(path.clone()), Box::new(fixed_now), true)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(!path.exists());
|
||||||
|
let mut appender =
|
||||||
|
BoundedFileAppender::with_now(options(path.clone()), Box::new(fixed_now), false)
|
||||||
|
.unwrap();
|
||||||
|
appender.write_all(b"compatibility\n").unwrap();
|
||||||
|
appender.flush().unwrap();
|
||||||
|
|
||||||
|
assert_eq!(fs::read(&path).unwrap(), b"compatibility\n");
|
||||||
|
assert_eq!(
|
||||||
|
fs::metadata(&path).unwrap().permissions().mode() & 0o777,
|
||||||
|
0o640
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn compatibility_appender_follows_symlinked_parents() {
|
||||||
|
use std::os::unix::fs::symlink;
|
||||||
|
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let real = dir.path().join("real");
|
||||||
|
let linked = dir.path().join("linked");
|
||||||
|
fs::create_dir(&real).unwrap();
|
||||||
|
symlink(&real, &linked).unwrap();
|
||||||
|
let path = linked.join("nested/telemt.log");
|
||||||
|
assert!(
|
||||||
|
BoundedFileAppender::with_now(options(path.clone()), Box::new(fixed_now), true).is_err()
|
||||||
|
);
|
||||||
|
assert!(!real.join("nested").exists());
|
||||||
|
let mut appender =
|
||||||
|
BoundedFileAppender::with_now(options(path), Box::new(fixed_now), false).unwrap();
|
||||||
|
appender.write_all(b"compatibility\n").unwrap();
|
||||||
|
appender.flush().unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(real.join("nested/telemt.log")).unwrap(),
|
||||||
|
b"compatibility\n"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn appender_rejects_final_symlinks_and_hard_links_in_both_modes() {
|
||||||
|
use std::os::unix::fs::symlink;
|
||||||
|
|
||||||
|
for strict_runtime_paths in [false, true] {
|
||||||
|
for hard_link in [false, true] {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let target = dir.path().join("sentinel");
|
||||||
|
let path = dir.path().join("telemt.log");
|
||||||
|
fs::write(&target, b"preserve\n").unwrap();
|
||||||
|
if hard_link {
|
||||||
|
fs::hard_link(&target, &path).unwrap();
|
||||||
|
} else {
|
||||||
|
symlink(&target, &path).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
BoundedFileAppender::with_now(
|
||||||
|
options(path),
|
||||||
|
Box::new(fixed_now),
|
||||||
|
strict_runtime_paths
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(fs::read(&target).unwrap(), b"preserve\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn time_rotation_and_retention_work_through_compatible_parent_alias() {
|
||||||
|
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||||
|
|
||||||
|
let root = tempdir().unwrap();
|
||||||
|
let real = root.path().join("logs");
|
||||||
|
let linked = root.path().join("linked");
|
||||||
|
fs::create_dir(&real).unwrap();
|
||||||
|
fs::set_permissions(&real, fs::Permissions::from_mode(0o777)).unwrap();
|
||||||
|
symlink(&real, &linked).unwrap();
|
||||||
|
let mut options = options(linked.join("telemt.log"));
|
||||||
|
options.rotation = LogRotation::Daily;
|
||||||
|
options.max_files = 1;
|
||||||
|
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now), false).unwrap();
|
||||||
|
appender.write_all(b"first\n").unwrap();
|
||||||
|
appender.now = Box::new(|| fixed_now() + ChronoDuration::days(1));
|
||||||
|
appender.write_all(b"second\n").unwrap();
|
||||||
|
appender.flush().unwrap();
|
||||||
|
|
||||||
|
let remaining = matching_logs(&real);
|
||||||
|
assert_eq!(remaining.len(), 1);
|
||||||
|
assert_eq!(fs::read(&remaining[0]).unwrap(), b"second\n");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn size_rotation_keeps_latest_write_in_active_file() {
|
fn size_rotation_keeps_latest_write_in_active_file() {
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
@@ -41,7 +150,7 @@ fn size_rotation_keeps_latest_write_in_active_file() {
|
|||||||
let mut options = options(path.clone());
|
let mut options = options(path.clone());
|
||||||
options.max_size_bytes = 6;
|
options.max_size_bytes = 6;
|
||||||
|
|
||||||
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now)).unwrap();
|
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now), false).unwrap();
|
||||||
appender.write_all(b"abc\n").unwrap();
|
appender.write_all(b"abc\n").unwrap();
|
||||||
appender.write_all(b"def\n").unwrap();
|
appender.write_all(b"def\n").unwrap();
|
||||||
appender.flush().unwrap();
|
appender.flush().unwrap();
|
||||||
@@ -58,7 +167,7 @@ fn max_files_retention_removes_oldest_archives() {
|
|||||||
options.max_size_bytes = 4;
|
options.max_size_bytes = 4;
|
||||||
options.max_files = 2;
|
options.max_files = 2;
|
||||||
|
|
||||||
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now)).unwrap();
|
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now), false).unwrap();
|
||||||
for line in [b"aa\n", b"bb\n", b"cc\n", b"dd\n"] {
|
for line in [b"aa\n", b"bb\n", b"cc\n", b"dd\n"] {
|
||||||
appender.write_all(line).unwrap();
|
appender.write_all(line).unwrap();
|
||||||
}
|
}
|
||||||
@@ -94,7 +203,7 @@ fn max_age_retention_removes_old_archives() {
|
|||||||
|
|
||||||
let mut options = options(path);
|
let mut options = options(path);
|
||||||
options.max_age_secs = 1;
|
options.max_age_secs = 1;
|
||||||
let _appender = BoundedFileAppender::with_now(options, Box::new(fixed_now)).unwrap();
|
let _appender = BoundedFileAppender::with_now(options, Box::new(fixed_now), false).unwrap();
|
||||||
|
|
||||||
assert!(!old_archive.exists());
|
assert!(!old_archive.exists());
|
||||||
}
|
}
|
||||||
@@ -112,7 +221,7 @@ fn rotation_stays_bound_to_opened_directory_after_path_replacement() {
|
|||||||
fs::create_dir(&redirect).unwrap();
|
fs::create_dir(&redirect).unwrap();
|
||||||
let mut options = options(original.join("telemt.log"));
|
let mut options = options(original.join("telemt.log"));
|
||||||
options.max_size_bytes = 4;
|
options.max_size_bytes = 4;
|
||||||
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now)).unwrap();
|
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now), false).unwrap();
|
||||||
appender.write_all(b"aa\n").unwrap();
|
appender.write_all(b"aa\n").unwrap();
|
||||||
fs::rename(&original, &moved).unwrap();
|
fs::rename(&original, &moved).unwrap();
|
||||||
symlink(&redirect, &original).unwrap();
|
symlink(&redirect, &original).unwrap();
|
||||||
@@ -137,7 +246,11 @@ fn appender_rejects_group_writable_log_directory() {
|
|||||||
fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o770)).unwrap();
|
fs::set_permissions(dir.path(), fs::Permissions::from_mode(0o770)).unwrap();
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
BoundedFileAppender::with_now(options(dir.path().join("telemt.log")), Box::new(fixed_now),)
|
BoundedFileAppender::with_now(
|
||||||
.is_err()
|
options(dir.path().join("telemt.log")),
|
||||||
|
Box::new(fixed_now),
|
||||||
|
true
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ use super::*;
|
|||||||
fn test_parse_log_cli_options_default() {
|
fn test_parse_log_cli_options_default() {
|
||||||
let args: Vec<String> = vec![];
|
let args: Vec<String> = vec![];
|
||||||
let options = parse_log_cli_options(&args).unwrap();
|
let options = parse_log_cli_options(&args).unwrap();
|
||||||
|
assert!(!LoggingOptions::default().strict_runtime_paths);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
resolve_log_destination(&LoggingConfig::default(), &options).unwrap(),
|
resolve_log_destination(&LoggingConfig::default(), &options).unwrap(),
|
||||||
LogDestination::Stderr
|
LogDestination::Stderr
|
||||||
|
|||||||
@@ -13,6 +13,11 @@ use super::generation::RuntimeTaskScope;
|
|||||||
const STARTUP_FALLBACK_AFTER: Duration = Duration::from_secs(80);
|
const STARTUP_FALLBACK_AFTER: Duration = Duration::from_secs(80);
|
||||||
const RUNTIME_FALLBACK_AFTER: Duration = Duration::from_secs(6);
|
const RUNTIME_FALLBACK_AFTER: Duration = Duration::from_secs(6);
|
||||||
|
|
||||||
|
// Admission regressions cover notification loss without network startup.
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests;
|
||||||
|
|
||||||
|
/// Keeps generation admission and routing synchronized with periodic ME readiness.
|
||||||
pub(crate) async fn configure_admission_gate(
|
pub(crate) async fn configure_admission_gate(
|
||||||
config: &Arc<ProxyConfig>,
|
config: &Arc<ProxyConfig>,
|
||||||
me_pool: Option<Arc<MePool>>,
|
me_pool: Option<Arc<MePool>>,
|
||||||
@@ -76,11 +81,18 @@ pub(crate) async fn configure_admission_gate(
|
|||||||
} else {
|
} else {
|
||||||
Some(Instant::now())
|
Some(Instant::now())
|
||||||
};
|
};
|
||||||
|
let mut config_watch_open = true;
|
||||||
|
let mut me_ready_watch_open = true;
|
||||||
loop {
|
loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
changed = config_rx_gate.changed() => {
|
changed = config_rx_gate.changed(), if config_watch_open => {
|
||||||
if changed.is_err() {
|
if changed.is_err() {
|
||||||
break;
|
config_watch_open = false;
|
||||||
|
warn!(
|
||||||
|
watch_channel = "config",
|
||||||
|
"Admission config watch closed; continuing readiness polling with last configuration"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
let cfg = config_rx_gate.borrow_and_update().clone();
|
let cfg = config_rx_gate.borrow_and_update().clone();
|
||||||
admission_poll_ms = cfg.general.me_admission_poll_ms.max(1);
|
admission_poll_ms = cfg.general.me_admission_poll_ms.max(1);
|
||||||
@@ -88,9 +100,13 @@ pub(crate) async fn configure_admission_gate(
|
|||||||
fast_fallback_enabled = cfg.general.me2dc_fallback && cfg.general.me2dc_fast;
|
fast_fallback_enabled = cfg.general.me2dc_fallback && cfg.general.me2dc_fast;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
changed = me_ready_rx_gate.changed() => {
|
changed = me_ready_rx_gate.changed(), if me_ready_watch_open => {
|
||||||
if changed.is_err() {
|
if changed.is_err() {
|
||||||
break;
|
me_ready_watch_open = false;
|
||||||
|
warn!(
|
||||||
|
watch_channel = "me_ready",
|
||||||
|
"Admission ME readiness watch closed; continuing periodic polling"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ = tokio::time::sleep(Duration::from_millis(admission_poll_ms)) => {}
|
_ = tokio::time::sleep(Duration::from_millis(admission_poll_ms)) => {}
|
||||||
|
|||||||
@@ -0,0 +1,210 @@
|
|||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
|
use tracing::{Event, Subscriber};
|
||||||
|
use tracing_subscriber::Layer;
|
||||||
|
use tracing_subscriber::layer::{Context, SubscriberExt};
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::transport::middle_proxy::admission_test_support::IdlePoolFixture;
|
||||||
|
|
||||||
|
const POLL_MS: u64 = 20;
|
||||||
|
|
||||||
|
struct ClosureWarnings(Arc<AtomicUsize>);
|
||||||
|
|
||||||
|
impl<S: Subscriber> Layer<S> for ClosureWarnings {
|
||||||
|
fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) {
|
||||||
|
struct ChannelField(bool);
|
||||||
|
impl tracing::field::Visit for ChannelField {
|
||||||
|
fn record_debug(
|
||||||
|
&mut self,
|
||||||
|
field: &tracing::field::Field,
|
||||||
|
_value: &dyn std::fmt::Debug,
|
||||||
|
) {
|
||||||
|
self.0 |= field.name() == "watch_channel";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if event.metadata().level() == &tracing::Level::WARN {
|
||||||
|
let mut visitor = ChannelField(false);
|
||||||
|
event.record(&mut visitor);
|
||||||
|
if visitor.0 {
|
||||||
|
self.0.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct GateFixture {
|
||||||
|
pool: IdlePoolFixture,
|
||||||
|
route: Arc<RouteRuntimeController>,
|
||||||
|
admission: watch::Receiver<bool>,
|
||||||
|
config_tx: Option<watch::Sender<Arc<ProxyConfig>>>,
|
||||||
|
ready_tx: Option<watch::Sender<u64>>,
|
||||||
|
config: Arc<ProxyConfig>,
|
||||||
|
scope: RuntimeTaskScope,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl GateFixture {
|
||||||
|
async fn new() -> Self {
|
||||||
|
let pool = IdlePoolFixture::new(false).await;
|
||||||
|
let mut cfg = ProxyConfig::default();
|
||||||
|
cfg.general.use_middle_proxy = true;
|
||||||
|
cfg.general.me2dc_fallback = true;
|
||||||
|
cfg.general.me2dc_fast = true;
|
||||||
|
cfg.general.me_admission_poll_ms = POLL_MS;
|
||||||
|
let config = Arc::new(cfg);
|
||||||
|
let (config_tx, config_rx) = watch::channel(config.clone());
|
||||||
|
let (ready_tx, ready_rx) = watch::channel(0);
|
||||||
|
let (admission_tx, mut admission) = watch::channel(false);
|
||||||
|
let route = Arc::new(RouteRuntimeController::new(RelayRouteMode::Direct));
|
||||||
|
let scope = RuntimeTaskScope::new();
|
||||||
|
configure_admission_gate(
|
||||||
|
&config,
|
||||||
|
Some(pool.pool()),
|
||||||
|
Arc::new(RwLock::new(Some(pool.pool()))),
|
||||||
|
route.clone(),
|
||||||
|
&admission_tx,
|
||||||
|
config_rx,
|
||||||
|
ready_rx,
|
||||||
|
scope.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
drop(admission_tx);
|
||||||
|
admission.borrow_and_update();
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
Self {
|
||||||
|
pool,
|
||||||
|
route,
|
||||||
|
admission,
|
||||||
|
config_tx: Some(config_tx),
|
||||||
|
ready_tx: Some(ready_tx),
|
||||||
|
config,
|
||||||
|
scope,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn poll(&self) {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
tokio::time::advance(Duration::from_millis(POLL_MS)).await;
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn expect_route(&self, mode: RelayRouteMode) {
|
||||||
|
let mut rx = self.route.subscribe();
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), async {
|
||||||
|
loop {
|
||||||
|
if rx.borrow_and_update().mode == mode {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
rx.changed().await.unwrap();
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("the generation must keep sampling readiness without client traffic");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn recovery_after_watch_closure(close_config: bool, close_ready: bool) {
|
||||||
|
let warnings = Arc::new(AtomicUsize::new(0));
|
||||||
|
let subscriber = tracing_subscriber::registry().with(ClosureWarnings(warnings.clone()));
|
||||||
|
let _default = tracing::subscriber::set_default(subscriber);
|
||||||
|
let mut fixture = GateFixture::new().await;
|
||||||
|
assert!(*fixture.admission.borrow());
|
||||||
|
if close_config {
|
||||||
|
fixture.config_tx.take();
|
||||||
|
}
|
||||||
|
if close_ready {
|
||||||
|
fixture.ready_tx.take();
|
||||||
|
}
|
||||||
|
for _ in 0..4 {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
let expected_warnings = usize::from(close_config) + usize::from(close_ready);
|
||||||
|
fixture.pool.set_ready(true);
|
||||||
|
for _ in 0..4 {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
assert_eq!(fixture.route.snapshot().mode, RelayRouteMode::Direct);
|
||||||
|
fixture.poll().await;
|
||||||
|
fixture.expect_route(RelayRouteMode::Middle).await;
|
||||||
|
fixture.pool.set_ready(false);
|
||||||
|
fixture.poll().await;
|
||||||
|
fixture.expect_route(RelayRouteMode::Direct).await;
|
||||||
|
fixture.pool.set_ready(true);
|
||||||
|
fixture.poll().await;
|
||||||
|
fixture.expect_route(RelayRouteMode::Middle).await;
|
||||||
|
for _ in 0..5 {
|
||||||
|
fixture.poll().await;
|
||||||
|
}
|
||||||
|
assert_eq!(warnings.load(Ordering::Relaxed), expected_warnings);
|
||||||
|
fixture.scope.stop().await;
|
||||||
|
assert!(fixture.admission.has_changed().is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn idle_gate_recovers_after_readiness_watch_closes() {
|
||||||
|
recovery_after_watch_closure(false, true).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn idle_gate_recovers_after_config_watch_closes() {
|
||||||
|
recovery_after_watch_closure(true, false).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn idle_gate_recovers_after_both_watches_close_without_spinning() {
|
||||||
|
recovery_after_watch_closure(true, true).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn closed_config_watch_preserves_last_effective_fallback_policy() {
|
||||||
|
let mut fixture = GateFixture::new().await;
|
||||||
|
let mut cfg = fixture.config.as_ref().clone();
|
||||||
|
cfg.general.me2dc_fallback = false;
|
||||||
|
fixture
|
||||||
|
.config_tx
|
||||||
|
.as_ref()
|
||||||
|
.unwrap()
|
||||||
|
.send_replace(Arc::new(cfg));
|
||||||
|
fixture.poll().await;
|
||||||
|
fixture.expect_route(RelayRouteMode::Middle).await;
|
||||||
|
assert!(!*fixture.admission.borrow_and_update());
|
||||||
|
fixture.config_tx.take();
|
||||||
|
fixture.ready_tx.take();
|
||||||
|
fixture.poll().await;
|
||||||
|
fixture.pool.set_ready(true);
|
||||||
|
fixture.poll().await;
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), fixture.admission.changed())
|
||||||
|
.await
|
||||||
|
.expect("restored readiness must open admission after notification loss")
|
||||||
|
.unwrap();
|
||||||
|
assert!(*fixture.admission.borrow_and_update());
|
||||||
|
fixture.pool.set_ready(false);
|
||||||
|
fixture.poll().await;
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), fixture.admission.changed())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(!*fixture.admission.borrow_and_update());
|
||||||
|
assert_eq!(fixture.route.snapshot().mode, RelayRouteMode::Middle);
|
||||||
|
fixture.scope.stop().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn generation_cancellation_stops_gate_after_notification_loss() {
|
||||||
|
let mut fixture = GateFixture::new().await;
|
||||||
|
fixture.config_tx.take();
|
||||||
|
fixture.ready_tx.take();
|
||||||
|
fixture.poll().await;
|
||||||
|
fixture.pool.set_ready(true);
|
||||||
|
fixture.poll().await;
|
||||||
|
fixture.expect_route(RelayRouteMode::Middle).await;
|
||||||
|
fixture.scope.stop().await;
|
||||||
|
assert!(fixture.admission.has_changed().is_err());
|
||||||
|
let state = fixture.route.snapshot();
|
||||||
|
fixture.pool.set_ready(false);
|
||||||
|
for _ in 0..3 {
|
||||||
|
fixture.poll().await;
|
||||||
|
}
|
||||||
|
assert_eq!(fixture.route.snapshot(), state);
|
||||||
|
assert!(*fixture.admission.borrow());
|
||||||
|
}
|
||||||
@@ -15,20 +15,32 @@ use super::helpers::{
|
|||||||
use super::runtime_tasks;
|
use super::runtime_tasks;
|
||||||
use super::validate_synlimit_privilege_drop;
|
use super::validate_synlimit_privilege_drop;
|
||||||
|
|
||||||
|
/// Process-level configuration and logging resources retained across runtime generations.
|
||||||
pub(super) struct BootstrapState {
|
pub(super) struct BootstrapState {
|
||||||
|
/// Monotonic process startup time.
|
||||||
pub(super) process_started_at: Instant,
|
pub(super) process_started_at: Instant,
|
||||||
|
/// Process startup time as Unix epoch seconds.
|
||||||
pub(super) process_started_at_epoch_secs: u64,
|
pub(super) process_started_at_epoch_secs: u64,
|
||||||
|
/// Startup component progress shared with the control plane.
|
||||||
pub(super) startup_tracker: Arc<StartupTracker>,
|
pub(super) startup_tracker: Arc<StartupTracker>,
|
||||||
|
/// Validated initial runtime configuration.
|
||||||
pub(super) config: ProxyConfig,
|
pub(super) config: ProxyConfig,
|
||||||
|
/// Resolved source path used by reload operations.
|
||||||
pub(super) config_path: PathBuf,
|
pub(super) config_path: PathBuf,
|
||||||
|
/// Whether the environment owns the log filter policy.
|
||||||
pub(super) has_rust_log: bool,
|
pub(super) has_rust_log: bool,
|
||||||
|
/// Initial verbosity after CLI overrides.
|
||||||
pub(super) effective_log_level: LogLevel,
|
pub(super) effective_log_level: LogLevel,
|
||||||
|
/// Process-owned dynamic tracing filter.
|
||||||
pub(super) runtime_log_filter: runtime_tasks::RuntimeLogFilter,
|
pub(super) runtime_log_filter: runtime_tasks::RuntimeLogFilter,
|
||||||
|
/// Keeps the file logging worker alive until process shutdown.
|
||||||
pub(super) logging_guard: Option<crate::logging::LoggingGuard>,
|
pub(super) logging_guard: Option<crate::logging::LoggingGuard>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Loads configuration and initializes process logging with the startup parent-path policy.
|
||||||
pub(super) async fn bootstrap(
|
pub(super) async fn bootstrap(
|
||||||
privilege_drop_requested: bool,
|
privilege_drop_requested: bool,
|
||||||
|
strict_runtime_paths: bool,
|
||||||
) -> std::result::Result<BootstrapState, Box<dyn std::error::Error>> {
|
) -> std::result::Result<BootstrapState, Box<dyn std::error::Error>> {
|
||||||
let process_started_at = Instant::now();
|
let process_started_at = Instant::now();
|
||||||
let process_started_at_epoch_secs = SystemTime::now()
|
let process_started_at_epoch_secs = SystemTime::now()
|
||||||
@@ -268,6 +280,7 @@ pub(super) async fn bootstrap(
|
|||||||
let logging_opts = crate::logging::LoggingOptions {
|
let logging_opts = crate::logging::LoggingOptions {
|
||||||
destination: log_destination,
|
destination: log_destination,
|
||||||
disable_colors: true,
|
disable_colors: true,
|
||||||
|
strict_runtime_paths,
|
||||||
};
|
};
|
||||||
let (_, guard) = crate::logging::init_logging(&logging_opts, &initial_filter_spec);
|
let (_, guard) = crate::logging::init_logging(&logging_opts, &initial_filter_spec);
|
||||||
logging_guard = Some(guard);
|
logging_guard = Some(guard);
|
||||||
@@ -276,6 +289,7 @@ pub(super) async fn bootstrap(
|
|||||||
let logging_opts = crate::logging::LoggingOptions {
|
let logging_opts = crate::logging::LoggingOptions {
|
||||||
destination: log_destination,
|
destination: log_destination,
|
||||||
disable_colors: true,
|
disable_colors: true,
|
||||||
|
strict_runtime_paths,
|
||||||
};
|
};
|
||||||
let (_, guard) = crate::logging::init_logging(&logging_opts, &initial_filter_spec);
|
let (_, guard) = crate::logging::init_logging(&logging_opts, &initial_filter_spec);
|
||||||
logging_guard = Some(guard);
|
logging_guard = Some(guard);
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ pub(crate) fn print_maestro_line(message: impl AsRef<str>) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Resolves the config source against startup cwd while retaining symlink components.
|
||||||
pub(crate) fn resolve_runtime_config_path(
|
pub(crate) fn resolve_runtime_config_path(
|
||||||
config_path_cli: &str,
|
config_path_cli: &str,
|
||||||
startup_cwd: &Path,
|
startup_cwd: &Path,
|
||||||
@@ -75,6 +76,7 @@ pub(crate) fn resolve_runtime_config_path(
|
|||||||
startup_cwd.join("config.toml")
|
startup_cwd.join("config.toml")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Selects the runtime directory from CLI, startup cwd, and explicit config location.
|
||||||
pub(crate) fn resolve_runtime_base_dir(
|
pub(crate) fn resolve_runtime_base_dir(
|
||||||
config_path: &Path,
|
config_path: &Path,
|
||||||
startup_cwd: &Path,
|
startup_cwd: &Path,
|
||||||
@@ -120,14 +122,21 @@ fn normalize_runtime_dir(path: &Path, startup_cwd: &Path) -> PathBuf {
|
|||||||
|
|
||||||
/// Parsed CLI arguments.
|
/// Parsed CLI arguments.
|
||||||
pub(crate) struct CliArgs {
|
pub(crate) struct CliArgs {
|
||||||
|
/// Config source selected by positional arguments.
|
||||||
pub config_path: String,
|
pub config_path: String,
|
||||||
|
/// Whether the config source was explicitly provided.
|
||||||
pub config_path_explicit: bool,
|
pub config_path_explicit: bool,
|
||||||
|
/// Runtime directory override from CLI.
|
||||||
pub data_path: Option<PathBuf>,
|
pub data_path: Option<PathBuf>,
|
||||||
|
/// Whether CLI requests minimal logging output.
|
||||||
pub silent: bool,
|
pub silent: bool,
|
||||||
|
/// Verbosity override from CLI.
|
||||||
pub log_level: Option<String>,
|
pub log_level: Option<String>,
|
||||||
|
/// Logging destination, rotation, and retention overrides.
|
||||||
pub log_cli_options: LogCliOptions,
|
pub log_cli_options: LogCliOptions,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Parses runtime arguments after early daemon and control-command handling.
|
||||||
pub(crate) fn parse_cli() -> CliArgs {
|
pub(crate) fn parse_cli() -> CliArgs {
|
||||||
let mut config_path = "config.toml".to_string();
|
let mut config_path = "config.toml".to_string();
|
||||||
let mut config_path_explicit = false;
|
let mut config_path_explicit = false;
|
||||||
@@ -222,6 +231,8 @@ pub(crate) fn parse_cli() -> CliArgs {
|
|||||||
}
|
}
|
||||||
// Skip daemon-related flags (already parsed)
|
// Skip daemon-related flags (already parsed)
|
||||||
"--daemon" | "-d" | "--foreground" | "-f" => {}
|
"--daemon" | "-d" | "--foreground" | "-f" => {}
|
||||||
|
#[cfg(unix)]
|
||||||
|
"--strict-runtime-paths" => {}
|
||||||
s if s.starts_with("--pid-file") => {
|
s if s.starts_with("--pid-file") => {
|
||||||
if !s.contains('=') {
|
if !s.contains('=') {
|
||||||
// Skip the pid-file value consumed by daemon argument parsing.
|
// Skip the pid-file value consumed by daemon argument parsing.
|
||||||
@@ -300,6 +311,15 @@ fn print_help() {
|
|||||||
eprintln!(" --daemon, -d Fork to background (daemonize)");
|
eprintln!(" --daemon, -d Fork to background (daemonize)");
|
||||||
eprintln!(" --foreground, -f Explicit foreground mode (for systemd)");
|
eprintln!(" --foreground, -f Explicit foreground mode (for systemd)");
|
||||||
eprintln!(" --pid-file <PATH> PID file path (default: /var/run/telemt.pid)");
|
eprintln!(" --pid-file <PATH> PID file path (default: /var/run/telemt.pid)");
|
||||||
|
eprintln!(
|
||||||
|
" --strict-runtime-paths Require trusted, symlink-free PID/log parents (default: off)"
|
||||||
|
);
|
||||||
|
eprintln!(
|
||||||
|
" Applies to run/start/stop/reload/status; parent symlinks and"
|
||||||
|
);
|
||||||
|
eprintln!(
|
||||||
|
" writable directories are allowed when this flag is absent"
|
||||||
|
);
|
||||||
eprintln!(" --run-as-user <USER> Drop privileges to this user after binding");
|
eprintln!(" --run-as-user <USER> Drop privileges to this user after binding");
|
||||||
eprintln!(" --run-as-group <GROUP> Drop privileges to this group after binding");
|
eprintln!(" --run-as-group <GROUP> Drop privileges to this group after binding");
|
||||||
eprintln!(" --working-dir <DIR> Working directory for daemon mode");
|
eprintln!(" --working-dir <DIR> Working directory for daemon mode");
|
||||||
|
|||||||
+18
-10
@@ -96,7 +96,10 @@ async fn run_inner(
|
|||||||
// Acquire PID file if daemonizing or if explicitly requested.
|
// Acquire PID file if daemonizing or if explicitly requested.
|
||||||
// Keep it alive until shutdown for RAII cleanup.
|
// Keep it alive until shutdown for RAII cleanup.
|
||||||
let _pid_file = if daemon_opts.daemonize || daemon_opts.pid_file.is_some() {
|
let _pid_file = if daemon_opts.daemonize || daemon_opts.pid_file.is_some() {
|
||||||
let mut pf = PidFile::new(daemon_opts.pid_file_path());
|
let mut pf = PidFile::new(
|
||||||
|
daemon_opts.pid_file_path(),
|
||||||
|
daemon_opts.strict_runtime_paths,
|
||||||
|
);
|
||||||
if let Err(e) = pf.acquire() {
|
if let Err(e) = pf.acquire() {
|
||||||
eprintln!("[telemt] {}", e);
|
eprintln!("[telemt] {}", e);
|
||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
@@ -109,20 +112,25 @@ async fn run_inner(
|
|||||||
let user = daemon_opts.user.clone();
|
let user = daemon_opts.user.clone();
|
||||||
let group = daemon_opts.group.clone();
|
let group = daemon_opts.group.clone();
|
||||||
|
|
||||||
orchestrator::run_telemt_core(user.is_some() || group.is_some(), || {
|
orchestrator::run_telemt_core(
|
||||||
if (user.is_some() || group.is_some())
|
user.is_some() || group.is_some(),
|
||||||
&& let Err(e) = drop_privileges(user.as_deref(), group.as_deref(), _pid_file.as_ref())
|
daemon_opts.strict_runtime_paths,
|
||||||
{
|
|| {
|
||||||
error!(error = %e, "Failed to drop privileges");
|
if (user.is_some() || group.is_some())
|
||||||
std::process::exit(1);
|
&& let Err(e) =
|
||||||
}
|
drop_privileges(user.as_deref(), group.as_deref(), _pid_file.as_ref())
|
||||||
})
|
{
|
||||||
|
error!(error = %e, "Failed to drop privileges");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(unix))]
|
#[cfg(not(unix))]
|
||||||
async fn run_inner() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
async fn run_inner() -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||||
orchestrator::run_telemt_core(false, || {}).await
|
orchestrator::run_telemt_core(false, false, || {}).await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -30,10 +30,11 @@ use super::{
|
|||||||
runtime_tasks, shutdown, tls_bootstrap,
|
runtime_tasks, shutdown, tls_bootstrap,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Shared maestro startup and main loop. `drop_after_bind` runs on Unix after listeners are bound
|
/// Runs startup and the main loop with explicit runtime-path policy.
|
||||||
// and privileged firewall setup completes; it is a no-op on other platforms.
|
/// `drop_after_bind` runs after listeners and privileged firewall setup are ready.
|
||||||
pub(super) async fn run_telemt_core(
|
pub(super) async fn run_telemt_core(
|
||||||
privilege_drop_requested: bool,
|
privilege_drop_requested: bool,
|
||||||
|
strict_runtime_paths: bool,
|
||||||
drop_after_bind: impl FnOnce(),
|
drop_after_bind: impl FnOnce(),
|
||||||
) -> std::result::Result<(), Box<dyn std::error::Error>> {
|
) -> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||||
let bootstrap::BootstrapState {
|
let bootstrap::BootstrapState {
|
||||||
@@ -46,7 +47,7 @@ pub(super) async fn run_telemt_core(
|
|||||||
effective_log_level,
|
effective_log_level,
|
||||||
runtime_log_filter,
|
runtime_log_filter,
|
||||||
logging_guard: _logging_guard,
|
logging_guard: _logging_guard,
|
||||||
} = bootstrap::bootstrap(privilege_drop_requested).await?;
|
} = bootstrap::bootstrap(privilege_drop_requested, strict_runtime_paths).await?;
|
||||||
|
|
||||||
if privilege_drop_requested && config.server.conntrack_control.inline_conntrack_control {
|
if privilege_drop_requested && config.server.conntrack_control.inline_conntrack_control {
|
||||||
warn!("Inline conntrack control is disabled when process privileges are dropped");
|
warn!("Inline conntrack control is disabled when process privileges are dropped");
|
||||||
@@ -374,17 +375,21 @@ pub(super) async fn run_telemt_core(
|
|||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
let conntrack_firewall = {
|
let conntrack_firewall = {
|
||||||
let authority = crate::conntrack_control::FirewallAuthority::spawn(&process_control_plane)
|
let authority = crate::conntrack_control::FirewallAuthority::spawn(
|
||||||
.map_err(std::io::Error::other)?;
|
&process_control_plane,
|
||||||
if !authority
|
&runtime.config,
|
||||||
.publish_initial(1, runtime.config.clone(), stats.clone())
|
)
|
||||||
.await
|
.map_err(std::io::Error::other)?;
|
||||||
|
if let Some(authority) = &authority
|
||||||
|
&& !authority
|
||||||
|
.publish_initial(1, runtime.config.clone(), stats.clone())
|
||||||
|
.await
|
||||||
{
|
{
|
||||||
warn!(
|
warn!(
|
||||||
"Initial conntrack firewall reconciliation failed; background retries remain active"
|
"Initial conntrack firewall reconciliation failed; background retries remain active"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
Some(authority)
|
authority
|
||||||
};
|
};
|
||||||
#[cfg(not(target_os = "linux"))]
|
#[cfg(not(target_os = "linux"))]
|
||||||
let conntrack_firewall = None::<crate::conntrack_control::FirewallAuthority>;
|
let conntrack_firewall = None::<crate::conntrack_control::FirewallAuthority>;
|
||||||
|
|||||||
+32
-7
@@ -104,11 +104,12 @@ pub fn generate_service_file(init_system: InitSystem, opts: &ServiceOptions) ->
|
|||||||
InitSystem::Systemd => generate_systemd_unit(opts),
|
InitSystem::Systemd => generate_systemd_unit(opts),
|
||||||
InitSystem::OpenRC => generate_openrc_script(opts),
|
InitSystem::OpenRC => generate_openrc_script(opts),
|
||||||
InitSystem::FreeBSDRc => generate_freebsd_rc_script(opts),
|
InitSystem::FreeBSDRc => generate_freebsd_rc_script(opts),
|
||||||
InitSystem::Unknown => generate_systemd_unit(opts), // Default to systemd format
|
// Preserve systemd generation when init detection is unavailable.
|
||||||
|
InitSystem::Unknown => generate_systemd_unit(opts),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Generates an enhanced systemd unit file.
|
/// Generates a hardened systemd unit with Netlink access for optional firewall helpers.
|
||||||
fn generate_systemd_unit(opts: &ServiceOptions) -> String {
|
fn generate_systemd_unit(opts: &ServiceOptions) -> String {
|
||||||
let user_line = opts.user.map(|u| format!("User={}", u)).unwrap_or_default();
|
let user_line = opts.user.map(|u| format!("User={}", u)).unwrap_or_default();
|
||||||
let group_line = opts
|
let group_line = opts
|
||||||
@@ -151,14 +152,14 @@ PrivateDevices=true
|
|||||||
ProtectKernelTunables=true
|
ProtectKernelTunables=true
|
||||||
ProtectKernelModules=true
|
ProtectKernelModules=true
|
||||||
ProtectControlGroups=true
|
ProtectControlGroups=true
|
||||||
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
|
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
|
||||||
RestrictNamespaces=true
|
RestrictNamespaces=true
|
||||||
RestrictRealtime=true
|
RestrictRealtime=true
|
||||||
RestrictSUIDSGID=true
|
RestrictSUIDSGID=true
|
||||||
MemoryDenyWriteExecute=true
|
MemoryDenyWriteExecute=true
|
||||||
LockPersonality=true
|
LockPersonality=true
|
||||||
|
|
||||||
# Allow binding to privileged ports and writing to specific paths
|
# Allow privileged port binding, optional firewall helpers, and runtime state writes
|
||||||
AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_NET_ADMIN
|
AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_NET_ADMIN
|
||||||
CapabilityBoundingSet=CAP_NET_BIND_SERVICE CAP_NET_ADMIN
|
CapabilityBoundingSet=CAP_NET_BIND_SERVICE CAP_NET_ADMIN
|
||||||
ReadWritePaths=/etc/telemt /var/run /var/lib/telemt
|
ReadWritePaths=/etc/telemt /var/run /var/lib/telemt
|
||||||
@@ -233,9 +234,11 @@ fn generate_freebsd_rc_script(opts: &ServiceOptions) -> String {
|
|||||||
# Add the following lines to /etc/rc.conf to enable telemt:
|
# Add the following lines to /etc/rc.conf to enable telemt:
|
||||||
#
|
#
|
||||||
# telemt_enable="YES"
|
# telemt_enable="YES"
|
||||||
# telemt_config="/etc/telemt/config.toml" # optional
|
# The configuration path can be overridden in rc.conf.
|
||||||
# telemt_user="telemt" # optional
|
# telemt_config="/etc/telemt/config.toml"
|
||||||
# telemt_group="telemt" # optional
|
# The service user and group can be overridden in rc.conf.
|
||||||
|
# telemt_user="telemt"
|
||||||
|
# telemt_group="telemt"
|
||||||
#
|
#
|
||||||
|
|
||||||
. /etc/rc.subr
|
. /etc/rc.subr
|
||||||
@@ -355,6 +358,28 @@ mod tests {
|
|||||||
assert!(unit.contains("PIDFile="));
|
assert!(unit.contains("PIDFile="));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn systemd_unit_allows_netlink_without_removing_hardening() {
|
||||||
|
let unit = generate_systemd_unit(&ServiceOptions::default());
|
||||||
|
let families = unit
|
||||||
|
.lines()
|
||||||
|
.find_map(|line| line.strip_prefix("RestrictAddressFamilies="))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
families.split_whitespace().collect::<Vec<_>>(),
|
||||||
|
["AF_INET", "AF_INET6", "AF_UNIX", "AF_NETLINK"],
|
||||||
|
);
|
||||||
|
for directive in [
|
||||||
|
"NoNewPrivileges=true",
|
||||||
|
"ProtectSystem=strict",
|
||||||
|
"RestrictNamespaces=true",
|
||||||
|
"AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_NET_ADMIN",
|
||||||
|
"CapabilityBoundingSet=CAP_NET_BIND_SERVICE CAP_NET_ADMIN",
|
||||||
|
] {
|
||||||
|
assert!(unit.lines().any(|line| line == directive), "{directive}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_openrc_script_generation() {
|
fn test_openrc_script_generation() {
|
||||||
let opts = ServiceOptions::default();
|
let opts = ServiceOptions::default();
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ use std::net::{IpAddr, SocketAddr};
|
|||||||
use std::os::fd::{AsRawFd, RawFd};
|
use std::os::fd::{AsRawFd, RawFd};
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
use std::os::raw::c_int;
|
use std::os::raw::c_int;
|
||||||
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::Ordering;
|
use std::sync::atomic::Ordering;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
@@ -252,7 +253,7 @@ impl MePool {
|
|||||||
/// Perform full ME RPC handshake on an established TCP stream.
|
/// Perform full ME RPC handshake on an established TCP stream.
|
||||||
/// Returns cipher keys/ivs and split halves; does not register writer.
|
/// Returns cipher keys/ivs and split halves; does not register writer.
|
||||||
pub(crate) async fn handshake_only(
|
pub(crate) async fn handshake_only(
|
||||||
&self,
|
self: &Arc<Self>,
|
||||||
stream: TcpStream,
|
stream: TcpStream,
|
||||||
addr: SocketAddr,
|
addr: SocketAddr,
|
||||||
upstream_egress: Option<UpstreamEgressInfo>,
|
upstream_egress: Option<UpstreamEgressInfo>,
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
// Outage regressions use loopback sockets and explicit family-state snapshots.
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests;
|
||||||
|
|
||||||
|
/// Reports whether replacing an endpoint could disrupt a bound client.
|
||||||
pub(super) fn has_bound_clients_on_endpoint(
|
pub(super) fn has_bound_clients_on_endpoint(
|
||||||
writer_ids: &[u64],
|
writer_ids: &[u64],
|
||||||
bound_clients_by_writer: &HashMap<u64, usize>,
|
bound_clients_by_writer: &HashMap<u64, usize>,
|
||||||
@@ -9,6 +14,7 @@ pub(super) fn has_bound_clients_on_endpoint(
|
|||||||
.any(|writer_id| bound_clients_by_writer.get(writer_id).copied().unwrap_or(0) > 0)
|
.any(|writer_id| bound_clients_by_writer.get(writer_id).copied().unwrap_or(0) > 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Restores missing single-endpoint coverage under the bounded outage retry policy.
|
||||||
pub(super) async fn recover_single_endpoint_outage(
|
pub(super) async fn recover_single_endpoint_outage(
|
||||||
pool: &Arc<MePool>,
|
pool: &Arc<MePool>,
|
||||||
rng: &Arc<SecureRandom>,
|
rng: &Arc<SecureRandom>,
|
||||||
@@ -53,13 +59,21 @@ pub(super) async fn recover_single_endpoint_outage(
|
|||||||
pool.stats
|
pool.stats
|
||||||
.increment_me_single_endpoint_outage_reconnect_attempt_total();
|
.increment_me_single_endpoint_outage_reconnect_attempt_total();
|
||||||
|
|
||||||
|
let generation = pool.current_generation();
|
||||||
let bypass_quarantine = pool.single_endpoint_outage_disable_quarantine();
|
let bypass_quarantine = pool.single_endpoint_outage_disable_quarantine();
|
||||||
let attempt_ok = if bypass_quarantine {
|
let attempt_ok = if bypass_quarantine {
|
||||||
pool.stats
|
pool.stats
|
||||||
.increment_me_single_endpoint_quarantine_bypass_total();
|
.increment_me_single_endpoint_quarantine_bypass_total();
|
||||||
match tokio::time::timeout(
|
match tokio::time::timeout(
|
||||||
pool.reconnect_runtime.me_one_timeout,
|
pool.reconnect_runtime.me_one_timeout,
|
||||||
pool.connect_one_for_dc(endpoint, key.0, rng.as_ref()),
|
pool.connect_one_with_generation_contour_for_dc_with_intent(
|
||||||
|
endpoint,
|
||||||
|
rng.as_ref(),
|
||||||
|
generation,
|
||||||
|
WriterContour::Active,
|
||||||
|
key.0,
|
||||||
|
WriterOpenIntent::Coverage,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -88,7 +102,14 @@ pub(super) async fn recover_single_endpoint_outage(
|
|||||||
let one_endpoint = [endpoint];
|
let one_endpoint = [endpoint];
|
||||||
match tokio::time::timeout(
|
match tokio::time::timeout(
|
||||||
pool.reconnect_runtime.me_one_timeout,
|
pool.reconnect_runtime.me_one_timeout,
|
||||||
pool.connect_endpoints_round_robin(key.0, &one_endpoint, rng.as_ref()),
|
pool.connect_endpoints_round_robin_with_generation_contour(
|
||||||
|
key.0,
|
||||||
|
&one_endpoint,
|
||||||
|
rng.as_ref(),
|
||||||
|
generation,
|
||||||
|
WriterContour::Active,
|
||||||
|
WriterOpenIntent::Coverage,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -142,6 +163,7 @@ pub(super) async fn recover_single_endpoint_outage(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Rotates an idle single-endpoint shadow without removing the existing coverage first.
|
||||||
pub(super) async fn maybe_rotate_single_endpoint_shadow(
|
pub(super) async fn maybe_rotate_single_endpoint_shadow(
|
||||||
pool: &Arc<MePool>,
|
pool: &Arc<MePool>,
|
||||||
rng: &Arc<SecureRandom>,
|
rng: &Arc<SecureRandom>,
|
||||||
|
|||||||
@@ -0,0 +1,148 @@
|
|||||||
|
use std::sync::atomic::Ordering;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::transport::middle_proxy::admission_test_support::unregistered_writer;
|
||||||
|
use crate::transport::middle_proxy::pool_writer_security_tests::make_pool;
|
||||||
|
|
||||||
|
async fn single_endpoint_reaches_tcp_with_a_full_cap(bypass_quarantine: bool) {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let endpoint = listener.local_addr().unwrap();
|
||||||
|
let donor = SocketAddr::new(endpoint.ip(), 1);
|
||||||
|
pool.update_proxy_maps(
|
||||||
|
HashMap::from([
|
||||||
|
(2, vec![(endpoint.ip(), endpoint.port())]),
|
||||||
|
(3, vec![(donor.ip(), donor.port())]),
|
||||||
|
]),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
pool.floor_runtime
|
||||||
|
.me_adaptive_floor_cpu_cores_override
|
||||||
|
.store(1, Ordering::Relaxed);
|
||||||
|
pool.floor_runtime
|
||||||
|
.me_adaptive_floor_max_active_writers_per_core
|
||||||
|
.store(1, Ordering::Relaxed);
|
||||||
|
pool.floor_runtime
|
||||||
|
.me_adaptive_floor_max_active_writers_global
|
||||||
|
.store(1, Ordering::Relaxed);
|
||||||
|
pool.single_endpoint_runtime
|
||||||
|
.me_single_endpoint_outage_disable_quarantine
|
||||||
|
.store(bypass_quarantine, Ordering::Relaxed);
|
||||||
|
let (writer, _receiver) = unregistered_writer(
|
||||||
|
&pool,
|
||||||
|
1,
|
||||||
|
3,
|
||||||
|
donor,
|
||||||
|
pool.current_generation(),
|
||||||
|
WriterContour::Active,
|
||||||
|
);
|
||||||
|
pool.writers.write().await.push(writer);
|
||||||
|
if bypass_quarantine {
|
||||||
|
pool.endpoint_quarantine
|
||||||
|
.lock()
|
||||||
|
.await
|
||||||
|
.insert(endpoint, Instant::now() + Duration::from_secs(60));
|
||||||
|
}
|
||||||
|
let rng = Arc::new(SecureRandom::new());
|
||||||
|
let key = (2, IpFamily::V4);
|
||||||
|
let mut backoff = HashMap::new();
|
||||||
|
let mut next_attempt = HashMap::new();
|
||||||
|
let semaphore = Arc::new(Semaphore::new(1));
|
||||||
|
let required = pool.required_writers_for_dc(1);
|
||||||
|
// TCP acceptance proves capacity admission; closing the peer deliberately rejects the handshake.
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), async {
|
||||||
|
tokio::join!(
|
||||||
|
recover_single_endpoint_outage(
|
||||||
|
&pool,
|
||||||
|
&rng,
|
||||||
|
key,
|
||||||
|
endpoint,
|
||||||
|
required,
|
||||||
|
&mut backoff,
|
||||||
|
&mut next_attempt,
|
||||||
|
&semaphore,
|
||||||
|
),
|
||||||
|
async {
|
||||||
|
let (stream, _) = listener.accept().await.unwrap();
|
||||||
|
drop(stream);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("missing-DC recovery must reach TCP even when another DC fills the cap");
|
||||||
|
assert_eq!(
|
||||||
|
pool.stats
|
||||||
|
.get_me_single_endpoint_outage_reconnect_attempt_total(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
pool.stats
|
||||||
|
.get_me_single_endpoint_outage_reconnect_success_total(),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
pool.stats.get_me_single_endpoint_quarantine_bypass_total(),
|
||||||
|
u64::from(bypass_quarantine)
|
||||||
|
);
|
||||||
|
assert!(next_attempt.contains_key(&key));
|
||||||
|
assert!(backoff.contains_key(&key));
|
||||||
|
assert_eq!(semaphore.available_permits(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn single_endpoint_quarantine_bypass_recovers_coverage_at_cap() {
|
||||||
|
single_endpoint_reaches_tcp_with_a_full_cap(true).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn single_endpoint_round_robin_recovers_coverage_at_cap() {
|
||||||
|
single_endpoint_reaches_tcp_with_a_full_cap(false).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn restored_coverage_preserves_family_suppression_hysteresis() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let now = MePool::now_epoch_secs();
|
||||||
|
let until = now + 3600;
|
||||||
|
pool.set_family_runtime_state(
|
||||||
|
IpFamily::V4,
|
||||||
|
MeFamilyRuntimeState::Suppressed,
|
||||||
|
now,
|
||||||
|
until,
|
||||||
|
5,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
super::super::update_family_runtime_state(&pool, IpFamily::V4, false);
|
||||||
|
assert_eq!(
|
||||||
|
pool.family_runtime_state(IpFamily::V4),
|
||||||
|
MeFamilyRuntimeState::Suppressed
|
||||||
|
);
|
||||||
|
assert_eq!(pool.family_suppressed_until_epoch_secs(IpFamily::V4), until);
|
||||||
|
assert_eq!(pool.family_recover_success_streak(IpFamily::V4), 0);
|
||||||
|
|
||||||
|
pool.set_family_runtime_state(
|
||||||
|
IpFamily::V4,
|
||||||
|
MeFamilyRuntimeState::Suppressed,
|
||||||
|
now,
|
||||||
|
now - 1,
|
||||||
|
5,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
super::super::update_family_runtime_state(&pool, IpFamily::V4, false);
|
||||||
|
assert_eq!(
|
||||||
|
pool.family_runtime_state(IpFamily::V4),
|
||||||
|
MeFamilyRuntimeState::Recovering
|
||||||
|
);
|
||||||
|
assert_eq!(pool.family_recover_success_streak(IpFamily::V4), 1);
|
||||||
|
super::super::update_family_runtime_state(&pool, IpFamily::V4, false);
|
||||||
|
assert_eq!(
|
||||||
|
pool.family_runtime_state(IpFamily::V4),
|
||||||
|
MeFamilyRuntimeState::Healthy
|
||||||
|
);
|
||||||
|
assert_eq!(pool.family_fail_streak(IpFamily::V4), 0);
|
||||||
|
}
|
||||||
@@ -49,6 +49,11 @@ mod send;
|
|||||||
mod send_adversarial_tests;
|
mod send_adversarial_tests;
|
||||||
mod wire;
|
mod wire;
|
||||||
|
|
||||||
|
/// Network-free writer fixtures shared by admission and recovery regressions.
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "tests/admission_test_support.rs"]
|
||||||
|
pub(crate) mod admission_test_support;
|
||||||
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use tokio::sync::OwnedSemaphorePermit;
|
use tokio::sync::OwnedSemaphorePermit;
|
||||||
|
|
||||||
|
|||||||
@@ -407,6 +407,7 @@ pub(super) struct BindingPolicyCore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(super) struct NatRuntimeCore {
|
pub(super) struct NatRuntimeCore {
|
||||||
|
pub(super) discovery: super::pool_nat::NatDiscovery,
|
||||||
pub(super) nat_ip_cfg: Option<IpAddr>,
|
pub(super) nat_ip_cfg: Option<IpAddr>,
|
||||||
pub(super) nat_ip_detected: Arc<RwLock<Option<IpAddr>>>,
|
pub(super) nat_ip_detected: Arc<RwLock<Option<IpAddr>>>,
|
||||||
pub(super) nat_probe: bool,
|
pub(super) nat_probe: bool,
|
||||||
|
|||||||
@@ -279,6 +279,7 @@ impl MePool {
|
|||||||
me_bind_stale_ttl_secs: AtomicU64::new(me_bind_stale_ttl_secs),
|
me_bind_stale_ttl_secs: AtomicU64::new(me_bind_stale_ttl_secs),
|
||||||
}),
|
}),
|
||||||
nat_runtime: Arc::new(NatRuntimeCore {
|
nat_runtime: Arc::new(NatRuntimeCore {
|
||||||
|
discovery: super::super::pool_nat::NatDiscovery::default(),
|
||||||
nat_ip_cfg: nat_ip,
|
nat_ip_cfg: nat_ip,
|
||||||
nat_ip_detected: Arc::new(RwLock::new(None)),
|
nat_ip_detected: Arc::new(RwLock::new(None)),
|
||||||
nat_probe,
|
nat_probe,
|
||||||
|
|||||||
@@ -61,13 +61,13 @@ impl Drop for WriterOpenReservation<'_> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl MePool {
|
impl MePool {
|
||||||
/// Computes the authoritative active-writer floor across enabled families and DCs.
|
/// Computes reservation coverage across configured families, including suppressed ones.
|
||||||
pub(in crate::transport::middle_proxy) async fn active_coverage_required_total(&self) -> usize {
|
pub(in crate::transport::middle_proxy) async fn active_coverage_required_total(&self) -> usize {
|
||||||
let now_epoch_secs = Self::now_epoch_secs();
|
|
||||||
let mut required_total = 0usize;
|
let mut required_total = 0usize;
|
||||||
let endpoint_snapshot = self.endpoint_snapshot.load_full();
|
let endpoint_snapshot = self.endpoint_snapshot.load_full();
|
||||||
|
|
||||||
if self.family_enabled_for_drain_coverage(IpFamily::V4, now_epoch_secs) {
|
// Suppression must not remove the capacity needed to recover active coverage.
|
||||||
|
if self.decision.ipv4_me {
|
||||||
for addrs in endpoint_snapshot.map_v4.values() {
|
for addrs in endpoint_snapshot.map_v4.values() {
|
||||||
let mut endpoints = HashSet::<SocketAddr>::new();
|
let mut endpoints = HashSet::<SocketAddr>::new();
|
||||||
for (ip, port) in addrs.iter().copied() {
|
for (ip, port) in addrs.iter().copied() {
|
||||||
@@ -79,7 +79,7 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if self.family_enabled_for_drain_coverage(IpFamily::V6, now_epoch_secs) {
|
if self.decision.ipv6_me {
|
||||||
for addrs in endpoint_snapshot.map_v6.values() {
|
for addrs in endpoint_snapshot.map_v6.values() {
|
||||||
let mut endpoints = HashSet::<SocketAddr>::new();
|
let mut endpoints = HashSet::<SocketAddr>::new();
|
||||||
for (ip, port) in addrs.iter().copied() {
|
for (ip, port) in addrs.iter().copied() {
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::net::IpAddr;
|
use std::net::IpAddr;
|
||||||
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use tokio::sync::oneshot;
|
||||||
use tokio::task::JoinSet;
|
use tokio::task::JoinSet;
|
||||||
use tokio::time::timeout;
|
use tokio::time::timeout;
|
||||||
use tracing::{debug, info};
|
use tracing::{debug, info};
|
||||||
@@ -18,7 +20,18 @@ use std::time::Instant;
|
|||||||
|
|
||||||
const STUN_BATCH_TIMEOUT: Duration = Duration::from_secs(5);
|
const STUN_BATCH_TIMEOUT: Duration = Duration::from_secs(5);
|
||||||
const STUN_BATCH_TCP_FALLBACK_TIMEOUT: Duration = Duration::from_secs(12);
|
const STUN_BATCH_TCP_FALLBACK_TIMEOUT: Duration = Duration::from_secs(12);
|
||||||
|
const STUN_CACHE_TTL: Duration = Duration::from_secs(600);
|
||||||
|
const HTTP_FAILURE_TTL: Duration = Duration::from_secs(60);
|
||||||
|
|
||||||
|
// Discovery ownership and source-bound cache admission for this pool generation.
|
||||||
|
mod state;
|
||||||
|
pub(super) use state::NatDiscovery;
|
||||||
|
|
||||||
|
// Local network regressions exercise cancellation before discovery publication.
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests;
|
||||||
|
|
||||||
|
/// Probes the configured or default STUN endpoint with both address families.
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub async fn stun_probe(stun_addr: Option<String>) -> Result<crate::network::stun::DualStunResult> {
|
pub async fn stun_probe(stun_addr: Option<String>) -> Result<crate::network::stun::DualStunResult> {
|
||||||
let stun_addr = stun_addr.unwrap_or_else(|| {
|
let stun_addr = stun_addr.unwrap_or_else(|| {
|
||||||
@@ -35,6 +48,7 @@ pub async fn stun_probe(stun_addr: Option<String>) -> Result<crate::network::stu
|
|||||||
stun_probe_dual_with_tcp_fallback(&stun_addr, false).await
|
stun_probe_dual_with_tcp_fallback(&stun_addr, false).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Detects a public IPv4 address using the default HTTP endpoints.
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub async fn detect_public_ip() -> Option<IpAddr> {
|
pub async fn detect_public_ip() -> Option<IpAddr> {
|
||||||
let urls = crate::config::defaults::default_http_ip_detect_urls();
|
let urls = crate::config::defaults::default_http_ip_detect_urls();
|
||||||
@@ -151,6 +165,7 @@ impl MePool {
|
|||||||
(live_servers, best_reflected)
|
(live_servers, best_reflected)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Applies the configured or HTTP-detected NAT IP to non-public addresses.
|
||||||
pub(super) fn translate_ip_for_nat(&self, ip: IpAddr) -> IpAddr {
|
pub(super) fn translate_ip_for_nat(&self, ip: IpAddr) -> IpAddr {
|
||||||
let nat_ip = self.nat_runtime.nat_ip_cfg.or_else(|| {
|
let nat_ip = self.nat_runtime.nat_ip_cfg.or_else(|| {
|
||||||
self.nat_runtime
|
self.nat_runtime
|
||||||
@@ -177,6 +192,7 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Selects the NAT source IP without replacing the actual TCP source port.
|
||||||
pub(super) fn translate_our_addr_with_reflection(
|
pub(super) fn translate_our_addr_with_reflection(
|
||||||
&self,
|
&self,
|
||||||
addr: std::net::SocketAddr,
|
addr: std::net::SocketAddr,
|
||||||
@@ -203,7 +219,8 @@ impl MePool {
|
|||||||
std::net::SocketAddr::new(ip, addr.port())
|
std::net::SocketAddr::new(ip, addr.port())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn maybe_detect_nat_ip(&self, local_ip: IpAddr) -> Option<IpAddr> {
|
/// Waits for pool-owned HTTP discovery without owning its cancellation.
|
||||||
|
pub(super) async fn maybe_detect_nat_ip(self: &Arc<Self>, local_ip: IpAddr) -> Option<IpAddr> {
|
||||||
if self.nat_runtime.nat_ip_cfg.is_some() {
|
if self.nat_runtime.nat_ip_cfg.is_some() {
|
||||||
return self.nat_runtime.nat_ip_cfg;
|
return self.nat_runtime.nat_ip_cfg;
|
||||||
}
|
}
|
||||||
@@ -220,103 +237,117 @@ impl MePool {
|
|||||||
return Some(ip);
|
return Some(ip);
|
||||||
}
|
}
|
||||||
|
|
||||||
let Some(ip) = detect_public_ipv4_http(&self.nat_runtime.http_ip_detect_urls).await else {
|
let mut flight = self.nat_runtime.discovery.http.clone().lock_owned().await;
|
||||||
return None;
|
if let Some(ip) = *self.nat_runtime.nat_ip_detected.read().await {
|
||||||
};
|
return Some(ip);
|
||||||
{
|
|
||||||
let mut guard = self.nat_runtime.nat_ip_detected.write().await;
|
|
||||||
*guard = Some(IpAddr::V4(ip));
|
|
||||||
}
|
}
|
||||||
info!(public_ip = %ip, "Auto-detected public IP for NAT translation");
|
if flight.is_some_and(|until| Instant::now() < until) {
|
||||||
Some(IpAddr::V4(ip))
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let pool = self.clone();
|
||||||
|
let (send, receive) = oneshot::channel();
|
||||||
|
// Transfer singleflight ownership before the producer can be cancelled or polled.
|
||||||
|
self.lifecycle
|
||||||
|
.spawn_producer(async move {
|
||||||
|
let detected = detect_public_ipv4_http(&pool.nat_runtime.http_ip_detect_urls)
|
||||||
|
.await
|
||||||
|
.map(IpAddr::V4);
|
||||||
|
if let Some(ip) = detected {
|
||||||
|
*pool.nat_runtime.nat_ip_detected.write().await = Some(ip);
|
||||||
|
*flight = None;
|
||||||
|
info!(public_ip = %ip, "Auto-detected public IP for NAT translation");
|
||||||
|
} else {
|
||||||
|
*flight = Some(Instant::now() + HTTP_FAILURE_TTL);
|
||||||
|
}
|
||||||
|
let _ = send.send(detected);
|
||||||
|
})
|
||||||
|
.ok()?;
|
||||||
|
receive.await.ok().flatten()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Waits for a pool-owned STUN refresh in the exact source-address context.
|
||||||
pub(super) async fn maybe_reflect_public_addr(
|
pub(super) async fn maybe_reflect_public_addr(
|
||||||
&self,
|
self: &Arc<Self>,
|
||||||
family: IpFamily,
|
family: IpFamily,
|
||||||
bind_ip: Option<IpAddr>,
|
bind_ip: Option<IpAddr>,
|
||||||
) -> Option<std::net::SocketAddr> {
|
) -> Option<std::net::SocketAddr> {
|
||||||
const STUN_CACHE_TTL: Duration = Duration::from_secs(600);
|
if let Some(bind_ip) = bind_ip {
|
||||||
let use_shared_cache = bind_ip.is_none();
|
return self.reflect_bound_addr(family, bind_ip).await;
|
||||||
if !use_shared_cache {
|
|
||||||
match (family, bind_ip) {
|
|
||||||
(IpFamily::V4, Some(IpAddr::V4(_)))
|
|
||||||
| (IpFamily::V6, Some(IpAddr::V6(_)))
|
|
||||||
| (_, None) => {}
|
|
||||||
_ => return None,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
// Backoff window
|
if let Some(cached) = self.cached_shared_reflection(family).await {
|
||||||
if use_shared_cache
|
return cached;
|
||||||
&& let Some(until) = *self.nat_runtime.stun_backoff_until.read().await
|
}
|
||||||
&& Instant::now() < until
|
let singleflight = match family {
|
||||||
{
|
IpFamily::V4 => self.nat_runtime.nat_reflection_singleflight_v4.clone(),
|
||||||
if let Ok(cache) = self.nat_runtime.nat_reflection_cache.try_lock() {
|
IpFamily::V6 => self.nat_runtime.nat_reflection_singleflight_v6.clone(),
|
||||||
let slot = match family {
|
};
|
||||||
IpFamily::V4 => cache.v4,
|
let flight = singleflight.lock_owned().await;
|
||||||
IpFamily::V6 => cache.v6,
|
if let Some(cached) = self.cached_shared_reflection(family).await {
|
||||||
};
|
return cached;
|
||||||
return slot.map(|(_, addr)| addr);
|
|
||||||
}
|
|
||||||
return None;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if use_shared_cache && let Ok(mut cache) = self.nat_runtime.nat_reflection_cache.try_lock()
|
let pool = self.clone();
|
||||||
{
|
let (send, receive) = oneshot::channel();
|
||||||
let slot = match family {
|
self.lifecycle
|
||||||
IpFamily::V4 => &mut cache.v4,
|
.spawn_producer(async move {
|
||||||
IpFamily::V6 => &mut cache.v6,
|
let _flight = flight;
|
||||||
};
|
let attempt = pool
|
||||||
if let Some((ts, addr)) = slot
|
.nat_runtime
|
||||||
&& ts.elapsed() < STUN_CACHE_TTL
|
.nat_probe_attempts
|
||||||
{
|
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||||
return Some(*addr);
|
let reflected = pool.refresh_stun(family, None, attempt).await;
|
||||||
}
|
if let Some(addr) = reflected {
|
||||||
}
|
// Cache readers must never cause a completed refresh to be discarded.
|
||||||
|
let mut cache = pool.nat_runtime.nat_reflection_cache.lock().await;
|
||||||
let _singleflight_guard = if use_shared_cache {
|
let slot = match family {
|
||||||
Some(match family {
|
IpFamily::V4 => &mut cache.v4,
|
||||||
IpFamily::V4 => self.nat_runtime.nat_reflection_singleflight_v4.lock().await,
|
IpFamily::V6 => &mut cache.v6,
|
||||||
IpFamily::V6 => self.nat_runtime.nat_reflection_singleflight_v6.lock().await,
|
};
|
||||||
|
*slot = Some((Instant::now(), addr));
|
||||||
|
pool.nat_runtime
|
||||||
|
.nat_probe_attempts
|
||||||
|
.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||||
|
} else {
|
||||||
|
let backoff = Duration::from_secs(60 * 2u64.pow((attempt as u32).min(6)));
|
||||||
|
*pool.nat_runtime.stun_backoff_until.write().await =
|
||||||
|
Some(Instant::now() + backoff);
|
||||||
|
}
|
||||||
|
let _ = send.send(reflected);
|
||||||
})
|
})
|
||||||
} else {
|
.ok()?;
|
||||||
None
|
receive.await.ok().flatten()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cached_shared_reflection(
|
||||||
|
&self,
|
||||||
|
family: IpFamily,
|
||||||
|
) -> Option<Option<std::net::SocketAddr>> {
|
||||||
|
let backoff = self
|
||||||
|
.nat_runtime
|
||||||
|
.stun_backoff_until
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.is_some_and(|until| Instant::now() < until);
|
||||||
|
let cache = self.nat_runtime.nat_reflection_cache.lock().await;
|
||||||
|
let slot = match family {
|
||||||
|
IpFamily::V4 => cache.v4,
|
||||||
|
IpFamily::V6 => cache.v6,
|
||||||
};
|
};
|
||||||
|
// Preserve the existing stale-result fallback during shared failure backoff.
|
||||||
if use_shared_cache
|
if backoff {
|
||||||
&& let Some(until) = *self.nat_runtime.stun_backoff_until.read().await
|
return Some(slot.map(|(_, addr)| addr));
|
||||||
&& Instant::now() < until
|
|
||||||
{
|
|
||||||
if let Ok(cache) = self.nat_runtime.nat_reflection_cache.try_lock() {
|
|
||||||
let slot = match family {
|
|
||||||
IpFamily::V4 => cache.v4,
|
|
||||||
IpFamily::V6 => cache.v6,
|
|
||||||
};
|
|
||||||
return slot.map(|(_, addr)| addr);
|
|
||||||
}
|
|
||||||
return None;
|
|
||||||
}
|
}
|
||||||
|
slot.filter(|(created, _)| created.elapsed() < STUN_CACHE_TTL)
|
||||||
|
.map(|(_, addr)| Some(addr))
|
||||||
|
}
|
||||||
|
|
||||||
if use_shared_cache && let Ok(mut cache) = self.nat_runtime.nat_reflection_cache.try_lock()
|
async fn refresh_stun(
|
||||||
{
|
&self,
|
||||||
let slot = match family {
|
family: IpFamily,
|
||||||
IpFamily::V4 => &mut cache.v4,
|
bind_ip: Option<IpAddr>,
|
||||||
IpFamily::V6 => &mut cache.v6,
|
attempt: u8,
|
||||||
};
|
) -> Option<std::net::SocketAddr> {
|
||||||
if let Some((ts, addr)) = slot
|
|
||||||
&& ts.elapsed() < STUN_CACHE_TTL
|
|
||||||
{
|
|
||||||
return Some(*addr);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let attempt = if use_shared_cache {
|
|
||||||
self.nat_runtime
|
|
||||||
.nat_probe_attempts
|
|
||||||
.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
|
|
||||||
} else {
|
|
||||||
0
|
|
||||||
};
|
|
||||||
let configured_servers = self.configured_stun_servers();
|
let configured_servers = self.configured_stun_servers();
|
||||||
let live_snapshot = self.nat_runtime.nat_stun_live_servers.read().await.clone();
|
let live_snapshot = self.nat_runtime.nat_stun_live_servers.read().await.clone();
|
||||||
let primary_servers = if live_snapshot.is_empty() {
|
let primary_servers = if live_snapshot.is_empty() {
|
||||||
@@ -348,33 +379,14 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some(reflected_addr) = selected_reflected {
|
if let Some(reflected_addr) = selected_reflected {
|
||||||
if use_shared_cache {
|
|
||||||
self.nat_runtime
|
|
||||||
.nat_probe_attempts
|
|
||||||
.store(0, std::sync::atomic::Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
info!(
|
info!(
|
||||||
family = ?family,
|
family = ?family,
|
||||||
live_servers = live_server_count,
|
live_servers = live_server_count,
|
||||||
"STUN-Quorum reached, IP: {}",
|
"STUN-Quorum reached, IP: {}",
|
||||||
reflected_addr.ip()
|
reflected_addr.ip()
|
||||||
);
|
);
|
||||||
if use_shared_cache
|
|
||||||
&& let Ok(mut cache) = self.nat_runtime.nat_reflection_cache.try_lock()
|
|
||||||
{
|
|
||||||
let slot = match family {
|
|
||||||
IpFamily::V4 => &mut cache.v4,
|
|
||||||
IpFamily::V6 => &mut cache.v6,
|
|
||||||
};
|
|
||||||
*slot = Some((Instant::now(), reflected_addr));
|
|
||||||
}
|
|
||||||
return Some(reflected_addr);
|
return Some(reflected_addr);
|
||||||
}
|
}
|
||||||
|
|
||||||
if use_shared_cache {
|
|
||||||
let backoff = Duration::from_secs(60 * 2u64.pow((attempt as u32).min(6)));
|
|
||||||
*self.nat_runtime.stun_backoff_until.write().await = Some(Instant::now() + backoff);
|
|
||||||
}
|
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
use std::collections::{BTreeSet, HashMap};
|
||||||
|
use std::net::{IpAddr, SocketAddr};
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use tokio::sync::{Mutex, oneshot};
|
||||||
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
|
use super::{IpFamily, MePool, STUN_CACHE_TTL};
|
||||||
|
|
||||||
|
// Context retirement and TTL tests use the same gates as production discovery.
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests;
|
||||||
|
|
||||||
|
/// Pool-local discovery gates; bound entries are limited to configured egress addresses.
|
||||||
|
#[derive(Default)]
|
||||||
|
pub(in crate::transport::middle_proxy) struct NatDiscovery {
|
||||||
|
pub(super) http: Arc<Mutex<Option<Instant>>>,
|
||||||
|
bound: Mutex<HashMap<(IpFamily, IpAddr), Arc<BoundDiscovery>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct BoundDiscovery {
|
||||||
|
state: Arc<Mutex<BoundDiscoveryState>>,
|
||||||
|
cancel: CancellationToken,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct BoundDiscoveryState {
|
||||||
|
reflection: Option<(Instant, SocketAddr)>,
|
||||||
|
retry_after: Option<Instant>,
|
||||||
|
attempt: u8,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NatDiscovery {
|
||||||
|
async fn bound_context(
|
||||||
|
&self,
|
||||||
|
family: IpFamily,
|
||||||
|
bind_ip: IpAddr,
|
||||||
|
allowed: &BTreeSet<IpAddr>,
|
||||||
|
) -> Option<Arc<BoundDiscovery>> {
|
||||||
|
let mut contexts = self.bound.lock().await;
|
||||||
|
// Retired producers cannot republish into the map or a replacement context.
|
||||||
|
contexts.retain(|(entry_family, ip), context| {
|
||||||
|
let keep = *entry_family != family || allowed.contains(ip);
|
||||||
|
if !keep {
|
||||||
|
context.cancel.cancel();
|
||||||
|
}
|
||||||
|
keep
|
||||||
|
});
|
||||||
|
if !allowed.contains(&bind_ip) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(contexts.entry((family, bind_ip)).or_default().clone())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MePool {
|
||||||
|
/// Refreshes one configured source without borrowing another source's cached reflection.
|
||||||
|
pub(super) async fn reflect_bound_addr(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
family: IpFamily,
|
||||||
|
bind_ip: IpAddr,
|
||||||
|
) -> Option<SocketAddr> {
|
||||||
|
if bind_ip.is_ipv6() != matches!(family, IpFamily::V6) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let allowed = self
|
||||||
|
.upstream
|
||||||
|
.as_ref()?
|
||||||
|
.direct_bind_ips_for_family(bind_ip.is_ipv6())
|
||||||
|
.await;
|
||||||
|
let context = self
|
||||||
|
.nat_runtime
|
||||||
|
.discovery
|
||||||
|
.bound_context(family, bind_ip, &allowed)
|
||||||
|
.await?;
|
||||||
|
let mut flight = tokio::select! {
|
||||||
|
biased;
|
||||||
|
_ = context.cancel.cancelled() => return None,
|
||||||
|
guard = context.state.clone().lock_owned() => guard,
|
||||||
|
};
|
||||||
|
if let Some((created, addr)) = flight.reflection
|
||||||
|
&& created.elapsed() < STUN_CACHE_TTL
|
||||||
|
{
|
||||||
|
return Some(addr);
|
||||||
|
}
|
||||||
|
if flight
|
||||||
|
.retry_after
|
||||||
|
.is_some_and(|until| Instant::now() < until)
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let pool = self.clone();
|
||||||
|
let (send, receive) = oneshot::channel();
|
||||||
|
self.lifecycle.spawn_producer(async move {
|
||||||
|
tokio::select! {
|
||||||
|
biased;
|
||||||
|
_ = context.cancel.cancelled() => {}
|
||||||
|
_ = async {
|
||||||
|
let reflected = pool.refresh_stun(family, Some(bind_ip), flight.attempt).await;
|
||||||
|
if let Some(addr) = reflected {
|
||||||
|
flight.reflection = Some((Instant::now(), addr));
|
||||||
|
flight.retry_after = None;
|
||||||
|
flight.attempt = 0;
|
||||||
|
} else {
|
||||||
|
// Completed failures must also survive a short reconnect deadline.
|
||||||
|
let backoff = Duration::from_secs(60 * 2u64.pow(flight.attempt as u32));
|
||||||
|
flight.retry_after = Some(Instant::now() + backoff);
|
||||||
|
flight.attempt = flight.attempt.saturating_add(1).min(6);
|
||||||
|
}
|
||||||
|
let _ = send.send(reflected);
|
||||||
|
} => {}
|
||||||
|
}
|
||||||
|
}).ok()?;
|
||||||
|
receive.await.ok().flatten()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::transport::middle_proxy::pool_nat::tests::lifecycle::set_bind_addresses;
|
||||||
|
use crate::transport::middle_proxy::pool_nat::tests::{
|
||||||
|
PUBLIC_IP, TEST_DEADLINE, nat_pool, stun_server,
|
||||||
|
};
|
||||||
|
use std::net::Ipv4Addr;
|
||||||
|
use std::sync::atomic::Ordering;
|
||||||
|
use tokio::time::timeout;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn bound_positive_ttl_refreshes_without_publishing_to_default_cache() {
|
||||||
|
let stun = stun_server("127.0.0.1:0".parse().unwrap(), PUBLIC_IP.into()).await;
|
||||||
|
stun.release.add_permits(16);
|
||||||
|
let mut pool = nat_pool(vec![stun.addr.to_string()], vec![]).await;
|
||||||
|
set_bind_addresses(&mut pool, &["127.0.0.1"]);
|
||||||
|
let ip = IpAddr::V4(Ipv4Addr::LOCALHOST);
|
||||||
|
let allowed = BTreeSet::from([ip]);
|
||||||
|
let context = pool
|
||||||
|
.nat_runtime
|
||||||
|
.discovery
|
||||||
|
.bound_context(IpFamily::V4, ip, &allowed)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V4, Some(ip))
|
||||||
|
.await
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V4, Some(ip))
|
||||||
|
.await
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
assert_eq!(stun.requests.load(Ordering::Relaxed), 1);
|
||||||
|
context.state.lock().await.reflection.as_mut().unwrap().0 = Instant::now() - STUN_CACHE_TTL;
|
||||||
|
assert!(
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V4, Some(ip))
|
||||||
|
.await
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
assert_eq!(stun.requests.load(Ordering::Relaxed), 2);
|
||||||
|
assert_eq!(pool.nat_runtime.nat_reflection_cache.lock().await.v4, None);
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn context_churn_is_bounded_and_retirement_does_not_mix_families() {
|
||||||
|
let discovery = NatDiscovery::default();
|
||||||
|
let v6: IpAddr = "::1".parse().unwrap();
|
||||||
|
let v6_context = discovery
|
||||||
|
.bound_context(IpFamily::V6, v6, &BTreeSet::from([v6]))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut previous: Option<Arc<BoundDiscovery>> = None;
|
||||||
|
for suffix in 1..=1000u32 {
|
||||||
|
let ip = IpAddr::V4(Ipv4Addr::from(0x7f000000 + suffix));
|
||||||
|
let context = discovery
|
||||||
|
.bound_context(IpFamily::V4, ip, &BTreeSet::from([ip]))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
if let Some(old) = previous.replace(context.clone()) {
|
||||||
|
assert!(old.cancel.is_cancelled());
|
||||||
|
}
|
||||||
|
assert!(!context.cancel.is_cancelled());
|
||||||
|
assert!(!v6_context.cancel.is_cancelled());
|
||||||
|
assert_eq!(discovery.bound.lock().await.len(), 2);
|
||||||
|
}
|
||||||
|
let denied = IpAddr::V4(Ipv4Addr::LOCALHOST);
|
||||||
|
assert!(
|
||||||
|
discovery
|
||||||
|
.bound_context(IpFamily::V4, denied, &BTreeSet::new())
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(previous.unwrap().cancel.is_cancelled());
|
||||||
|
assert_eq!(discovery.bound.lock().await.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn retiring_bound_context_cancels_its_producer_and_prevents_late_publication() {
|
||||||
|
let stun = stun_server("127.0.0.1:0".parse().unwrap(), PUBLIC_IP.into()).await;
|
||||||
|
let mut pool = nat_pool(vec![stun.addr.to_string()], vec![]).await;
|
||||||
|
set_bind_addresses(&mut pool, &["127.0.0.1"]);
|
||||||
|
let ip = IpAddr::V4(Ipv4Addr::LOCALHOST);
|
||||||
|
let context = pool
|
||||||
|
.nat_runtime
|
||||||
|
.discovery
|
||||||
|
.bound_context(IpFamily::V4, ip, &BTreeSet::from([ip]))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let waiter_pool = pool.clone();
|
||||||
|
let waiter = tokio::spawn(async move {
|
||||||
|
waiter_pool
|
||||||
|
.maybe_reflect_public_addr(IpFamily::V4, Some(ip))
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
stun.received().await;
|
||||||
|
assert!(
|
||||||
|
pool.nat_runtime
|
||||||
|
.discovery
|
||||||
|
.bound_context(IpFamily::V4, ip, &BTreeSet::new())
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(timeout(TEST_DEADLINE, waiter).await.unwrap().unwrap(), None);
|
||||||
|
stun.release.add_permits(16);
|
||||||
|
assert_eq!(context.state.lock().await.reflection, None);
|
||||||
|
assert!(pool.nat_runtime.discovery.bound.lock().await.is_empty());
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn bound_failure_retry_deadline_expires_and_success_resets_backoff() {
|
||||||
|
let stun = stun_server("127.0.0.1:0".parse().unwrap(), PUBLIC_IP.into()).await;
|
||||||
|
stun.release.add_permits(16);
|
||||||
|
let mut pool = nat_pool(vec![stun.addr.to_string()], vec![]).await;
|
||||||
|
set_bind_addresses(&mut pool, &["127.0.0.1"]);
|
||||||
|
let ip = IpAddr::V4(Ipv4Addr::LOCALHOST);
|
||||||
|
let context = pool
|
||||||
|
.nat_runtime
|
||||||
|
.discovery
|
||||||
|
.bound_context(IpFamily::V4, ip, &BTreeSet::from([ip]))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
{
|
||||||
|
let mut state = context.state.lock().await;
|
||||||
|
state.retry_after = Some(Instant::now() + Duration::from_secs(60));
|
||||||
|
state.attempt = 3;
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V4, Some(ip))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(stun.requests.load(Ordering::Relaxed), 0);
|
||||||
|
context.state.lock().await.retry_after = Some(Instant::now() - Duration::from_secs(1));
|
||||||
|
assert!(
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V4, Some(ip))
|
||||||
|
.await
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
assert_eq!(context.state.lock().await.attempt, 0);
|
||||||
|
assert_eq!(context.state.lock().await.retry_after, None);
|
||||||
|
assert_eq!(stun.requests.load(Ordering::Relaxed), 1);
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
@@ -0,0 +1,294 @@
|
|||||||
|
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
|
use tokio::net::{TcpListener, UdpSocket};
|
||||||
|
use tokio::sync::Semaphore;
|
||||||
|
use tokio::task::JoinHandle;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::network::probe::NetworkDecision;
|
||||||
|
use crate::transport::middle_proxy::pool_writer_security_tests::make_pool_with_decision;
|
||||||
|
|
||||||
|
/// Public IPv4 used as source material by local discovery and encrypted ME fixtures.
|
||||||
|
pub(super) const PUBLIC_IP: Ipv4Addr = Ipv4Addr::new(91, 108, 56, 1);
|
||||||
|
/// Bounds asynchronous fixture assertions without relying on external services.
|
||||||
|
pub(super) const TEST_DEADLINE: Duration = Duration::from_secs(2);
|
||||||
|
|
||||||
|
// Concurrency and end-to-end regressions share gated local discovery servers.
|
||||||
|
pub(super) mod lifecycle;
|
||||||
|
mod recovery;
|
||||||
|
|
||||||
|
/// Gated local discovery service with observable request counts.
|
||||||
|
pub(super) struct DiscoveryServer {
|
||||||
|
/// Listening address selected by the kernel.
|
||||||
|
pub(super) addr: SocketAddr,
|
||||||
|
/// Number of requests actually received from clients.
|
||||||
|
pub(super) requests: Arc<AtomicUsize>,
|
||||||
|
arrived: Arc<Semaphore>,
|
||||||
|
/// One permit releases one discovery response.
|
||||||
|
pub(super) release: Arc<Semaphore>,
|
||||||
|
task: JoinHandle<()>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for DiscoveryServer {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.task.abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DiscoveryServer {
|
||||||
|
/// Waits until a real discovery request reaches the server.
|
||||||
|
pub(super) async fn received(&self) {
|
||||||
|
tokio::time::timeout(TEST_DEADLINE, self.arrived.acquire())
|
||||||
|
.await
|
||||||
|
.expect("discovery request must reach the local server")
|
||||||
|
.unwrap()
|
||||||
|
.forget();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Serves valid STUN responses with a deliberately different reflected port.
|
||||||
|
pub(super) async fn stun_server(bind: SocketAddr, reflected: IpAddr) -> DiscoveryServer {
|
||||||
|
stun_server_response(bind, Some(reflected), Duration::ZERO).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn stun_server_response(
|
||||||
|
bind: SocketAddr,
|
||||||
|
reflected: Option<IpAddr>,
|
||||||
|
delay: Duration,
|
||||||
|
) -> DiscoveryServer {
|
||||||
|
let socket = UdpSocket::bind(bind).await.unwrap();
|
||||||
|
let addr = socket.local_addr().unwrap();
|
||||||
|
let requests = Arc::new(AtomicUsize::new(0));
|
||||||
|
let arrived = Arc::new(Semaphore::new(0));
|
||||||
|
let release = Arc::new(Semaphore::new(0));
|
||||||
|
let count = requests.clone();
|
||||||
|
let signal = arrived.clone();
|
||||||
|
let gate = release.clone();
|
||||||
|
let task = tokio::spawn(async move {
|
||||||
|
let mut request = [0u8; 512];
|
||||||
|
loop {
|
||||||
|
let (len, peer) = socket.recv_from(&mut request).await.unwrap();
|
||||||
|
assert!(len >= 20);
|
||||||
|
count.fetch_add(1, Ordering::Relaxed);
|
||||||
|
signal.add_permits(1);
|
||||||
|
gate.acquire().await.unwrap().forget();
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
let Some(reflected) = reflected else {
|
||||||
|
socket.send_to(&[0u8], peer).await.unwrap();
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let (family, ip) = match reflected {
|
||||||
|
IpAddr::V4(ip) => (1u8, ip.octets().to_vec()),
|
||||||
|
IpAddr::V6(ip) => (2u8, ip.octets().to_vec()),
|
||||||
|
};
|
||||||
|
let attribute_len = 4 + ip.len();
|
||||||
|
let mut response = vec![0u8; 24 + attribute_len];
|
||||||
|
response[..2].copy_from_slice(&0x0101u16.to_be_bytes());
|
||||||
|
response[2..4].copy_from_slice(&((4 + attribute_len) as u16).to_be_bytes());
|
||||||
|
response[4..20].copy_from_slice(&request[4..20]);
|
||||||
|
response[20..22].copy_from_slice(&1u16.to_be_bytes());
|
||||||
|
response[22..24].copy_from_slice(&(attribute_len as u16).to_be_bytes());
|
||||||
|
response[25] = family;
|
||||||
|
response[26..28].copy_from_slice(&45678u16.to_be_bytes());
|
||||||
|
response[28..].copy_from_slice(&ip);
|
||||||
|
socket.send_to(&response, peer).await.unwrap();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
DiscoveryServer {
|
||||||
|
addr,
|
||||||
|
requests,
|
||||||
|
arrived,
|
||||||
|
release,
|
||||||
|
task,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn http_server(body: &'static str) -> DiscoveryServer {
|
||||||
|
http_server_with_delay(body, Duration::ZERO).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn http_server_with_delay(body: &'static str, delay: Duration) -> DiscoveryServer {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let addr = listener.local_addr().unwrap();
|
||||||
|
let requests = Arc::new(AtomicUsize::new(0));
|
||||||
|
let arrived = Arc::new(Semaphore::new(0));
|
||||||
|
let release = Arc::new(Semaphore::new(0));
|
||||||
|
let count = requests.clone();
|
||||||
|
let signal = arrived.clone();
|
||||||
|
let gate = release.clone();
|
||||||
|
let task = tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
let (mut stream, _) = listener.accept().await.unwrap();
|
||||||
|
let mut request = [0u8; 4096];
|
||||||
|
let _ = stream.read(&mut request).await.unwrap();
|
||||||
|
count.fetch_add(1, Ordering::Relaxed);
|
||||||
|
signal.add_permits(1);
|
||||||
|
gate.acquire().await.unwrap().forget();
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
let response = format!(
|
||||||
|
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
|
||||||
|
body.len(),
|
||||||
|
body
|
||||||
|
);
|
||||||
|
let _ = stream.write_all(response.as_bytes()).await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
DiscoveryServer {
|
||||||
|
addr,
|
||||||
|
requests,
|
||||||
|
arrived,
|
||||||
|
release,
|
||||||
|
task,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds an isolated dual-stack pool using only caller-supplied discovery services.
|
||||||
|
pub(super) async fn nat_pool(servers: Vec<String>, urls: Vec<String>) -> Arc<MePool> {
|
||||||
|
let mut pool = make_pool_with_decision(NetworkDecision {
|
||||||
|
ipv4_me: true,
|
||||||
|
ipv6_me: true,
|
||||||
|
..NetworkDecision::default()
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
let pool_mut = Arc::get_mut(&mut pool).unwrap();
|
||||||
|
let nat = Arc::get_mut(&mut pool_mut.nat_runtime).unwrap();
|
||||||
|
nat.nat_probe = true;
|
||||||
|
nat.nat_stun_servers = servers;
|
||||||
|
nat.http_ip_detect_urls = urls;
|
||||||
|
nat.nat_probe_concurrency = 2;
|
||||||
|
pool
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_for_reflection(pool: &MePool, family: IpFamily) -> SocketAddr {
|
||||||
|
tokio::time::timeout(TEST_DEADLINE, async {
|
||||||
|
loop {
|
||||||
|
let cached = {
|
||||||
|
let cache = pool.nat_runtime.nat_reflection_cache.lock().await;
|
||||||
|
match family {
|
||||||
|
IpFamily::V4 => cache.v4,
|
||||||
|
IpFamily::V6 => cache.v6,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if let Some((created, addr)) = cached
|
||||||
|
&& created.elapsed() < Duration::from_secs(600)
|
||||||
|
{
|
||||||
|
return addr;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("pool-owned STUN refresh must publish after its caller is cancelled")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn expired_stun_refresh_survives_reconnect_timeout() {
|
||||||
|
let fast = stun_server("127.0.0.1:0".parse().unwrap(), PUBLIC_IP.into()).await;
|
||||||
|
let slow = stun_server("127.0.0.1:0".parse().unwrap(), PUBLIC_IP.into()).await;
|
||||||
|
fast.release.add_permits(16);
|
||||||
|
let pool = nat_pool(vec![fast.addr.to_string(), slow.addr.to_string()], vec![]).await;
|
||||||
|
pool.nat_runtime.nat_reflection_cache.lock().await.v4 = Some((
|
||||||
|
Instant::now() - Duration::from_secs(601),
|
||||||
|
SocketAddr::new(PUBLIC_IP.into(), 1),
|
||||||
|
));
|
||||||
|
let caller_pool = pool.clone();
|
||||||
|
let caller = tokio::spawn(async move {
|
||||||
|
tokio::time::timeout(
|
||||||
|
Duration::from_millis(100),
|
||||||
|
caller_pool.maybe_reflect_public_addr(IpFamily::V4, None),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
fast.received().await;
|
||||||
|
slow.received().await;
|
||||||
|
assert!(caller.await.unwrap().is_err());
|
||||||
|
slow.release.add_permits(16);
|
||||||
|
assert_eq!(
|
||||||
|
wait_for_reflection(&pool, IpFamily::V4).await.ip(),
|
||||||
|
PUBLIC_IP
|
||||||
|
);
|
||||||
|
assert_eq!(fast.requests.load(Ordering::Relaxed), 1);
|
||||||
|
assert_eq!(slow.requests.load(Ordering::Relaxed), 1);
|
||||||
|
assert_eq!(
|
||||||
|
pool.nat_runtime.nat_probe_attempts.load(Ordering::Relaxed),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn http_discovery_survives_reconnect_timeout() {
|
||||||
|
let http = http_server("91.108.56.1").await;
|
||||||
|
let pool = nat_pool(vec![], vec![format!("http://{}/", http.addr)]).await;
|
||||||
|
let caller_pool = pool.clone();
|
||||||
|
let caller = tokio::spawn(async move {
|
||||||
|
tokio::time::timeout(
|
||||||
|
Duration::from_millis(100),
|
||||||
|
caller_pool.maybe_detect_nat_ip(Ipv4Addr::LOCALHOST.into()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
http.received().await;
|
||||||
|
assert!(caller.await.unwrap().is_err());
|
||||||
|
http.release.add_permits(16);
|
||||||
|
tokio::time::timeout(TEST_DEADLINE, async {
|
||||||
|
while pool.nat_runtime.nat_ip_detected.read().await.is_none() {
|
||||||
|
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("pool-owned HTTP discovery must publish after caller cancellation");
|
||||||
|
assert_eq!(
|
||||||
|
*pool.nat_runtime.nat_ip_detected.read().await,
|
||||||
|
Some(PUBLIC_IP.into())
|
||||||
|
);
|
||||||
|
assert_eq!(http.requests.load(Ordering::Relaxed), 1);
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn failed_http_discovery_is_reused_during_negative_cache_window() {
|
||||||
|
let http = http_server("not an IP address").await;
|
||||||
|
http.release.add_permits(16);
|
||||||
|
let pool = nat_pool(vec![], vec![format!("http://{}/", http.addr)]).await;
|
||||||
|
for _ in 0..2 {
|
||||||
|
assert!(
|
||||||
|
pool.maybe_detect_nat_ip(Ipv4Addr::LOCALHOST.into())
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
http.requests.load(Ordering::Relaxed),
|
||||||
|
1,
|
||||||
|
"a failed HTTP lookup must not restart ahead of every STUN/handshake attempt"
|
||||||
|
);
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn successful_stun_publication_waits_for_cache_reader() {
|
||||||
|
let server = stun_server("127.0.0.1:0".parse().unwrap(), PUBLIC_IP.into()).await;
|
||||||
|
let pool = nat_pool(vec![server.addr.to_string()], vec![]).await;
|
||||||
|
let caller_pool = pool.clone();
|
||||||
|
let caller = tokio::spawn(async move {
|
||||||
|
caller_pool
|
||||||
|
.maybe_reflect_public_addr(IpFamily::V4, None)
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
server.received().await;
|
||||||
|
let cache_reader = pool.nat_runtime.nat_reflection_cache.lock().await;
|
||||||
|
server.release.add_permits(16);
|
||||||
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
|
drop(cache_reader);
|
||||||
|
assert_eq!(caller.await.unwrap().unwrap().ip(), PUBLIC_IP);
|
||||||
|
assert_eq!(
|
||||||
|
wait_for_reflection(&pool, IpFamily::V4).await.ip(),
|
||||||
|
PUBLIC_IP
|
||||||
|
);
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
@@ -0,0 +1,342 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::config::{UpstreamConfig, UpstreamType};
|
||||||
|
use crate::transport::UpstreamManager;
|
||||||
|
|
||||||
|
/// Gives an isolated pool the same finite bind-address authority as production egress.
|
||||||
|
pub(in crate::transport::middle_proxy::pool_nat) fn set_bind_addresses(
|
||||||
|
pool: &mut Arc<MePool>,
|
||||||
|
addresses: &[&str],
|
||||||
|
) {
|
||||||
|
let manager = UpstreamManager::new(
|
||||||
|
vec![UpstreamConfig {
|
||||||
|
upstream_type: UpstreamType::Direct {
|
||||||
|
interface: None,
|
||||||
|
bind_addresses: Some(addresses.iter().map(|value| value.to_string()).collect()),
|
||||||
|
bindtodevice: None,
|
||||||
|
},
|
||||||
|
weight: 1,
|
||||||
|
enabled: true,
|
||||||
|
scopes: String::new(),
|
||||||
|
selected_scope: String::new(),
|
||||||
|
ipv4: None,
|
||||||
|
ipv6: None,
|
||||||
|
prefer: None,
|
||||||
|
}],
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
1000,
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
true,
|
||||||
|
pool.stats.clone(),
|
||||||
|
);
|
||||||
|
Arc::get_mut(pool).unwrap().upstream = Some(Arc::new(manager));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn cancelled_waiter_storm_shares_one_http_and_stun_producer() {
|
||||||
|
let http = http_server("91.108.56.1").await;
|
||||||
|
let stun = stun_server("127.0.0.1:0".parse().unwrap(), PUBLIC_IP.into()).await;
|
||||||
|
let pool = nat_pool(
|
||||||
|
vec![stun.addr.to_string()],
|
||||||
|
vec![format!("http://{}/", http.addr)],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let mut waiters = JoinSet::new();
|
||||||
|
for _ in 0..32 {
|
||||||
|
let pool = pool.clone();
|
||||||
|
waiters.spawn(async move {
|
||||||
|
tokio::join!(
|
||||||
|
pool.maybe_detect_nat_ip(Ipv4Addr::LOCALHOST.into()),
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V4, None),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
http.received().await;
|
||||||
|
stun.received().await;
|
||||||
|
waiters.abort_all();
|
||||||
|
while waiters.join_next().await.is_some() {}
|
||||||
|
http.release.add_permits(64);
|
||||||
|
stun.release.add_permits(64);
|
||||||
|
assert_eq!(
|
||||||
|
pool.maybe_detect_nat_ip(Ipv4Addr::LOCALHOST.into()).await,
|
||||||
|
Some(PUBLIC_IP.into())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V4, None)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.ip(),
|
||||||
|
PUBLIC_IP
|
||||||
|
);
|
||||||
|
assert_eq!(http.requests.load(Ordering::Relaxed), 1);
|
||||||
|
assert_eq!(stun.requests.load(Ordering::Relaxed), 1);
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn shutdown_cancels_discovery_and_releases_all_waiters() {
|
||||||
|
let http = http_server("91.108.56.1").await;
|
||||||
|
let stun = stun_server("127.0.0.1:0".parse().unwrap(), PUBLIC_IP.into()).await;
|
||||||
|
let pool = nat_pool(
|
||||||
|
vec![stun.addr.to_string()],
|
||||||
|
vec![format!("http://{}/", http.addr)],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let waiter_pool = pool.clone();
|
||||||
|
let waiter = tokio::spawn(async move {
|
||||||
|
tokio::join!(
|
||||||
|
waiter_pool.maybe_detect_nat_ip(Ipv4Addr::LOCALHOST.into()),
|
||||||
|
waiter_pool.maybe_reflect_public_addr(IpFamily::V4, None),
|
||||||
|
)
|
||||||
|
});
|
||||||
|
http.received().await;
|
||||||
|
stun.received().await;
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
assert_eq!(
|
||||||
|
timeout(TEST_DEADLINE, waiter).await.unwrap().unwrap(),
|
||||||
|
(None, None)
|
||||||
|
);
|
||||||
|
http.release.add_permits(64);
|
||||||
|
stun.release.add_permits(64);
|
||||||
|
assert_eq!(
|
||||||
|
pool.maybe_detect_nat_ip(Ipv4Addr::LOCALHOST.into()).await,
|
||||||
|
None
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V4, None).await,
|
||||||
|
None
|
||||||
|
);
|
||||||
|
assert_eq!(*pool.nat_runtime.nat_ip_detected.read().await, None);
|
||||||
|
assert_eq!(pool.nat_runtime.nat_reflection_cache.lock().await.v4, None);
|
||||||
|
assert_eq!(http.requests.load(Ordering::Relaxed), 1);
|
||||||
|
assert_eq!(stun.requests.load(Ordering::Relaxed), 1);
|
||||||
|
assert!(
|
||||||
|
pool.nat_runtime
|
||||||
|
.nat_reflection_singleflight_v4
|
||||||
|
.try_lock()
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
assert!(pool.nat_runtime.discovery.http.try_lock().is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn bound_refresh_survives_cancellation_and_reuses_only_its_source() {
|
||||||
|
let stun = stun_server("127.0.0.1:0".parse().unwrap(), PUBLIC_IP.into()).await;
|
||||||
|
let mut pool = nat_pool(vec![stun.addr.to_string()], vec![]).await;
|
||||||
|
set_bind_addresses(&mut pool, &["127.0.0.1", "127.0.0.2"]);
|
||||||
|
let bind = Some(Ipv4Addr::LOCALHOST.into());
|
||||||
|
let waiter_pool = pool.clone();
|
||||||
|
let waiter = tokio::spawn(async move {
|
||||||
|
waiter_pool
|
||||||
|
.maybe_reflect_public_addr(IpFamily::V4, bind)
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
stun.received().await;
|
||||||
|
waiter.abort();
|
||||||
|
assert!(waiter.await.unwrap_err().is_cancelled());
|
||||||
|
stun.release.add_permits(64);
|
||||||
|
assert_eq!(
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V4, bind)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.ip(),
|
||||||
|
PUBLIC_IP
|
||||||
|
);
|
||||||
|
assert_eq!(stun.requests.load(Ordering::Relaxed), 1);
|
||||||
|
assert_eq!(pool.nat_runtime.nat_reflection_cache.lock().await.v4, None);
|
||||||
|
let other = Some("127.0.0.2".parse().unwrap());
|
||||||
|
assert_eq!(
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V4, other)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.ip(),
|
||||||
|
PUBLIC_IP
|
||||||
|
);
|
||||||
|
assert_eq!(stun.requests.load(Ordering::Relaxed), 2);
|
||||||
|
assert!(
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V4, Some("127.0.0.3".parse().unwrap()))
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V6, bind)
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(stun.requests.load(Ordering::Relaxed), 2);
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn failed_bound_refresh_is_observable_after_caller_cancellation() {
|
||||||
|
let stun = stun_server_response("127.0.0.1:0".parse().unwrap(), None, Duration::ZERO).await;
|
||||||
|
let mut pool = nat_pool(vec![stun.addr.to_string()], vec![]).await;
|
||||||
|
set_bind_addresses(&mut pool, &["127.0.0.1"]);
|
||||||
|
let bind = Some(Ipv4Addr::LOCALHOST.into());
|
||||||
|
let waiter_pool = pool.clone();
|
||||||
|
let waiter = tokio::spawn(async move {
|
||||||
|
waiter_pool
|
||||||
|
.maybe_reflect_public_addr(IpFamily::V4, bind)
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
stun.received().await;
|
||||||
|
waiter.abort();
|
||||||
|
let _ = waiter.await;
|
||||||
|
// One response completes the original probe; a restarted probe would block forever here.
|
||||||
|
stun.release.add_permits(1);
|
||||||
|
assert_eq!(
|
||||||
|
timeout(
|
||||||
|
TEST_DEADLINE,
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V4, bind)
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
timeout(
|
||||||
|
TEST_DEADLINE,
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V4, bind)
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
assert_eq!(stun.requests.load(Ordering::Relaxed), 1);
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn http_negative_cache_expires_without_waiting_a_minute() {
|
||||||
|
let http = http_server("invalid").await;
|
||||||
|
http.release.add_permits(64);
|
||||||
|
let pool = nat_pool(vec![], vec![format!("http://{}/", http.addr)]).await;
|
||||||
|
assert_eq!(
|
||||||
|
pool.maybe_detect_nat_ip(Ipv4Addr::LOCALHOST.into()).await,
|
||||||
|
None
|
||||||
|
);
|
||||||
|
*pool.nat_runtime.discovery.http.lock().await = Some(Instant::now() - Duration::from_secs(1));
|
||||||
|
assert_eq!(
|
||||||
|
pool.maybe_detect_nat_ip(Ipv4Addr::LOCALHOST.into()).await,
|
||||||
|
None
|
||||||
|
);
|
||||||
|
assert_eq!(http.requests.load(Ordering::Relaxed), 2);
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn ipv6_refresh_is_independent_of_blocked_ipv4_refresh() {
|
||||||
|
let v4 = stun_server("127.0.0.1:0".parse().unwrap(), PUBLIC_IP.into()).await;
|
||||||
|
let public_v6: IpAddr = "2001:4860:4860::8888".parse().unwrap();
|
||||||
|
let v6 = stun_server("[::1]:0".parse().unwrap(), public_v6).await;
|
||||||
|
v6.release.add_permits(16);
|
||||||
|
let pool = nat_pool(vec![v4.addr.to_string(), v6.addr.to_string()], vec![]).await;
|
||||||
|
let waiter_pool = pool.clone();
|
||||||
|
let waiter = tokio::spawn(async move {
|
||||||
|
waiter_pool
|
||||||
|
.maybe_reflect_public_addr(IpFamily::V4, None)
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
v4.received().await;
|
||||||
|
assert_eq!(
|
||||||
|
timeout(
|
||||||
|
TEST_DEADLINE,
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V6, None)
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.ip(),
|
||||||
|
public_v6
|
||||||
|
);
|
||||||
|
assert!(!waiter.is_finished());
|
||||||
|
v4.release.add_permits(16);
|
||||||
|
assert_eq!(waiter.await.unwrap().unwrap().ip(), PUBLIC_IP);
|
||||||
|
assert_eq!(
|
||||||
|
pool.nat_runtime
|
||||||
|
.nat_reflection_cache
|
||||||
|
.lock()
|
||||||
|
.await
|
||||||
|
.v4
|
||||||
|
.unwrap()
|
||||||
|
.1
|
||||||
|
.ip(),
|
||||||
|
PUBLIC_IP
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
pool.nat_runtime
|
||||||
|
.nat_reflection_cache
|
||||||
|
.lock()
|
||||||
|
.await
|
||||||
|
.v6
|
||||||
|
.unwrap()
|
||||||
|
.1
|
||||||
|
.ip(),
|
||||||
|
public_v6
|
||||||
|
);
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn failed_shared_refresh_publishes_backoff_after_caller_cancellation() {
|
||||||
|
let stun = stun_server_response("127.0.0.1:0".parse().unwrap(), None, Duration::ZERO).await;
|
||||||
|
let pool = nat_pool(vec![stun.addr.to_string()], vec![]).await;
|
||||||
|
let waiter_pool = pool.clone();
|
||||||
|
let waiter = tokio::spawn(async move {
|
||||||
|
waiter_pool
|
||||||
|
.maybe_reflect_public_addr(IpFamily::V4, None)
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
stun.received().await;
|
||||||
|
waiter.abort();
|
||||||
|
let _ = waiter.await;
|
||||||
|
stun.release.add_permits(1);
|
||||||
|
assert_eq!(
|
||||||
|
timeout(
|
||||||
|
TEST_DEADLINE,
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V4, None)
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
assert!(pool.nat_runtime.stun_backoff_until.read().await.unwrap() > Instant::now());
|
||||||
|
assert_eq!(
|
||||||
|
pool.maybe_reflect_public_addr(IpFamily::V4, None).await,
|
||||||
|
None
|
||||||
|
);
|
||||||
|
assert_eq!(stun.requests.load(Ordering::Relaxed), 1);
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn shutdown_before_producer_first_poll_releases_owned_gates() {
|
||||||
|
let http = http_server("91.108.56.1").await;
|
||||||
|
let stun = stun_server("127.0.0.1:0".parse().unwrap(), PUBLIC_IP.into()).await;
|
||||||
|
let pool = nat_pool(
|
||||||
|
vec![stun.addr.to_string()],
|
||||||
|
vec![format!("http://{}/", http.addr)],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let mut http_waiter =
|
||||||
|
tokio_test::task::spawn(pool.maybe_detect_nat_ip(Ipv4Addr::LOCALHOST.into()));
|
||||||
|
let mut stun_waiter =
|
||||||
|
tokio_test::task::spawn(pool.maybe_reflect_public_addr(IpFamily::V4, None));
|
||||||
|
assert!(http_waiter.poll().is_pending());
|
||||||
|
assert!(stun_waiter.poll().is_pending());
|
||||||
|
// On this current-thread runtime neither producer can run before shutdown admission closes.
|
||||||
|
pool.begin_shutdown();
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
assert_eq!(http_waiter.poll(), std::task::Poll::Ready(None));
|
||||||
|
assert_eq!(stun_waiter.poll(), std::task::Poll::Ready(None));
|
||||||
|
assert_eq!(http.requests.load(Ordering::Relaxed), 0);
|
||||||
|
assert_eq!(stun.requests.load(Ordering::Relaxed), 0);
|
||||||
|
assert!(pool.nat_runtime.discovery.http.try_lock().is_ok());
|
||||||
|
assert!(
|
||||||
|
pool.nat_runtime
|
||||||
|
.nat_reflection_singleflight_v4
|
||||||
|
.try_lock()
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,266 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::crypto::{SecureRandom, derive_middleproxy_keys};
|
||||||
|
use crate::transport::middle_proxy::codec::{
|
||||||
|
RpcChecksumMode, build_handshake_payload, build_nonce_payload, build_rpc_frame,
|
||||||
|
cbc_decrypt_inplace, cbc_encrypt_padded, parse_handshake_flags, parse_nonce_payload,
|
||||||
|
read_rpc_frame_plaintext, rpc_crc,
|
||||||
|
};
|
||||||
|
use crate::transport::middle_proxy::me_health_monitor;
|
||||||
|
use crate::transport::middle_proxy::pool::MeFamilyRuntimeState;
|
||||||
|
use tokio::net::TcpStream;
|
||||||
|
|
||||||
|
struct MiddlePeer {
|
||||||
|
addr: SocketAddr,
|
||||||
|
handshakes: Arc<AtomicUsize>,
|
||||||
|
cancelled: Arc<AtomicUsize>,
|
||||||
|
task: JoinHandle<()>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for MiddlePeer {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.task.abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MiddlePeer {
|
||||||
|
async fn new(server_ip: Ipv4Addr) -> Self {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let addr = listener.local_addr().unwrap();
|
||||||
|
let handshakes = Arc::new(AtomicUsize::new(0));
|
||||||
|
let cancelled = Arc::new(AtomicUsize::new(0));
|
||||||
|
let succeeded = handshakes.clone();
|
||||||
|
let closed = cancelled.clone();
|
||||||
|
let task = tokio::spawn(async move {
|
||||||
|
let mut pending = JoinSet::new();
|
||||||
|
let mut established = Vec::new();
|
||||||
|
loop {
|
||||||
|
tokio::select! {
|
||||||
|
accepted = listener.accept() => {
|
||||||
|
let (stream, _) = accepted.unwrap();
|
||||||
|
pending.spawn(accept_handshake(stream, server_ip));
|
||||||
|
}
|
||||||
|
completed = pending.join_next(), if !pending.is_empty() => {
|
||||||
|
match completed.unwrap().expect("fake ME handshake task must not panic") {
|
||||||
|
Some(stream) => {
|
||||||
|
established.push(stream);
|
||||||
|
succeeded.fetch_add(1, Ordering::Release);
|
||||||
|
}
|
||||||
|
None => { closed.fetch_add(1, Ordering::Release); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
Self {
|
||||||
|
addr,
|
||||||
|
handshakes,
|
||||||
|
cancelled,
|
||||||
|
task,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn accept_handshake(mut stream: TcpStream, server_ip: Ipv4Addr) -> Option<TcpStream> {
|
||||||
|
let client_port = stream.peer_addr().unwrap().port();
|
||||||
|
let server_port = stream.local_addr().unwrap().port();
|
||||||
|
let (seq, payload) = match read_rpc_frame_plaintext(&mut stream).await {
|
||||||
|
Ok(frame) => frame,
|
||||||
|
// Reconnect may time out during NAT discovery, before sending any nonce.
|
||||||
|
Err(ProxyError::Io(error)) if error.kind() == std::io::ErrorKind::UnexpectedEof => {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Err(error) => panic!("invalid client nonce: {error}"),
|
||||||
|
};
|
||||||
|
assert_eq!(seq, -2);
|
||||||
|
let (selector, _, timestamp, client_nonce) = parse_nonce_payload(&payload).unwrap();
|
||||||
|
let server_nonce = [0x42u8; 16];
|
||||||
|
let nonce = build_nonce_payload(selector, timestamp, &server_nonce);
|
||||||
|
stream
|
||||||
|
.write_all(&build_rpc_frame(-2, &nonce, RpcChecksumMode::Crc32))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut client_ip = PUBLIC_IP.octets();
|
||||||
|
client_ip.reverse();
|
||||||
|
let mut server_ip = server_ip.octets();
|
||||||
|
server_ip.reverse();
|
||||||
|
let (read_key, read_iv) = derive_middleproxy_keys(
|
||||||
|
&server_nonce,
|
||||||
|
&client_nonce,
|
||||||
|
×tamp.to_le_bytes(),
|
||||||
|
Some(&server_ip),
|
||||||
|
&client_port.to_le_bytes(),
|
||||||
|
b"CLIENT",
|
||||||
|
Some(&client_ip),
|
||||||
|
&server_port.to_le_bytes(),
|
||||||
|
&[1u8; 32],
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
let (write_key, write_iv) = derive_middleproxy_keys(
|
||||||
|
&server_nonce,
|
||||||
|
&client_nonce,
|
||||||
|
×tamp.to_le_bytes(),
|
||||||
|
Some(&server_ip),
|
||||||
|
&client_port.to_le_bytes(),
|
||||||
|
b"SERVER",
|
||||||
|
Some(&client_ip),
|
||||||
|
&server_port.to_le_bytes(),
|
||||||
|
&[1u8; 32],
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
let mut encrypted = [0u8; 48];
|
||||||
|
stream.read_exact(&mut encrypted).await.unwrap();
|
||||||
|
cbc_decrypt_inplace(&read_key, &read_iv, &mut encrypted).unwrap();
|
||||||
|
assert_eq!(u32::from_le_bytes(encrypted[..4].try_into().unwrap()), 44);
|
||||||
|
assert_eq!(i32::from_le_bytes(encrypted[4..8].try_into().unwrap()), -1);
|
||||||
|
assert_eq!(
|
||||||
|
u32::from_le_bytes(encrypted[40..44].try_into().unwrap()),
|
||||||
|
rpc_crc(RpcChecksumMode::Crc32, &encrypted[..40])
|
||||||
|
);
|
||||||
|
parse_handshake_flags(&encrypted[8..40]).unwrap();
|
||||||
|
assert_eq!(&encrypted[16..20], &client_ip);
|
||||||
|
assert_eq!(
|
||||||
|
u16::from_le_bytes(encrypted[20..22].try_into().unwrap()),
|
||||||
|
client_port
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
u16::from_le_bytes(encrypted[32..34].try_into().unwrap()),
|
||||||
|
server_port
|
||||||
|
);
|
||||||
|
let reply = build_handshake_payload(server_ip, server_port, client_ip, client_port, 0);
|
||||||
|
let (encrypted, _) = cbc_encrypt_padded(
|
||||||
|
&write_key,
|
||||||
|
&write_iv,
|
||||||
|
&build_rpc_frame(-1, &reply, RpcChecksumMode::Crc32),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
stream.write_all(&encrypted).await.unwrap();
|
||||||
|
Some(stream)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn suppressed_pool_recovers_after_delayed_discovery(http_slow: bool) {
|
||||||
|
// Every lookup exceeds me_one_timeout_ms, not just the first request of the outage.
|
||||||
|
let delay = Duration::from_millis(1500);
|
||||||
|
let stun = stun_server_response(
|
||||||
|
"127.0.0.1:0".parse().unwrap(),
|
||||||
|
Some(PUBLIC_IP.into()),
|
||||||
|
delay,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let http = http_server_with_delay("91.108.56.1", delay).await;
|
||||||
|
stun.release.add_permits(64);
|
||||||
|
http.release.add_permits(64);
|
||||||
|
let peer = MiddlePeer::new(if http_slow {
|
||||||
|
PUBLIC_IP
|
||||||
|
} else {
|
||||||
|
Ipv4Addr::LOCALHOST
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
let urls = if http_slow {
|
||||||
|
vec![format!("http://{}/", http.addr)]
|
||||||
|
} else {
|
||||||
|
vec![]
|
||||||
|
};
|
||||||
|
let pool = nat_pool(vec![stun.addr.to_string()], urls).await;
|
||||||
|
pool.health_runtime
|
||||||
|
.me_health_interval_ms_unhealthy
|
||||||
|
.store(10, Ordering::Relaxed);
|
||||||
|
pool.health_runtime
|
||||||
|
.me_health_interval_ms_healthy
|
||||||
|
.store(10, Ordering::Relaxed);
|
||||||
|
pool.single_endpoint_runtime
|
||||||
|
.me_single_endpoint_outage_backoff_min_ms
|
||||||
|
.store(100, Ordering::Relaxed);
|
||||||
|
pool.single_endpoint_runtime
|
||||||
|
.me_single_endpoint_outage_backoff_max_ms
|
||||||
|
.store(100, Ordering::Relaxed);
|
||||||
|
pool.single_endpoint_runtime
|
||||||
|
.me_single_endpoint_shadow_writers
|
||||||
|
.store(0, Ordering::Relaxed);
|
||||||
|
pool.update_proxy_maps(
|
||||||
|
HashMap::from([(2, vec![(peer.addr.ip(), peer.addr.port())])]),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let generation = pool.current_generation();
|
||||||
|
let now = MePool::now_epoch_secs();
|
||||||
|
pool.set_family_runtime_state(
|
||||||
|
IpFamily::V4,
|
||||||
|
MeFamilyRuntimeState::Suppressed,
|
||||||
|
now,
|
||||||
|
now + 3600,
|
||||||
|
5,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
pool.nat_runtime.nat_reflection_cache.lock().await.v4 = Some((
|
||||||
|
if http_slow {
|
||||||
|
Instant::now()
|
||||||
|
} else {
|
||||||
|
Instant::now() - Duration::from_secs(601)
|
||||||
|
},
|
||||||
|
SocketAddr::new(PUBLIC_IP.into(), 45678),
|
||||||
|
));
|
||||||
|
let monitor = tokio::spawn(me_health_monitor(
|
||||||
|
pool.clone(),
|
||||||
|
Arc::new(SecureRandom::new()),
|
||||||
|
0,
|
||||||
|
));
|
||||||
|
if http_slow {
|
||||||
|
http.received().await;
|
||||||
|
} else {
|
||||||
|
stun.received().await;
|
||||||
|
}
|
||||||
|
timeout(Duration::from_secs(5), async {
|
||||||
|
while peer.cancelled.load(Ordering::Acquire) == 0 {
|
||||||
|
assert!(!peer.task.is_finished());
|
||||||
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("the actual 1200ms reconnect must cancel at least one pre-KDF TCP connection");
|
||||||
|
assert_eq!(peer.handshakes.load(Ordering::Acquire), 0);
|
||||||
|
let expected = 2 * pool.required_writers_for_dc(1);
|
||||||
|
timeout(Duration::from_secs(10), async {
|
||||||
|
while !pool.admission_ready_full_floor().await
|
||||||
|
|| peer.handshakes.load(Ordering::Acquire) < expected
|
||||||
|
{
|
||||||
|
assert!(
|
||||||
|
!peer.task.is_finished(),
|
||||||
|
"fake ME must validate every encrypted handshake"
|
||||||
|
);
|
||||||
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("suppressed pool must autonomously regain its full floor without restart");
|
||||||
|
monitor.abort();
|
||||||
|
let _ = monitor.await;
|
||||||
|
assert_eq!(pool.current_generation(), generation);
|
||||||
|
assert_eq!(pool.reinit.status.load().pending_hardswap_generation, 0);
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
assert_eq!(pool.writers.read().await.len(), expected);
|
||||||
|
assert!(peer.handshakes.load(Ordering::Acquire) >= expected);
|
||||||
|
assert_eq!(
|
||||||
|
http.requests.load(Ordering::Relaxed),
|
||||||
|
usize::from(http_slow)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
stun.requests.load(Ordering::Relaxed),
|
||||||
|
usize::from(!http_slow)
|
||||||
|
);
|
||||||
|
assert!(pool.shutdown_until(TEST_DEADLINE).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn suppressed_pool_recovers_full_floor_after_expired_stun_refresh() {
|
||||||
|
suppressed_pool_recovers_after_delayed_discovery(false).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn suppressed_pool_recovers_full_floor_after_slow_http_with_fresh_stun() {
|
||||||
|
suppressed_pool_recovers_after_delayed_discovery(true).await;
|
||||||
|
}
|
||||||
@@ -33,6 +33,10 @@ mod publication;
|
|||||||
mod replacement;
|
mod replacement;
|
||||||
mod runtime;
|
mod runtime;
|
||||||
|
|
||||||
|
// Recovery regressions exercise the serialized production publication boundary.
|
||||||
|
#[cfg(test)]
|
||||||
|
mod recovery_tests;
|
||||||
|
|
||||||
struct PreparedWriter<'a> {
|
struct PreparedWriter<'a> {
|
||||||
writer: MeWriter,
|
writer: MeWriter,
|
||||||
tx: mpsc::Sender<WriterCommand>,
|
tx: mpsc::Sender<WriterCommand>,
|
||||||
|
|||||||
@@ -84,7 +84,17 @@ impl MePool {
|
|||||||
crate::network::IpFamily::V6
|
crate::network::IpFamily::V6
|
||||||
};
|
};
|
||||||
let now_epoch_secs = Self::now_epoch_secs();
|
let now_epoch_secs = Self::now_epoch_secs();
|
||||||
if !self.family_enabled_for_drain_coverage(family, now_epoch_secs) {
|
let configured = match family {
|
||||||
|
crate::network::IpFamily::V4 => self.decision.ipv4_me,
|
||||||
|
crate::network::IpFamily::V6 => self.decision.ipv6_me,
|
||||||
|
};
|
||||||
|
// Recovery coverage must remain publishable during suppression; the serialized
|
||||||
|
// per-group floor check below still bounds it and generation authority is unchanged.
|
||||||
|
let recovery_coverage =
|
||||||
|
contour == WriterContour::Active && intent == WriterOpenIntent::Coverage;
|
||||||
|
if !configured
|
||||||
|
|| (self.is_family_temporarily_suppressed(family, now_epoch_secs) && !recovery_coverage)
|
||||||
|
{
|
||||||
return Err(ProxyError::Proxy(
|
return Err(ProxyError::Proxy(
|
||||||
"ME writer family lost publication authority".into(),
|
"ME writer family lost publication authority".into(),
|
||||||
));
|
));
|
||||||
|
|||||||
@@ -0,0 +1,466 @@
|
|||||||
|
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::network::{IpFamily, probe::NetworkDecision};
|
||||||
|
use crate::transport::middle_proxy::admission_test_support::unregistered_writer;
|
||||||
|
use crate::transport::middle_proxy::pool::MeFamilyRuntimeState;
|
||||||
|
use crate::transport::middle_proxy::pool_writer_security_tests::{
|
||||||
|
make_pool, make_pool_with_decision,
|
||||||
|
};
|
||||||
|
|
||||||
|
fn cap_at_one(pool: &MePool) {
|
||||||
|
pool.floor_runtime
|
||||||
|
.me_adaptive_floor_cpu_cores_override
|
||||||
|
.store(1, Ordering::Relaxed);
|
||||||
|
pool.floor_runtime
|
||||||
|
.me_adaptive_floor_max_active_writers_per_core
|
||||||
|
.store(1, Ordering::Relaxed);
|
||||||
|
pool.floor_runtime
|
||||||
|
.me_adaptive_floor_max_active_writers_global
|
||||||
|
.store(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn suppress(pool: &MePool, family: IpFamily) {
|
||||||
|
let now = MePool::now_epoch_secs();
|
||||||
|
pool.set_family_runtime_state(
|
||||||
|
family,
|
||||||
|
MeFamilyRuntimeState::Suppressed,
|
||||||
|
now,
|
||||||
|
now + 3600,
|
||||||
|
5,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn prepared<'a>(
|
||||||
|
pool: &'a Arc<MePool>,
|
||||||
|
writer: MeWriter,
|
||||||
|
intent: WriterOpenIntent,
|
||||||
|
reservation: WriterOpenReservation<'a>,
|
||||||
|
task_started: Arc<AtomicBool>,
|
||||||
|
) -> PreparedWriter<'a> {
|
||||||
|
let cancel = writer.cancel.clone();
|
||||||
|
PreparedWriter {
|
||||||
|
tx: writer.tx.clone(),
|
||||||
|
byte_budget: writer.byte_budget.clone(),
|
||||||
|
writer,
|
||||||
|
task_registration: pool.lifecycle.try_register().unwrap(),
|
||||||
|
writer_task: Box::pin(async move {
|
||||||
|
task_started.store(true, Ordering::Release);
|
||||||
|
cancel.cancelled().await;
|
||||||
|
}),
|
||||||
|
intent,
|
||||||
|
_open_reservation: reservation,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn suppressed_active_coverage_publishes_only_the_missing_floor() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||||
|
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||||
|
.await;
|
||||||
|
cap_at_one(&pool);
|
||||||
|
suppress(&pool, IpFamily::V4);
|
||||||
|
let required = pool.required_writers_for_dc(1);
|
||||||
|
let mut receivers = Vec::new();
|
||||||
|
for id in 1..=required as u64 {
|
||||||
|
let reservation = pool
|
||||||
|
.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, addr)
|
||||||
|
.await
|
||||||
|
.expect("missing coverage must have a reservation despite suppression");
|
||||||
|
let (writer, receiver) = unregistered_writer(
|
||||||
|
&pool,
|
||||||
|
id,
|
||||||
|
2,
|
||||||
|
addr,
|
||||||
|
pool.current_generation(),
|
||||||
|
WriterContour::Active,
|
||||||
|
);
|
||||||
|
receivers.push(receiver);
|
||||||
|
pool.publish_connected_writer(prepared(
|
||||||
|
&pool,
|
||||||
|
writer,
|
||||||
|
WriterOpenIntent::Coverage,
|
||||||
|
reservation,
|
||||||
|
Arc::new(AtomicBool::new(false)),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("authenticated coverage must publish without waiting for suppression expiry");
|
||||||
|
assert_eq!(pool.conn_count.load(Ordering::Relaxed), id as usize);
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
pool.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, addr)
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
let (extra, _receiver) = unregistered_writer(
|
||||||
|
&pool,
|
||||||
|
99,
|
||||||
|
2,
|
||||||
|
addr,
|
||||||
|
pool.current_generation(),
|
||||||
|
WriterContour::Active,
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
pool.authorize_writer_publication_capacity(
|
||||||
|
&extra,
|
||||||
|
WriterContour::Active,
|
||||||
|
WriterOpenIntent::Coverage,
|
||||||
|
&pool.writers.read().await,
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
pool.family_runtime_state(IpFamily::V4),
|
||||||
|
MeFamilyRuntimeState::Suppressed
|
||||||
|
);
|
||||||
|
assert!(pool.shutdown_until(Duration::from_secs(1)).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn concurrent_suppressed_coverage_publications_cannot_overfill_a_group() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||||
|
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||||
|
.await;
|
||||||
|
cap_at_one(&pool);
|
||||||
|
suppress(&pool, IpFamily::V4);
|
||||||
|
let required = pool.required_writers_for_dc(1);
|
||||||
|
let mut receivers = Vec::new();
|
||||||
|
let mut publications = Vec::new();
|
||||||
|
for id in 1..=(required + 1) as u64 {
|
||||||
|
let reservation = pool
|
||||||
|
.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, addr)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let (writer, receiver) = unregistered_writer(
|
||||||
|
&pool,
|
||||||
|
id,
|
||||||
|
2,
|
||||||
|
addr,
|
||||||
|
pool.current_generation(),
|
||||||
|
WriterContour::Active,
|
||||||
|
);
|
||||||
|
receivers.push(receiver);
|
||||||
|
publications.push(pool.publish_connected_writer(prepared(
|
||||||
|
&pool,
|
||||||
|
writer,
|
||||||
|
WriterOpenIntent::Coverage,
|
||||||
|
reservation,
|
||||||
|
Arc::new(AtomicBool::new(false)),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let guard = pool.writers.write().await;
|
||||||
|
let results = futures::future::join_all(publications);
|
||||||
|
tokio::pin!(results);
|
||||||
|
tokio::select! {
|
||||||
|
biased;
|
||||||
|
results = &mut results => panic!("publication bypassed the writer barrier: {results:?}"),
|
||||||
|
_ = tokio::task::yield_now() => {}
|
||||||
|
}
|
||||||
|
drop(guard);
|
||||||
|
let results = results.await;
|
||||||
|
assert_eq!(
|
||||||
|
results.iter().filter(|result| result.is_ok()).count(),
|
||||||
|
required
|
||||||
|
);
|
||||||
|
assert_eq!(results.iter().filter(|result| result.is_err()).count(), 1);
|
||||||
|
assert_eq!(pool.writers.read().await.len(), required);
|
||||||
|
assert_eq!(pool.conn_count.load(Ordering::Relaxed), required);
|
||||||
|
assert_eq!(
|
||||||
|
pool.registry.writer_idle_since_snapshot().await.len(),
|
||||||
|
required
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
assert!(pool.shutdown_until(Duration::from_secs(1)).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn committed_warm_writer_uses_resolved_active_coverage_authority() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||||
|
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||||
|
.await;
|
||||||
|
suppress(&pool, IpFamily::V4);
|
||||||
|
let reservation = pool
|
||||||
|
.reserve_writer_open(WriterContour::Warm, WriterOpenIntent::Coverage, 2, addr)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let (writer, _receiver) = unregistered_writer(
|
||||||
|
&pool,
|
||||||
|
1,
|
||||||
|
2,
|
||||||
|
addr,
|
||||||
|
pool.current_generation(),
|
||||||
|
WriterContour::Warm,
|
||||||
|
);
|
||||||
|
pool.publish_connected_writer(prepared(
|
||||||
|
&pool,
|
||||||
|
writer,
|
||||||
|
WriterOpenIntent::Coverage,
|
||||||
|
reservation,
|
||||||
|
Arc::new(AtomicBool::new(false)),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.expect("a committed generation must resolve warm coverage to its active role");
|
||||||
|
assert_eq!(
|
||||||
|
pool.writers.read().await[0].contour.load(Ordering::Acquire),
|
||||||
|
WriterContour::Active.as_u8()
|
||||||
|
);
|
||||||
|
assert_eq!(pool.writer_connect_warm_reserved.load(Ordering::Acquire), 0);
|
||||||
|
assert!(pool.shutdown_until(Duration::from_secs(1)).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn suppressed_families_keep_their_full_coverage_reservation_budget() {
|
||||||
|
let pool = make_pool_with_decision(NetworkDecision {
|
||||||
|
ipv4_me: true,
|
||||||
|
ipv6_me: true,
|
||||||
|
..NetworkDecision::default()
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
let v4 = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||||
|
let v6 = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 443);
|
||||||
|
pool.update_proxy_maps(
|
||||||
|
HashMap::from([(2, vec![(v4.ip(), v4.port())])]),
|
||||||
|
Some(HashMap::from([(2, vec![(v6.ip(), v6.port())])])),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
cap_at_one(&pool);
|
||||||
|
suppress(&pool, IpFamily::V4);
|
||||||
|
suppress(&pool, IpFamily::V6);
|
||||||
|
let required = pool.required_writers_for_dc(1);
|
||||||
|
assert_eq!(pool.active_coverage_required_total().await, required * 4);
|
||||||
|
|
||||||
|
let mut receivers = Vec::new();
|
||||||
|
let mut writers = pool.writers.write().await;
|
||||||
|
for (index, (dc, addr)) in [(2, v4), (-2, v4), (-2, v6)].into_iter().enumerate() {
|
||||||
|
for offset in 0..required {
|
||||||
|
let (writer, receiver) = unregistered_writer(
|
||||||
|
&pool,
|
||||||
|
(index * required + offset + 1) as u64,
|
||||||
|
dc,
|
||||||
|
addr,
|
||||||
|
pool.current_generation(),
|
||||||
|
WriterContour::Active,
|
||||||
|
);
|
||||||
|
writers.push(writer);
|
||||||
|
receivers.push(receiver);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
drop(writers);
|
||||||
|
let reservation = pool
|
||||||
|
.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, v6)
|
||||||
|
.await
|
||||||
|
.expect("other covered groups must not consume the suppressed family's floor budget");
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
drop(reservation);
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn coverage_reservations_remain_bounded_and_release_on_drop() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||||
|
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||||
|
.await;
|
||||||
|
cap_at_one(&pool);
|
||||||
|
suppress(&pool, IpFamily::V4);
|
||||||
|
let limit = pool.required_writers_for_dc(1) * 2
|
||||||
|
+ pool
|
||||||
|
.reconnect_runtime
|
||||||
|
.me_reconnect_max_concurrent_per_dc
|
||||||
|
.max(1) as usize;
|
||||||
|
let mut reservations = Vec::new();
|
||||||
|
for _ in 0..limit {
|
||||||
|
reservations.push(
|
||||||
|
pool.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, addr)
|
||||||
|
.await
|
||||||
|
.expect("the configured coverage budget must remain available"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
pool.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, addr,)
|
||||||
|
.await
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||||
|
limit
|
||||||
|
);
|
||||||
|
drop(reservations);
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn suppression_exception_does_not_admit_growth_replacement_or_pending_warm() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||||
|
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||||
|
.await;
|
||||||
|
suppress(&pool, IpFamily::V4);
|
||||||
|
for (contour, intent) in [
|
||||||
|
(WriterContour::Active, WriterOpenIntent::Normal),
|
||||||
|
(WriterContour::Active, WriterOpenIntent::Replacement),
|
||||||
|
(WriterContour::Warm, WriterOpenIntent::Coverage),
|
||||||
|
(WriterContour::Warm, WriterOpenIntent::Normal),
|
||||||
|
] {
|
||||||
|
let (writer, _receiver) =
|
||||||
|
unregistered_writer(&pool, 1, 2, addr, pool.current_generation(), contour);
|
||||||
|
assert!(
|
||||||
|
pool.authorize_writer_publication_capacity(&writer, contour, intent, &[])
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn coverage_exception_never_enables_a_disabled_family() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let v4 = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||||
|
let v6 = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 443);
|
||||||
|
pool.update_proxy_maps(
|
||||||
|
HashMap::from([(2, vec![(v4.ip(), v4.port())])]),
|
||||||
|
Some(HashMap::from([(2, vec![(v6.ip(), v6.port())])])),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
suppress(&pool, IpFamily::V6);
|
||||||
|
assert_eq!(
|
||||||
|
pool.active_coverage_required_total().await,
|
||||||
|
pool.required_writers_for_dc(1) * 2
|
||||||
|
);
|
||||||
|
let (writer, _receiver) = unregistered_writer(
|
||||||
|
&pool,
|
||||||
|
1,
|
||||||
|
2,
|
||||||
|
v6,
|
||||||
|
pool.current_generation(),
|
||||||
|
WriterContour::Active,
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
pool.authorize_writer_publication_capacity(
|
||||||
|
&writer,
|
||||||
|
WriterContour::Active,
|
||||||
|
WriterOpenIntent::Coverage,
|
||||||
|
&[],
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn suppressed_coverage_cannot_publish_a_stale_generation() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||||
|
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||||
|
.await;
|
||||||
|
suppress(&pool, IpFamily::V4);
|
||||||
|
let reservation = pool
|
||||||
|
.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, addr)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let (writer, _receiver) = unregistered_writer(
|
||||||
|
&pool,
|
||||||
|
1,
|
||||||
|
2,
|
||||||
|
addr,
|
||||||
|
pool.current_generation().saturating_sub(1),
|
||||||
|
WriterContour::Active,
|
||||||
|
);
|
||||||
|
let task_started = Arc::new(AtomicBool::new(false));
|
||||||
|
let error = pool
|
||||||
|
.publish_connected_writer(prepared(
|
||||||
|
&pool,
|
||||||
|
writer,
|
||||||
|
WriterOpenIntent::Coverage,
|
||||||
|
reservation,
|
||||||
|
task_started.clone(),
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
error
|
||||||
|
.to_string()
|
||||||
|
.contains("generation lost publication authority")
|
||||||
|
);
|
||||||
|
assert!(pool.writers.read().await.is_empty());
|
||||||
|
assert!(pool.registry.writer_idle_since_snapshot().await.is_empty());
|
||||||
|
assert_eq!(pool.conn_count.load(Ordering::Relaxed), 0);
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
assert!(!task_started.load(Ordering::Acquire));
|
||||||
|
assert!(pool.shutdown_until(Duration::from_secs(1)).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn suppressed_coverage_rechecks_endpoints_after_publication_waits() {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||||
|
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||||
|
.await;
|
||||||
|
suppress(&pool, IpFamily::V4);
|
||||||
|
let reservation = pool
|
||||||
|
.reserve_writer_open(WriterContour::Active, WriterOpenIntent::Coverage, 2, addr)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let (writer, _receiver) = unregistered_writer(
|
||||||
|
&pool,
|
||||||
|
1,
|
||||||
|
2,
|
||||||
|
addr,
|
||||||
|
pool.current_generation(),
|
||||||
|
WriterContour::Active,
|
||||||
|
);
|
||||||
|
let task_started = Arc::new(AtomicBool::new(false));
|
||||||
|
let publication = pool.publish_connected_writer(prepared(
|
||||||
|
&pool,
|
||||||
|
writer,
|
||||||
|
WriterOpenIntent::Coverage,
|
||||||
|
reservation,
|
||||||
|
task_started.clone(),
|
||||||
|
));
|
||||||
|
tokio::pin!(publication);
|
||||||
|
let guard = pool.writers.write().await;
|
||||||
|
tokio::select! {
|
||||||
|
biased;
|
||||||
|
result = &mut publication => panic!("publication bypassed the writer barrier: {result:?}"),
|
||||||
|
_ = tokio::task::yield_now() => {}
|
||||||
|
}
|
||||||
|
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), 444)])]), None)
|
||||||
|
.await;
|
||||||
|
drop(guard);
|
||||||
|
let error = publication.await.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
error
|
||||||
|
.to_string()
|
||||||
|
.contains("target changed before publication")
|
||||||
|
);
|
||||||
|
assert!(pool.writers.read().await.is_empty());
|
||||||
|
assert!(pool.registry.writer_idle_since_snapshot().await.is_empty());
|
||||||
|
assert_eq!(pool.conn_count.load(Ordering::Relaxed), 0);
|
||||||
|
assert_eq!(
|
||||||
|
pool.writer_connect_active_reserved.load(Ordering::Acquire),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
assert!(!task_started.load(Ordering::Acquire));
|
||||||
|
assert!(pool.shutdown_until(Duration::from_secs(1)).await);
|
||||||
|
}
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering};
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use tokio::sync::mpsc;
|
||||||
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
|
use super::codec::WriterCommand;
|
||||||
|
use super::pool::{MePool, MeWriter, WriterContour};
|
||||||
|
use super::pool_writer_security_tests::make_pool;
|
||||||
|
|
||||||
|
/// Owns idle writer visibility without spawning ME transports or client sessions.
|
||||||
|
pub(crate) struct IdlePoolFixture {
|
||||||
|
pool: Arc<MePool>,
|
||||||
|
draining: Vec<Arc<AtomicBool>>,
|
||||||
|
_receivers: Vec<mpsc::Receiver<WriterCommand>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl IdlePoolFixture {
|
||||||
|
/// Builds both signed DC groups with a controllable readiness quorum.
|
||||||
|
pub(crate) async fn new(ready: bool) -> Self {
|
||||||
|
let pool = make_pool().await;
|
||||||
|
let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
|
||||||
|
pool.update_proxy_maps(HashMap::from([(2, vec![(addr.ip(), addr.port())])]), None)
|
||||||
|
.await;
|
||||||
|
let mut draining = Vec::new();
|
||||||
|
let mut receivers = Vec::new();
|
||||||
|
{
|
||||||
|
let mut writers = pool.writers.write().await;
|
||||||
|
for (id, dc) in [(1, 2), (2, -2)] {
|
||||||
|
let (writer, receiver) = unregistered_writer(
|
||||||
|
&pool,
|
||||||
|
id,
|
||||||
|
dc,
|
||||||
|
addr,
|
||||||
|
pool.current_generation(),
|
||||||
|
WriterContour::Active,
|
||||||
|
);
|
||||||
|
writer.draining.store(!ready, Ordering::Release);
|
||||||
|
draining.push(writer.draining.clone());
|
||||||
|
receivers.push(receiver);
|
||||||
|
writers.push(writer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Self {
|
||||||
|
pool,
|
||||||
|
draining,
|
||||||
|
_receivers: receivers,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the pool sampled by the production admission gate.
|
||||||
|
pub(crate) fn pool(&self) -> Arc<MePool> {
|
||||||
|
self.pool.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Changes the fixture quorum without notifying the admission gate.
|
||||||
|
pub(crate) fn set_ready(&self, ready: bool) {
|
||||||
|
for draining in &self.draining {
|
||||||
|
draining.store(!ready, Ordering::Release);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Builds a writer whose command receiver remains owned by the test.
|
||||||
|
pub(super) fn unregistered_writer(
|
||||||
|
pool: &Arc<MePool>,
|
||||||
|
id: u64,
|
||||||
|
dc: i32,
|
||||||
|
addr: SocketAddr,
|
||||||
|
generation: u64,
|
||||||
|
contour: WriterContour,
|
||||||
|
) -> (MeWriter, mpsc::Receiver<WriterCommand>) {
|
||||||
|
let (tx, rx) = mpsc::channel(8);
|
||||||
|
(
|
||||||
|
MeWriter {
|
||||||
|
id,
|
||||||
|
addr,
|
||||||
|
source_ip: addr.ip(),
|
||||||
|
writer_dc: dc,
|
||||||
|
generation,
|
||||||
|
contour: Arc::new(AtomicU8::new(contour.as_u8())),
|
||||||
|
created_at: Instant::now(),
|
||||||
|
tx,
|
||||||
|
byte_budget: pool.new_writer_byte_budget(),
|
||||||
|
cancel: CancellationToken::new(),
|
||||||
|
degraded: Arc::new(AtomicBool::new(false)),
|
||||||
|
rtt_ema_ms_x10: Arc::new(AtomicU32::new(0)),
|
||||||
|
draining: Arc::new(AtomicBool::new(false)),
|
||||||
|
draining_started_at_epoch_secs: Arc::new(AtomicU64::new(0)),
|
||||||
|
drain_deadline_epoch_secs: Arc::new(AtomicU64::new(0)),
|
||||||
|
allow_drain_fallback: Arc::new(AtomicBool::new(false)),
|
||||||
|
},
|
||||||
|
rx,
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -339,6 +339,8 @@ pub struct UpstreamManager {
|
|||||||
|
|
||||||
// Upstream manager configuration, DNS resolution, and API snapshots.
|
// Upstream manager configuration, DNS resolution, and API snapshots.
|
||||||
mod manager_config;
|
mod manager_config;
|
||||||
|
// Current direct egress addresses eligible for source-bound NAT discovery.
|
||||||
|
mod nat_contexts;
|
||||||
// Latency- and scope-aware upstream selection.
|
// Latency- and scope-aware upstream selection.
|
||||||
mod selection;
|
mod selection;
|
||||||
// Direct and proxied connection establishment.
|
// Direct and proxied connection establishment.
|
||||||
|
|||||||
@@ -0,0 +1,122 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
impl UpstreamManager {
|
||||||
|
/// Resolves this generation's direct bind candidates without advancing round-robin state.
|
||||||
|
pub(crate) async fn direct_bind_ips_for_family(&self, ipv6: bool) -> BTreeSet<IpAddr> {
|
||||||
|
let guard = self.upstreams.read().await;
|
||||||
|
let target = SocketAddr::new(
|
||||||
|
if ipv6 {
|
||||||
|
std::net::Ipv6Addr::UNSPECIFIED.into()
|
||||||
|
} else {
|
||||||
|
std::net::Ipv4Addr::UNSPECIFIED.into()
|
||||||
|
},
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
let mut allowed = BTreeSet::new();
|
||||||
|
for state in guard.iter() {
|
||||||
|
let config = &state.config;
|
||||||
|
let UpstreamType::Direct {
|
||||||
|
interface,
|
||||||
|
bind_addresses,
|
||||||
|
..
|
||||||
|
} = &config.upstream_type
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if (if ipv6 { config.ipv6 } else { config.ipv4 }) == Some(false) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Some(addresses) = bind_addresses.as_ref().filter(|v| !v.is_empty()) {
|
||||||
|
for address in addresses {
|
||||||
|
if !address
|
||||||
|
.parse::<IpAddr>()
|
||||||
|
.is_ok_and(|ip| ip.is_ipv6() == ipv6)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Some(ip) = Self::resolve_bind_address(
|
||||||
|
interface,
|
||||||
|
&Some(vec![address.clone()]),
|
||||||
|
target,
|
||||||
|
None,
|
||||||
|
true,
|
||||||
|
) {
|
||||||
|
allowed.insert(ip);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if let Some(ip) =
|
||||||
|
Self::resolve_bind_address(interface, bind_addresses, target, None, true)
|
||||||
|
{
|
||||||
|
allowed.insert(ip);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
allowed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn config(source: &str) -> UpstreamConfig {
|
||||||
|
toml::from_str(source).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn candidates_follow_generation_policy_without_advancing_round_robin() {
|
||||||
|
let manager = UpstreamManager::new(
|
||||||
|
vec![
|
||||||
|
config("type = 'direct'\nbind_addresses = ['127.0.0.1', '127.0.0.2', '::1']"),
|
||||||
|
config("type = 'direct'\ninterface = '127.0.0.3'"),
|
||||||
|
config("type = 'direct'\ninterface = '127.0.0.4'\nenabled = false"),
|
||||||
|
config("type = 'direct'\ninterface = '127.0.0.5'\nipv4 = false"),
|
||||||
|
config("type = 'socks5'\naddress = '127.0.0.1:1080'\ninterface = '127.0.0.6'"),
|
||||||
|
],
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
1000,
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
true,
|
||||||
|
Arc::new(Stats::new()),
|
||||||
|
);
|
||||||
|
manager.upstreams.write().await[0].healthy = false;
|
||||||
|
let expected = ["127.0.0.1", "127.0.0.2", "127.0.0.3"]
|
||||||
|
.into_iter()
|
||||||
|
.map(|value| value.parse().unwrap())
|
||||||
|
.collect();
|
||||||
|
assert_eq!(manager.direct_bind_ips_for_family(false).await, expected);
|
||||||
|
assert_eq!(
|
||||||
|
manager.direct_bind_ips_for_family(true).await,
|
||||||
|
BTreeSet::from(["::1".parse().unwrap()])
|
||||||
|
);
|
||||||
|
for state in manager.upstreams.read().await.iter() {
|
||||||
|
assert_eq!(state.bind_rr.load(Ordering::Relaxed), 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn named_interface_candidates_match_direct_socket_binding() {
|
||||||
|
let manager = UpstreamManager::new(
|
||||||
|
vec![config(
|
||||||
|
"type = 'direct'\ninterface = 'lo'\nbind_addresses = ['127.0.0.1', '::1']",
|
||||||
|
)],
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
1000,
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
true,
|
||||||
|
Arc::new(Stats::new()),
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
manager.direct_bind_ips_for_family(false).await,
|
||||||
|
BTreeSet::from(["127.0.0.1".parse().unwrap()])
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
manager.direct_bind_ips_for_family(true).await,
|
||||||
|
BTreeSet::from(["::1".parse().unwrap()])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,14 +1,15 @@
|
|||||||
//! Descriptor-anchored filesystem operations for privileged runtime paths.
|
//! Descriptor-anchored filesystem operations for privileged runtime paths.
|
||||||
//!
|
//!
|
||||||
//! Submodules:
|
//! Submodules:
|
||||||
//! - `path`: symlink-free directory traversal and anchored path ownership
|
//! - `path`: strict or compatible directory traversal and anchored path ownership
|
||||||
//! - `write`: regular-file opening and durable atomic replacement
|
//! - `write`: regular-file opening and durable atomic replacement
|
||||||
|
|
||||||
mod path;
|
mod path;
|
||||||
mod write;
|
mod write;
|
||||||
|
|
||||||
pub(crate) use path::{
|
pub(crate) use path::{
|
||||||
AnchoredPath, chdir_nofollow_or_create, open_dir_nofollow, open_trusted_dir_nofollow_or_create,
|
AnchoredPath, chdir_nofollow_or_create, open_compatible_dir, open_dir_nofollow,
|
||||||
|
open_trusted_dir_nofollow_or_create,
|
||||||
};
|
};
|
||||||
pub(crate) use write::{
|
pub(crate) use write::{
|
||||||
atomic_replace, atomic_replace_async, open_append_regular, open_append_regular_at,
|
atomic_replace, atomic_replace_async, open_append_regular, open_append_regular_at,
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
use std::ffi::{OsStr, OsString};
|
use std::ffi::{OsStr, OsString};
|
||||||
use std::io;
|
use std::io;
|
||||||
use std::os::fd::OwnedFd;
|
use std::os::fd::OwnedFd;
|
||||||
use std::os::unix::fs::{MetadataExt, PermissionsExt};
|
use std::os::unix::fs::{DirBuilderExt, MetadataExt, PermissionsExt};
|
||||||
use std::path::{Component, Path};
|
use std::path::{Component, Path};
|
||||||
|
|
||||||
use nix::fcntl::{OFlag, open, openat};
|
use nix::fcntl::{OFlag, open, openat};
|
||||||
@@ -53,6 +53,28 @@ impl AnchoredPath {
|
|||||||
Ok(Self { parent, name })
|
Ok(Self { parent, name })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Anchors a runtime parent, applying trusted traversal only when explicitly requested.
|
||||||
|
pub(crate) fn open_runtime_parent(
|
||||||
|
path: &Path,
|
||||||
|
create_mode: Option<u32>,
|
||||||
|
strict_runtime_paths: bool,
|
||||||
|
) -> io::Result<Self> {
|
||||||
|
if strict_runtime_paths {
|
||||||
|
return match create_mode {
|
||||||
|
Some(mode) => Self::open_trusted_parent_or_create(path, mode),
|
||||||
|
None => Self::open_trusted_parent(path),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
let name = path
|
||||||
|
.file_name()
|
||||||
|
.filter(|name| !name.is_empty())
|
||||||
|
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "path has no file name"))?
|
||||||
|
.to_os_string();
|
||||||
|
let parent =
|
||||||
|
open_compatible_dir(path.parent().unwrap_or_else(|| Path::new(".")), create_mode)?;
|
||||||
|
Ok(Self { parent, name })
|
||||||
|
}
|
||||||
|
|
||||||
fn open_with_parent_creation(path: &Path, create_mode: Option<u32>) -> io::Result<Self> {
|
fn open_with_parent_creation(path: &Path, create_mode: Option<u32>) -> io::Result<Self> {
|
||||||
let name = path
|
let name = path
|
||||||
.file_name()
|
.file_name()
|
||||||
@@ -97,6 +119,23 @@ pub(crate) fn open_trusted_dir_nofollow_or_create(path: &Path, mode: u32) -> io:
|
|||||||
open_dir_components(path, Some(mode), true)
|
open_dir_components(path, Some(mode), true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Follows parent-directory symlinks without imposing ownership or permission policy.
|
||||||
|
pub(crate) fn open_compatible_dir(path: &Path, create_mode: Option<u32>) -> io::Result<OwnedFd> {
|
||||||
|
let path = if path.as_os_str().is_empty() {
|
||||||
|
Path::new(".")
|
||||||
|
} else {
|
||||||
|
path
|
||||||
|
};
|
||||||
|
if let Some(mode) = create_mode {
|
||||||
|
std::fs::DirBuilder::new()
|
||||||
|
.recursive(true)
|
||||||
|
.mode(mode)
|
||||||
|
.create(path)?;
|
||||||
|
}
|
||||||
|
// Retain the resolved directory inode so subsequent file operations do not rewalk the path.
|
||||||
|
open(path, DIRECTORY_FLAGS & !OFlag::O_NOFOLLOW, Mode::empty()).map_err(errno_to_io)
|
||||||
|
}
|
||||||
|
|
||||||
/// Opens a directory only when its entire path is owned by root or the effective user.
|
/// Opens a directory only when its entire path is owned by root or the effective user.
|
||||||
fn open_trusted_dir_nofollow(path: &Path) -> io::Result<OwnedFd> {
|
fn open_trusted_dir_nofollow(path: &Path) -> io::Result<OwnedFd> {
|
||||||
open_dir_components(path, None, true)
|
open_dir_components(path, None, true)
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
use super::path::AnchoredPath;
|
use super::path::AnchoredPath;
|
||||||
use super::write::atomic_replace_after_anchor;
|
use super::write::atomic_replace_after_anchor;
|
||||||
@@ -15,6 +16,50 @@ fn directory_walk_rejects_intermediate_symlink() {
|
|||||||
assert!(open_dir_nofollow(&link).is_err());
|
assert!(open_dir_nofollow(&link).is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_parent_policy_creates_and_reads_relative_paths() {
|
||||||
|
use std::os::unix::fs::MetadataExt;
|
||||||
|
|
||||||
|
let current = std::env::current_dir().unwrap();
|
||||||
|
let directory = tempfile::tempdir_in(¤t).unwrap();
|
||||||
|
let relative = directory.path().strip_prefix(¤t).unwrap();
|
||||||
|
for strict_runtime_paths in [false, true] {
|
||||||
|
let mode = if strict_runtime_paths {
|
||||||
|
"strict"
|
||||||
|
} else {
|
||||||
|
"compatible"
|
||||||
|
};
|
||||||
|
let path = relative.join(mode).join("nested/telemt.pid");
|
||||||
|
let created =
|
||||||
|
AnchoredPath::open_runtime_parent(&path, Some(0o750), strict_runtime_paths).unwrap();
|
||||||
|
let opened = AnchoredPath::open_runtime_parent(&path, None, strict_runtime_paths).unwrap();
|
||||||
|
let created = std::fs::File::from(created.parent().try_clone().unwrap())
|
||||||
|
.metadata()
|
||||||
|
.unwrap();
|
||||||
|
let opened = std::fs::File::from(opened.parent().try_clone().unwrap())
|
||||||
|
.metadata()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!((created.dev(), created.ino()), (opened.dev(), opened.ino()));
|
||||||
|
assert_eq!(created.permissions().mode() & 0o777, 0o750);
|
||||||
|
}
|
||||||
|
assert!(AnchoredPath::open_runtime_parent(Path::new("telemt.pid"), None, false).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn runtime_parent_policy_only_requires_trusted_ownership_when_strict() {
|
||||||
|
if !nix::unistd::Uid::effective().is_root() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let parent = directory.path().join("foreign-owner");
|
||||||
|
std::fs::create_dir(&parent).unwrap();
|
||||||
|
nix::unistd::chown(&parent, Some(nix::unistd::Uid::from_raw(65534)), None).unwrap();
|
||||||
|
let path = parent.join("telemt.pid");
|
||||||
|
|
||||||
|
assert!(AnchoredPath::open_runtime_parent(&path, None, false).is_ok());
|
||||||
|
assert!(AnchoredPath::open_runtime_parent(&path, None, true).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn atomic_replace_does_not_follow_final_symlink() {
|
fn atomic_replace_does_not_follow_final_symlink() {
|
||||||
let directory = tempfile::tempdir().unwrap();
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use base64::Engine as _;
|
use base64::Engine as _;
|
||||||
|
|
||||||
|
use crate::config::WebCarrierMethod;
|
||||||
use crate::crypto::SecureRandom;
|
use crate::crypto::SecureRandom;
|
||||||
|
|
||||||
/// Browser security policy for the transient Telegram Desktop bridge page.
|
/// Browser security policy for the transient Telegram Desktop bridge page.
|
||||||
@@ -34,6 +35,7 @@ pub(crate) fn render(
|
|||||||
reconnect_grace_secs: u64,
|
reconnect_grace_secs: u64,
|
||||||
carrier_probe_coalesce_ms: u64,
|
carrier_probe_coalesce_ms: u64,
|
||||||
bridge_diagnostics_enabled: bool,
|
bridge_diagnostics_enabled: bool,
|
||||||
|
carrier_method: WebCarrierMethod,
|
||||||
rng: &SecureRandom,
|
rng: &SecureRandom,
|
||||||
) -> BridgePage {
|
) -> BridgePage {
|
||||||
let mut nonce = [0u8; 18];
|
let mut nonce = [0u8; 18];
|
||||||
@@ -63,6 +65,8 @@ pub(crate) fn render(
|
|||||||
.replace("__REQUEST_RUNTIME__", REQUEST_RUNTIME)
|
.replace("__REQUEST_RUNTIME__", REQUEST_RUNTIME)
|
||||||
.replace("__BUFFER_RUNTIME__", BUFFER_RUNTIME)
|
.replace("__BUFFER_RUNTIME__", BUFFER_RUNTIME)
|
||||||
.replace("__RECOVERY_RUNTIME__", RECOVERY_RUNTIME)
|
.replace("__RECOVERY_RUNTIME__", RECOVERY_RUNTIME)
|
||||||
|
.replace("__DOWNLINK_RUNTIME__", DOWNLINK_RUNTIME)
|
||||||
|
.replace("__CONVEYOR_RUNTIME__", CONVEYOR_RUNTIME)
|
||||||
.replace("__RUNTIME__", RUNTIME)
|
.replace("__RUNTIME__", RUNTIME)
|
||||||
.replace(
|
.replace(
|
||||||
"__DIAGNOSTIC_BINDING__;\n",
|
"__DIAGNOSTIC_BINDING__;\n",
|
||||||
@@ -124,6 +128,7 @@ pub(crate) fn render(
|
|||||||
.replace("__HOST__", host)
|
.replace("__HOST__", host)
|
||||||
.replace("__BASE_PREFIX__", base_prefix)
|
.replace("__BASE_PREFIX__", base_prefix)
|
||||||
.replace("__BOOTSTRAP__", bootstrap)
|
.replace("__BOOTSTRAP__", bootstrap)
|
||||||
|
.replace("__CARRIER_METHOD__", carrier_method.as_str())
|
||||||
.replace("__BATCH_LIMIT__", &batch_limit.to_string())
|
.replace("__BATCH_LIMIT__", &batch_limit.to_string())
|
||||||
.replace("__QUEUE_LIMIT__", &queue_limit.to_string())
|
.replace("__QUEUE_LIMIT__", &queue_limit.to_string())
|
||||||
.replace("__QUEUE_ITEMS__", &queue_items.to_string())
|
.replace("__QUEUE_ITEMS__", &queue_items.to_string())
|
||||||
@@ -171,6 +176,8 @@ const RESPONSE_RUNTIME: &str = include_str!("bridge/response.js");
|
|||||||
const REQUEST_RUNTIME: &str = include_str!("bridge/request.js");
|
const REQUEST_RUNTIME: &str = include_str!("bridge/request.js");
|
||||||
const BUFFER_RUNTIME: &str = include_str!("bridge/buffers.js");
|
const BUFFER_RUNTIME: &str = include_str!("bridge/buffers.js");
|
||||||
const RECOVERY_RUNTIME: &str = include_str!("bridge/recovery.js");
|
const RECOVERY_RUNTIME: &str = include_str!("bridge/recovery.js");
|
||||||
|
const DOWNLINK_RUNTIME: &str = include_str!("bridge/downlink.js");
|
||||||
|
const CONVEYOR_RUNTIME: &str = include_str!("bridge/conveyor.js");
|
||||||
const RUNTIME: &str = include_str!("bridge/runtime.js");
|
const RUNTIME: &str = include_str!("bridge/runtime.js");
|
||||||
|
|
||||||
// Rendered wire-contract tests remain separate from the embedded document.
|
// Rendered wire-contract tests remain separate from the embedded document.
|
||||||
|
|||||||
@@ -0,0 +1,265 @@
|
|||||||
|
'use strict';
|
||||||
|
const assert=require('node:assert/strict');
|
||||||
|
const fs=require('node:fs');
|
||||||
|
const vm=require('node:vm');
|
||||||
|
const path=require('node:path');
|
||||||
|
|
||||||
|
function renderedPage(){
|
||||||
|
if(process.argv.includes('--stdin'))return fs.readFileSync(0,'utf8');
|
||||||
|
let page=fs.readFileSync(path.join(__dirname,'document.html'),'utf8');
|
||||||
|
const modules={RESPONSE:'response',REQUEST:'request',BUFFER:'buffers',RECOVERY:'recovery',CONVEYOR:'conveyor',DOWNLINK:'downlink'};
|
||||||
|
page=page.replace('__DIAGNOSTIC_RUNTIME__','');
|
||||||
|
for(const [key,file] of Object.entries(modules))if(page.includes('__'+key+'_RUNTIME__'))page=page.replace('__'+key+'_RUNTIME__',()=>fs.readFileSync(path.join(__dirname,file+'.js'),'utf8'));
|
||||||
|
page=page.replace('__RUNTIME__',()=>fs.readFileSync(path.join(__dirname,'runtime.js'),'utf8'));
|
||||||
|
const values={BOOTSTRAP:'A'.repeat(43),HOST:'proxy.example.com',BASE_PREFIX:'',CARRIER_METHOD:'POST',NEGOTIATION_ENABLED:'true',CANDIDATE_COUNT:4,CARRIER_DEADLINES:'3,5,8,12',LONG_POLL_SECS:25,BRIDGE_REQUEST_SECS:10,BRIDGE_RETRY_SECS:90,BRIDGE_RECOVERY_SECS:15,WEBSOCKET_OPEN_SECS:15,RECONNECT_GRACE_SECS:120,CARRIER_PROBE_COALESCE_MS:0,BATCH_LIMIT:2097152,QUEUE_LIMIT:33554432,QUEUE_ITEMS:16384,MAX_STREAMS:1024,STATUS_FUNCTION:"state=>{if(port&&!closed)port.postMessage({t:'status',state})}",HELLO_TIMEOUT_CALLBACK:"()=>fail('timeout')",PAGEHIDE_CALLBACK:'()=>close(true)'};
|
||||||
|
return page.replace(/__([A-Z_]+)__/g,(all,key)=>key.startsWith('DIAGNOSTIC_')?'':String(values[key]??''));
|
||||||
|
}
|
||||||
|
function frame(type,id=0,payload=[]){
|
||||||
|
const data=new Uint8Array(8+payload.length),view=new DataView(data.buffer);
|
||||||
|
data[0]=type;data[1]=id>>>16;data[2]=id>>>8;data[3]=id;view.setUint32(4,payload.length);data.set(payload,8);return data.buffer;
|
||||||
|
}
|
||||||
|
function join(...frames){const data=new Uint8Array(frames.reduce((n,f)=>n+f.byteLength,0));let offset=0;for(const f of frames){data.set(new Uint8Array(f),offset);offset+=f.byteLength}return data.buffer}
|
||||||
|
async function flush(){for(let i=0;i<40;i++)await Promise.resolve()}
|
||||||
|
function environment(page,android=false){
|
||||||
|
let now=1000,nextTimer=1;const timers=new Map(),events=new Map(),requests=[],received=[];
|
||||||
|
const port={onmessage:null,start(){},close(){},postMessage(value,transfer){received.push(structuredClone(value,{transfer:transfer||[]}))}};
|
||||||
|
const context={ArrayBuffer,Uint8Array,DataView,TextDecoder,TextEncoder,URL,Headers,AbortController,ReadableStream,structuredClone,console,
|
||||||
|
location:{hash:android?'#android='+'A'.repeat(43):'',pathname:'/',search:'?bridge=test'},history:{replaceState(){}},parent:{},
|
||||||
|
performance:{now:()=>now},Date:class extends Date{static now(){return now}},document:{visibilityState:'visible',addEventListener(){}},
|
||||||
|
addEventListener:(name,fn)=>events.set(name,fn),setTimeout:(fn,delay)=>{const id=nextTimer++;timers.set(id,{fn,at:now+delay});return id},clearTimeout:id=>timers.delete(id),
|
||||||
|
fetch:(url,options)=>new Promise((resolve,reject)=>{const request={url,options,resolve,reject};requests.push(request);options.signal?.addEventListener('abort',()=>reject(new Error('aborted')),{once:true})}),
|
||||||
|
WebSocket:class {static OPEN=1;static CLOSING=2;constructor(){throw new Error('unexpected websocket')}}
|
||||||
|
};
|
||||||
|
if(android)context.TelegramWebProxy={onmessage:null,postMessage(value){received.push(typeof value==='string'?JSON.parse(value):structuredClone(value))}};
|
||||||
|
vm.createContext(context);
|
||||||
|
for(const match of page.matchAll(/<script\b[^>]*>([\s\S]*?)<\/script>/g))vm.runInContext(match[1],context);
|
||||||
|
if(!android)events.get('message')({source:context.parent,origin:'http://127.0.0.1:12345',data:{t:'tproxy-init',v:1},ports:[port]});
|
||||||
|
const send=data=>android?context.TelegramWebProxy.onmessage({data}):port.onmessage({data});
|
||||||
|
const pending=suffix=>requests.filter(r=>r.url.endsWith(suffix)&&!r.answered);
|
||||||
|
const answer=(request,status,body=null,headers={})=>{request.answered=true;request.resolve({status,headers:new Headers(headers),body:body===null?null:new ReadableStream({start(c){c.enqueue(new Uint8Array(body));c.close()}})})};
|
||||||
|
return {context,send,pending,answer,received,requests,async tick(ms){const until=now+ms;for(;;){const entry=[...timers].filter(([,t])=>t.at<=until).sort((a,b)=>a[1].at-b[1].at)[0];if(!entry)break;now=entry[1].at;timers.delete(entry[0]);entry[1].fn();await flush()}now=until;await flush()},close(){events.get('pagehide')();}};
|
||||||
|
}
|
||||||
|
async function session(page,carrier='https',window=1,android=false){
|
||||||
|
const env=environment(page,android);env.send(frame(16,0,[1]));await flush();
|
||||||
|
const request=env.pending('/api/v1/session')[0];assert.ok(request,'session requested');
|
||||||
|
const headers={'X-Session-Token':'B'.repeat(43),'X-Down-Cursor':'0','X-Carrier-Mode':carrier,'X-Carrier-Attempt':'1','X-Carrier-Candidate-Count':'4','X-Carrier-Deadline':'12','X-Carrier-State':'provisional'};
|
||||||
|
if(window!==null)headers['X-Telemt-Up-Window']=String(window);
|
||||||
|
env.answer(request,200,frame(17),headers);await flush();return env;
|
||||||
|
}
|
||||||
|
const tests=[];
|
||||||
|
function test(name,run){tests.push({name,run})}
|
||||||
|
function ack(env,request){env.answer(request,204,null,{'X-Up-Ack':request.options.headers['X-Up-Seq']})}
|
||||||
|
function binary(env){return env.received.filter(value=>value instanceof ArrayBuffer&&new Uint8Array(value)[0]!==17)}
|
||||||
|
function streaming(env,request,next='1',length=null){
|
||||||
|
let controller;const body=new ReadableStream({start(value){controller=value}}),headers={'X-Down-Cursor':next};if(length!==null)headers['Content-Length']=String(length);
|
||||||
|
request.answered=true;request.resolve({status:200,headers:new Headers(headers),body});return controller;
|
||||||
|
}
|
||||||
|
test('already queued OPEN and DATA share the zero-wait startup request',async page=>{
|
||||||
|
const env=await session(page);const body=join(frame(1,1),frame(2,1,[7,8]));env.send(body);await flush();
|
||||||
|
assert.equal(env.pending('/api/v1/up')[0].options.body.byteLength,body.byteLength);env.close();await flush();
|
||||||
|
});
|
||||||
|
test('negotiated conveyor sends DATA before the OPEN acknowledgement',async page=>{
|
||||||
|
for(const android of [false,true])for(const carrier of ['https','https-lanes']){
|
||||||
|
const env=await session(page,carrier,4,android);env.send(frame(1,1));await flush();env.send(frame(2,1,[9]));await flush();
|
||||||
|
assert.equal(env.pending('/api/v1/up').length,2,carrier+' startup must not wait for ACK1');env.close();await flush();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
test('unsupported server and disabled conveyor remain serialized',async page=>{
|
||||||
|
for(const window of [null,1]){
|
||||||
|
const env=await session(page,'https',window);env.send(frame(1,1));await flush();env.send(frame(2,1,[9]));await flush();
|
||||||
|
assert.equal(env.pending('/api/v1/up').length,1);env.close();await flush();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
test('lost ACK1 cannot slide the window after ACK2 through ACK4',async page=>{
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();
|
||||||
|
for(let i=0;i<6;i++){env.send(frame(2,1,[i]));await flush()}
|
||||||
|
const up=env.pending('/api/v1/up');assert.equal(up.length,4);
|
||||||
|
for(const request of up.slice(1)){ack(env,request);await flush()}
|
||||||
|
assert.equal(env.requests.filter(r=>r.url.endsWith('/api/v1/up')).length,4);
|
||||||
|
ack(env,up[0]);await flush();
|
||||||
|
const next=env.pending('/api/v1/up')[0];assert.equal(next.options.headers['X-Up-Seq'],'5');assert.equal(next.options.headers['X-Telemt-Up-Confirmed'],'4');
|
||||||
|
env.close();await flush();
|
||||||
|
});
|
||||||
|
test('early downlink cannot deliver native data before a verified commit',async page=>{
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();
|
||||||
|
const down=env.pending('/api/v1/down')[0];assert.ok(down,'down starts before ACK');const data=frame(2,1,[4]);
|
||||||
|
const stream=streaming(env,down,'1',data.byteLength);stream.enqueue(new Uint8Array(data));await flush();assert.equal(binary(env).length,0);
|
||||||
|
ack(env,env.pending('/api/v1/up')[0]);await flush();assert.equal(binary(env).length,1);
|
||||||
|
assert.equal(env.requests.filter(r=>r.url.endsWith('/api/v1/down')).length,1,'cursor cannot advance before EOF');
|
||||||
|
stream.close();await flush();assert.equal(env.pending('/api/v1/down')[0].options.headers['X-Down-Cursor'],'1');env.close();await flush();
|
||||||
|
});
|
||||||
|
test('DATA and WINDOW prefixes are delivered once across a mid-body network failure',async page=>{
|
||||||
|
for(const carrier of ['https','https-lanes']){
|
||||||
|
const env=await session(page,carrier,4);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const first=frame(2,1,[1,2]),credit=frame(4,1,[0,0,0,2]),last=frame(3,1),body=join(first,credit,last);
|
||||||
|
const stream=streaming(env,env.pending('/api/v1/down')[0],'1',body.byteLength);
|
||||||
|
stream.enqueue(new Uint8Array(join(first,credit)));await flush();assert.equal(binary(env).length,2);
|
||||||
|
stream.error(new Error('connection lost'));await flush();
|
||||||
|
const replay=env.pending('/api/v1/down')[0];assert.ok(replay);assert.equal(replay.options.headers['X-Down-Cursor'],'0');
|
||||||
|
env.answer(replay,200,body,{'X-Down-Cursor':'1','Content-Length':String(body.byteLength)});await flush();
|
||||||
|
assert.deepEqual(binary(env).map(data=>new Uint8Array(data)[0]),[2,4,3]);
|
||||||
|
assert.equal(env.pending('/api/v1/down')[0].options.headers['X-Down-Cursor'],'1');
|
||||||
|
assert.equal(env.requests.filter(r=>r.url.endsWith('/api/v1/down')).length,3,'successful replay is consumed without another old-cursor request');env.close();await flush();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
test('changed replay prefix fails closed without duplicate native frames',async page=>{
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const data=frame(2,1,[7]),tail=frame(2,1,[8]),body=join(data,tail),stream=streaming(env,env.pending('/api/v1/down')[0],'1',body.byteLength);
|
||||||
|
stream.enqueue(new Uint8Array(data));await flush();stream.error(new Error('lost'));await flush();
|
||||||
|
const changed=join(frame(2,1,[9]),tail);env.answer(env.pending('/api/v1/down')[0],200,changed,{'X-Down-Cursor':'1','Content-Length':String(changed.byteLength)});await flush();
|
||||||
|
assert.equal(binary(env).length,1);assert.ok(env.received.some(value=>value&&value.t==='close'));env.close();await flush();
|
||||||
|
});
|
||||||
|
test('partial frame boundaries never escape to native and cancellation owns every lease',async page=>{
|
||||||
|
const env=await session(page,'https-lanes',4,true);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const data=frame(2,1,[1,2,3]),stream=streaming(env,env.pending('/api/v1/down')[0],'1',data.byteLength);
|
||||||
|
stream.enqueue(new Uint8Array(data,0,5));await flush();assert.equal(binary(env).length,0);
|
||||||
|
stream.enqueue(new Uint8Array(data,5));await flush();assert.equal(binary(env).length,1);
|
||||||
|
env.close();await flush();assert.equal(binary(env).length,1);
|
||||||
|
});
|
||||||
|
test('post-commit lane batching keeps OPEN and DATA from one native message together',async page=>{
|
||||||
|
const env=await session(page,'https-lanes',1);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const body=join(frame(1,2),frame(2,2,[7]));env.send(body);await flush();
|
||||||
|
assert.equal(env.pending('/api/v1/up')[0].options.body.byteLength,body.byteLength);env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('page-owned HTTP method and identity survive frozen retries',async page=>{
|
||||||
|
const expected=/const carrierMethod='(POST|PUT)'/.exec(page)[1],env=await session(page,'https',4);
|
||||||
|
assert.equal(env.requests.find(r=>r.url.endsWith('/api/v1/session')).options.method,'POST');
|
||||||
|
env.send(frame(1,1));await flush();const first=env.pending('/api/v1/up')[0];ack(env,first);await flush();
|
||||||
|
env.send(frame(2,1,[7]));await flush();const original=env.pending('/api/v1/up')[0];
|
||||||
|
original.answered=true;original.reject(new Error('lost'));await flush();const replay=env.pending('/api/v1/up')[0];
|
||||||
|
assert.equal(replay.options.body,original.options.body);assert.deepEqual(replay.options.headers,original.options.headers);
|
||||||
|
assert.equal(replay.options.method,expected);ack(env,replay);await flush();
|
||||||
|
for(const request of env.requests.filter(r=>/\/api\/v1\/(up|down)$/.test(r.url)))assert.equal(request.options.method,expected);
|
||||||
|
env.close();await flush();assert.equal(env.requests.find(r=>r.options.method==='DELETE').options.method,'DELETE');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('RTT scales time but adds no OPEN ACK barrier or coalescing timer',async page=>{
|
||||||
|
for(const rtt of [40,100,400]){
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();await env.tick(1);
|
||||||
|
env.send(frame(2,1,[5]));await flush();assert.equal(env.pending('/api/v1/up').length,2);
|
||||||
|
await env.tick(rtt-1);for(const request of env.pending('/api/v1/up'))ack(env,request);await flush();
|
||||||
|
assert.equal(env.received.filter(value=>value instanceof ArrayBuffer&&new Uint8Array(value)[0]===17).length,1);
|
||||||
|
assert.ok(env.received.filter(value=>value&&value.t==='status').every(value=>Object.keys(value).sort().join(',')==='state,t'));
|
||||||
|
env.close();await flush();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('simultaneous failed lane polls each consume their own exact replay',async page=>{
|
||||||
|
const env=await session(page,'https-lanes',4);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
env.send(frame(1,2));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const streams=[];
|
||||||
|
for(const request of env.pending('/api/v1/down')){
|
||||||
|
const id=Number(request.options.headers['X-Lane-ID']),data=frame(2,id,[id]),body=join(data,frame(4,id,[0,0,0,1]));
|
||||||
|
const response=streaming(env,request,'1',body.byteLength);response.enqueue(new Uint8Array(data));streams.push({id,response,body});
|
||||||
|
}
|
||||||
|
await flush();assert.equal(binary(env).length,2);for(const item of streams)item.response.error(new Error('lost'));await flush();
|
||||||
|
for(let round=0;round<3;round++){
|
||||||
|
for(const request of env.pending('/api/v1/down').filter(r=>r.options.headers['X-Down-Cursor']==='0')){
|
||||||
|
const body=streams.find(item=>String(item.id)===request.options.headers['X-Lane-ID']).body;
|
||||||
|
env.answer(request,200,body,{'X-Down-Cursor':'1','Content-Length':String(body.byteLength)});
|
||||||
|
}await flush();
|
||||||
|
}
|
||||||
|
assert.equal(binary(env).length,4);assert.equal(env.pending('/api/v1/down').length,2);
|
||||||
|
assert.ok(env.pending('/api/v1/down').every(r=>r.options.headers['X-Down-Cursor']==='1'));env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('incremental decoder rejects changed cursor, invalid WINDOW and cross-lane frames',async page=>{
|
||||||
|
for(const invalid of ['cursor','credit','lane']){
|
||||||
|
const env=await session(page,'https-lanes',4);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const body=invalid==='credit'?frame(4,1,[0,0,0,0]):frame(2,invalid==='lane'?2:1,[1]);
|
||||||
|
env.answer(env.pending('/api/v1/down')[0],200,body,{'X-Down-Cursor':invalid==='cursor'?'2':'1','Content-Length':String(body.byteLength)});await flush();
|
||||||
|
assert.equal(binary(env).length,0);assert.ok(env.received.some(value=>value&&value.t==='close'));env.close();await flush();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a replay without Content-Length cannot truncate an already known batch',async page=>{
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const prefix=frame(2,1,[1]),tail=frame(2,1,[2]),body=join(prefix,tail),response=streaming(env,env.pending('/api/v1/down')[0],'1',body.byteLength);
|
||||||
|
response.enqueue(new Uint8Array(prefix));await flush();response.error(new Error('lost'));await flush();
|
||||||
|
env.answer(env.pending('/api/v1/down')[0],200,prefix,{'X-Down-Cursor':'1'});await flush();
|
||||||
|
assert.equal(binary(env).length,1);assert.ok(env.received.some(value=>value&&value.t==='close'));
|
||||||
|
assert.ok(!env.requests.some(r=>r.url.endsWith('/api/v1/down')&&r.options.headers['X-Down-Cursor']==='1'));env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a pipelined startup flight adopts the recovery budget once another ACK commits',async page=>{
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();env.send(frame(2,1,[7]));await flush();
|
||||||
|
const [head,tail]=env.pending('/api/v1/up');ack(env,head);await flush();tail.answered=true;tail.reject(new Error('lost'));await flush();
|
||||||
|
const replay=env.pending('/api/v1/up')[0];assert.ok(replay,'post-commit recovery replays immediately instead of staying in the startup backoff');
|
||||||
|
assert.equal(replay.options.body,tail.options.body);ack(env,replay);await flush();env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('candidate rollback restores every unacknowledged OPEN and DATA without a second WELCOME',async page=>{
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();env.send(frame(2,1,[7]));await flush();
|
||||||
|
const retired=env.requests.filter(r=>/\/api\/v1\/(up|down)$/.test(r.url));await env.tick(3001);
|
||||||
|
const next=env.pending('/api/v1/session')[0];assert.ok(next,'candidate fallback creates a new session');
|
||||||
|
env.answer(next,200,frame(17),{'X-Session-Token':'C'.repeat(43),'X-Down-Cursor':'0','X-Carrier-Mode':'https-lanes','X-Carrier-Attempt':'2','X-Carrier-Candidate-Count':'4','X-Carrier-Deadline':'12','X-Carrier-State':'provisional','X-Telemt-Up-Window':'4'});await flush();
|
||||||
|
const first=env.requests.find(r=>r.url.endsWith('/api/v1/up')&&r.options.headers.Authorization==='Bearer '+'C'.repeat(43));
|
||||||
|
assert.ok(first);assert.deepEqual(new Uint8Array(first.options.body),new Uint8Array(join(frame(1,1),frame(2,1,[7]))));
|
||||||
|
assert.ok(retired.every(r=>r.options.signal.aborted));assert.equal(env.received.filter(value=>value instanceof ArrayBuffer&&new Uint8Array(value)[0]===17).length,1);
|
||||||
|
ack(env,first);await flush();env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('downlink prefix reservations are bounded, cancellable and released on retirement',async page=>{
|
||||||
|
const env=environment(page),budget=env.context.TelemtBridgeDownlink.budget(()=>16),first=await budget.acquire(16);
|
||||||
|
const controller=new AbortController();let acquired=false;
|
||||||
|
const parked=budget.acquire(8,controller.signal).then(release=>{acquired=true;release()},()=>{});await flush();assert.equal(acquired,false);
|
||||||
|
controller.abort();await parked;first();assert.equal(budget.used(),0);
|
||||||
|
const release=await budget.acquire(16);release();release();assert.equal(budget.used(),0);env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('two interrupted and differently chunked replays never duplicate DATA WINDOW or CLOSE',async page=>{
|
||||||
|
const env=await session(page,'https',4);env.send(frame(1,1));await flush();ack(env,env.pending('/api/v1/up')[0]);await flush();
|
||||||
|
const data=frame(2,1,[1]),credit=frame(4,1,[0,0,0,1]),close=frame(3,1),body=join(data,credit,close);
|
||||||
|
const first=streaming(env,env.pending('/api/v1/down')[0],'1',body.byteLength);
|
||||||
|
first.enqueue(new Uint8Array(data));await flush();first.error(new Error('lost once'));await flush();
|
||||||
|
const second=streaming(env,env.pending('/api/v1/down')[0],'1',body.byteLength);
|
||||||
|
const prefix=new Uint8Array(join(data,credit));second.enqueue(prefix.subarray(0,3));second.enqueue(prefix.subarray(3));await flush();
|
||||||
|
assert.deepEqual(binary(env).map(value=>new Uint8Array(value)[0]),[2,4]);second.error(new Error('lost twice'));await flush();await env.tick(1000);
|
||||||
|
const last=streaming(env,env.pending('/api/v1/down')[0],'1',body.byteLength);
|
||||||
|
for(const byte of new Uint8Array(body))last.enqueue(new Uint8Array([byte]));last.close();
|
||||||
|
for(let i=0;i<20&&(!env.pending('/api/v1/down').length);i++)await flush();
|
||||||
|
assert.deepEqual(binary(env).map(value=>new Uint8Array(value)[0]),[2,4,3]);assert.equal(env.pending('/api/v1/down')[0].options.headers['X-Down-Cursor'],'1');
|
||||||
|
env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('multiple precommit lanes survive fallback without losing stream ordering',async page=>{
|
||||||
|
const env=await session(page,'https-lanes',4);env.send(join(frame(1,1),frame(1,2)));await flush();
|
||||||
|
env.send(join(frame(2,2,[2]),frame(2,1,[1])));await flush();await env.tick(3001);
|
||||||
|
const next=env.pending('/api/v1/session')[0];env.answer(next,200,frame(17),{'X-Session-Token':'C'.repeat(43),'X-Down-Cursor':'0','X-Carrier-Mode':'https','X-Carrier-Attempt':'2','X-Carrier-Candidate-Count':'4','X-Carrier-Deadline':'12','X-Carrier-State':'provisional','X-Telemt-Up-Window':'4'});await flush();
|
||||||
|
const requests=env.requests.filter(r=>r.url.endsWith('/api/v1/up')&&r.options.headers.Authorization==='Bearer '+'C'.repeat(43));
|
||||||
|
const frames=[];for(const request of requests){const bytes=new Uint8Array(request.options.body);for(let offset=0;offset<bytes.length;){const view=new DataView(bytes.buffer,offset);frames.push([bytes[offset],bytes[offset+3]]);offset+=8+view.getUint32(4);}}
|
||||||
|
for(const id of [1,2])assert.deepEqual(frames.filter(value=>value[1]===id),[[1,id],[2,id]]);
|
||||||
|
assert.equal(frames.length,4);env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('receiver retirement releases shared prefix capacity without exposing a partial frame',async page=>{
|
||||||
|
const env=environment(page),module=env.context.TelemtBridgeDownlink,budget=module.budget(()=>18),received=[];
|
||||||
|
function receiver(id){
|
||||||
|
let cursor='0',stream;const controller=new AbortController(),response={status:200,headers:new Headers({'X-Down-Cursor':'1','Content-Length':'18'}),body:new ReadableStream({start(value){stream=value}})};
|
||||||
|
const owner=module.create({failure:(reason,message)=>Object.assign(new Error(message),{telemtReason:reason}),alive:()=>true,cursor:()=>cursor,lane:id,limit:()=>18,budget,cancel(){},ready:()=>Promise.resolve(),deliver:data=>received.push(new Uint8Array(data)),advance:value=>{cursor=value},finished(){}});
|
||||||
|
const result=owner.read(response,controller.signal).catch(()=>{});return {owner,result,stream,controller,cursor:()=>cursor};
|
||||||
|
}
|
||||||
|
const a=receiver(1);a.stream.enqueue(new Uint8Array(frame(2,1,[1])));await flush();
|
||||||
|
const b=receiver(2);b.stream.enqueue(new Uint8Array(join(frame(2,2,[2]),frame(2,2,[3]))));b.stream.close();await flush();
|
||||||
|
assert.equal(budget.used(),18);assert.equal(received.length,1);assert.equal(b.cursor(),'0');
|
||||||
|
a.owner.close();a.controller.abort();a.stream.error(new Error('retired'));await a.result;await b.result;
|
||||||
|
assert.equal(received.length,3);assert.equal(b.cursor(),'1');assert.equal(budget.used(),0);
|
||||||
|
b.owner.close();env.close();await flush();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('native close cancels every HTTP request and pagehide cannot repeat cleanup',async page=>{
|
||||||
|
for(const android of [false,true])for(const carrier of ['https','https-lanes']){
|
||||||
|
const env=await session(page,carrier,4,android);env.send(frame(1,1));await flush();env.send(frame(2,1,[7]));await flush();
|
||||||
|
const up=env.pending('/api/v1/up'),down=env.pending('/api/v1/down'),requests=up.concat(down);
|
||||||
|
assert.equal(up.length,2);assert.equal(down.length,1);assert.ok(requests.every(request=>!request.options.signal.aborted));
|
||||||
|
env.send({t:'close'});await flush();
|
||||||
|
assert.ok(requests.every(request=>request.options.signal.aborted));
|
||||||
|
assert.equal(env.requests.filter(request=>request.options.method==='DELETE').length,1);
|
||||||
|
const count=env.requests.length;env.close();await env.tick(120000);
|
||||||
|
assert.equal(env.requests.filter(request=>request.options.method==='DELETE').length,1);
|
||||||
|
assert.equal(env.requests.length,count);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
(async()=>{const page=renderedPage();let failed=0;for(const {name,run} of tests){let timer;try{await Promise.race([run(page),new Promise((_,reject)=>{timer=setTimeout(()=>reject(new Error('test made no bounded progress')),5000)})]);console.log('ok - '+name)}catch(error){failed++;console.error('not ok - '+name+'\n'+error.stack)}finally{clearTimeout(timer)}}if(failed)process.exitCode=1})();
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
use std::io::Write;
|
||||||
|
use std::process::{Command, Stdio};
|
||||||
|
|
||||||
|
/// Exercises the actual generated page with deterministic network and native-boundary doubles.
|
||||||
|
pub(super) fn run(page: &super::BridgePage) {
|
||||||
|
let mut child = Command::new("node")
|
||||||
|
.args(["-e", include_str!("behavior_tests.js"), "--", "--stdin"])
|
||||||
|
.stdin(Stdio::piped())
|
||||||
|
.stdout(Stdio::piped())
|
||||||
|
.stderr(Stdio::piped())
|
||||||
|
.spawn()
|
||||||
|
.expect("Node.js 22+ is required for executable WEB bridge regressions");
|
||||||
|
child
|
||||||
|
.stdin
|
||||||
|
.take()
|
||||||
|
.unwrap()
|
||||||
|
.write_all(page.body.as_bytes())
|
||||||
|
.unwrap();
|
||||||
|
let output = child.wait_with_output().unwrap();
|
||||||
|
assert!(
|
||||||
|
output.status.success(),
|
||||||
|
"bridge regressions failed:\n{}\n{}",
|
||||||
|
String::from_utf8_lossy(&output.stdout),
|
||||||
|
String::from_utf8_lossy(&output.stderr)
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
(()=>{'use strict';
|
||||||
|
// HTTP request ownership is separate from the native WINDOW credit protocol.
|
||||||
|
function create(settings){
|
||||||
|
const channels=new Map(),window=settings.window,token=settings.token,downBudget=globalThis.TelemtBridgeDownlink.budget(settings.queueLimit);
|
||||||
|
let closed=false,order=0,ready=settings.committed(),resolveReady;
|
||||||
|
const readyPromise=new Promise(resolve=>{resolveReady=resolve});if(ready)resolveReady();
|
||||||
|
const alive=channel=>!closed&&!channel.closed&&settings.alive(channel.lane);
|
||||||
|
const markReady=()=>{if(!ready){ready=true;resolveReady();settings.commit()}};
|
||||||
|
function waitReady(signal){
|
||||||
|
if(ready)return Promise.resolve();
|
||||||
|
return new Promise((resolve,reject)=>{const abort=()=>reject(new Error('carrier retired'));if(signal&&signal.aborted){abort();return}if(signal)signal.addEventListener('abort',abort,{once:true});readyPromise.then(()=>{if(signal)signal.removeEventListener('abort',abort);resolve()})});
|
||||||
|
}
|
||||||
|
function get(id,type){
|
||||||
|
let channel=channels.get(id);if(channel)return channel;
|
||||||
|
const lane=id===null?null:settings.lane(id,type);if(id!==null&&!lane)return null;
|
||||||
|
channel={id,lane,pending:[],flights:new Map(),acks:new Set(),next:1,confirmed:0,cursor:'0',controller:null,polling:false,closed:false};
|
||||||
|
channel.receiver=globalThis.TelemtBridgeDownlink.create({
|
||||||
|
failure:settings.failure,alive:()=>alive(channel),cursor:()=>channel.cursor,lane:id,limit:settings.batchLimit,budget:downBudget,
|
||||||
|
cancel:settings.cancel,ready:waitReady,deliver:settings.deliver,advance:next=>{channel.cursor=next},finished:()=>{if(lane)settings.finish(lane,false)}
|
||||||
|
});channels.set(id,channel);return channel;
|
||||||
|
}
|
||||||
|
function enqueue(data,defer){
|
||||||
|
if(closed)return;
|
||||||
|
const entries=settings.lanes?settings.buffers.splitFrames(data):[{id:null,type:0,data}],touched=new Set();
|
||||||
|
for(const entry of entries){
|
||||||
|
const channel=get(entry.id,entry.type);if(!channel)continue;
|
||||||
|
if(!settings.buffers.reserve(entry.data,channel.lane))throw settings.failure('capacity','carrier queue full');
|
||||||
|
channel.pending.push(entry.data);touched.add(channel);
|
||||||
|
}
|
||||||
|
if(!defer)for(const channel of touched)pump(channel);
|
||||||
|
}
|
||||||
|
function checkAck(response,sequence){
|
||||||
|
if(response.status!==204)throw settings.failure('http','uplink rejected');
|
||||||
|
if(response.headers.get('X-Up-Ack')!==String(sequence))throw settings.failure('protocol','uplink acknowledgement rejected');
|
||||||
|
}
|
||||||
|
function acknowledge(channel,sequence,lease){
|
||||||
|
if(!alive(channel)||channel.flights.get(sequence)!==lease)return;
|
||||||
|
settings.buffers.settleBatch(lease);channel.flights.delete(sequence);channel.acks.add(sequence);
|
||||||
|
while(channel.acks.delete(channel.confirmed+1))channel.confirmed++;
|
||||||
|
markReady();settings.traffic(lease.total,0);pump(channel);
|
||||||
|
}
|
||||||
|
async function send(channel,sequence,lease){
|
||||||
|
const headers={'X-Up-Seq':String(sequence)};if(channel.id!==null)headers['X-Lane-ID']=String(channel.id);
|
||||||
|
if(window>1)headers['X-Telemt-Up-Confirmed']=String(channel.confirmed);
|
||||||
|
const frozen=settings.options(settings.method,token,lease.body,headers,lease.controller.signal);
|
||||||
|
try{
|
||||||
|
while(alive(channel)&&!lease.cancelled){
|
||||||
|
try{const response=await settings.request('/api/v1/up',frozen,null,()=>ready?1:9);if(!alive(channel))return;checkAck(response,sequence);acknowledge(channel,sequence,lease);return}
|
||||||
|
catch(error){
|
||||||
|
if(!alive(channel)||lease.cancelled)return;
|
||||||
|
if(!ready){settings.failed(error);return}
|
||||||
|
let replayed=false;
|
||||||
|
const recovered=await settings.recover(error,async(signal,remaining)=>{
|
||||||
|
const response=await settings.request('/api/v1/up',Object.assign({},frozen,{signal}),remaining,2);
|
||||||
|
if(!alive(channel)||lease.cancelled)return;checkAck(response,sequence);replayed=true;acknowledge(channel,sequence,lease);
|
||||||
|
});
|
||||||
|
if(!recovered||!alive(channel)||replayed)return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}catch(error){if(alive(channel)&&!lease.cancelled)settings.failed(error)}
|
||||||
|
}
|
||||||
|
function pump(channel){
|
||||||
|
while(alive(channel)&&channel.pending.length&&channel.next<=channel.confirmed+window){
|
||||||
|
const lease=settings.buffers.takeBatch(channel.pending,channel.lane),sequence=channel.next++;
|
||||||
|
lease.controller=new AbortController();lease.order=order++;channel.flights.set(sequence,lease);send(channel,sequence,lease);
|
||||||
|
if(!channel.polling)poll(channel);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
async function poll(channel){
|
||||||
|
if(channel.polling||!alive(channel))return;channel.polling=true;
|
||||||
|
try{
|
||||||
|
while(alive(channel)){
|
||||||
|
const cursor=channel.cursor,headers={'X-Down-Cursor':cursor};if(channel.id!==null)headers['X-Lane-ID']=String(channel.id);
|
||||||
|
channel.controller=new AbortController();const frozen=settings.options(settings.method,token,null,headers,channel.controller.signal);
|
||||||
|
try{
|
||||||
|
const response=await settings.request('/api/v1/down',frozen,null,1,channel.receiver.read);
|
||||||
|
if(!alive(channel))return;if(response.status!==200&&response.status!==204)throw settings.failure('http','downlink rejected');
|
||||||
|
if(ready)settings.connected();
|
||||||
|
}catch(error){
|
||||||
|
if(!alive(channel))return;
|
||||||
|
if(!ready){settings.failed(error);return}
|
||||||
|
const recovered=await settings.recover(error,async(signal,remaining)=>{
|
||||||
|
const response=await settings.request('/api/v1/down',Object.assign({},frozen,{signal}),remaining,2,channel.receiver.read);
|
||||||
|
if(response.status!==200&&response.status!==204)throw settings.failure('http','downlink replay rejected');
|
||||||
|
});
|
||||||
|
if(!recovered||!alive(channel))return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}catch(error){if(alive(channel))settings.failed(error)}
|
||||||
|
finally{channel.polling=false;channel.controller=null}
|
||||||
|
}
|
||||||
|
function retire(channel,restore){
|
||||||
|
channel.closed=true;if(channel.controller)channel.controller.abort();channel.receiver.close();
|
||||||
|
const saved=[];
|
||||||
|
for(const lease of channel.flights.values()){if(restore)saved.push({order:lease.order,data:lease.body});settings.buffers.cancelBatch(lease);}
|
||||||
|
channel.flights.clear();channel.acks.clear();
|
||||||
|
for(const data of channel.pending)if(restore)saved.push({order:order++,data});
|
||||||
|
settings.buffers.releasePending(channel.pending,channel.lane);return saved;
|
||||||
|
}
|
||||||
|
return {enqueue,flush(){for(const channel of channels.values())pump(channel)},finish(lane){const channel=channels.get(lane.id);if(channel){channels.delete(lane.id);retire(channel,false)}},close(restore){
|
||||||
|
if(closed)return [];closed=true;const saved=[];for(const channel of channels.values())saved.push(...retire(channel,restore));channels.clear();
|
||||||
|
return saved.sort((a,b)=>a.order-b.order).map(entry=>entry.data);
|
||||||
|
}};
|
||||||
|
}
|
||||||
|
globalThis.TelemtBridgeConveyor=Object.freeze({create});
|
||||||
|
})();
|
||||||
@@ -20,6 +20,12 @@ __BUFFER_RUNTIME__
|
|||||||
__RECOVERY_RUNTIME__
|
__RECOVERY_RUNTIME__
|
||||||
</script>
|
</script>
|
||||||
<script nonce="__NONCE__">
|
<script nonce="__NONCE__">
|
||||||
|
__DOWNLINK_RUNTIME__
|
||||||
|
</script>
|
||||||
|
<script nonce="__NONCE__">
|
||||||
|
__CONVEYOR_RUNTIME__
|
||||||
|
</script>
|
||||||
|
<script nonce="__NONCE__">
|
||||||
__RUNTIME__
|
__RUNTIME__
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
(()=>{'use strict';
|
||||||
|
// Prefix ownership survives HTTP retries, but never a session or lane replacement.
|
||||||
|
function budget(limit){
|
||||||
|
let used=0;const waiters=[];
|
||||||
|
function wake(){for(const next of waiters.slice())next()}
|
||||||
|
function acquire(size,signal){
|
||||||
|
if(size>limit())return Promise.reject(new Error('downlink budget exceeded'));
|
||||||
|
return new Promise((resolve,reject)=>{
|
||||||
|
const remove=()=>{const at=waiters.indexOf(tryAcquire);if(at>=0)waiters.splice(at,1);if(signal)signal.removeEventListener('abort',abort)};
|
||||||
|
const abort=()=>{remove();reject(new Error('downlink aborted'))};
|
||||||
|
const tryAcquire=()=>{if(signal&&signal.aborted){abort();return}if(size<=limit()-used){remove();used+=size;let released=false;resolve(()=>{if(!released){released=true;used-=size;wake()}})}};
|
||||||
|
waiters.push(tryAcquire);if(signal)signal.addEventListener('abort',abort,{once:true});tryAcquire();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return {acquire,used:()=>used};
|
||||||
|
}
|
||||||
|
function create(settings){
|
||||||
|
let transaction=null,closed=false;
|
||||||
|
const protocol=message=>settings.failure('protocol',message);
|
||||||
|
const current=signal=>!closed&&settings.alive()&&!(signal&&signal.aborted);
|
||||||
|
function dispose(){if(transaction){transaction.release();transaction=null}}
|
||||||
|
function shape(type,id,size){
|
||||||
|
if(type===2)return id!==0&&size>0&&size<=1048576;
|
||||||
|
if(type===3)return id!==0&&size===0;
|
||||||
|
if(type===4)return id!==0&&size===4;
|
||||||
|
if(type===5)return id===0&&size<=64;
|
||||||
|
return type===31&&id===0&&size===0;
|
||||||
|
}
|
||||||
|
async function read(response,signal){
|
||||||
|
if(!current(signal))throw new Error('downlink retired');
|
||||||
|
if(response.status===204){
|
||||||
|
settings.cancel(response);if(transaction)throw protocol('missing replay batch');
|
||||||
|
const cursor=response.headers.get('X-Down-Cursor');
|
||||||
|
if(cursor!==null&&cursor!==settings.cursor())throw protocol('changed empty cursor');
|
||||||
|
if(response.headers.get('X-Lane-Closed')==='1')settings.finished();
|
||||||
|
return new ArrayBuffer(0);
|
||||||
|
}
|
||||||
|
const next=response.headers.get('X-Down-Cursor')||'',base=settings.cursor();
|
||||||
|
if(!/^[1-9]\d*$/.test(next)||BigInt(next)!==BigInt(base)+1n||BigInt(next)>18446744073709551615n)throw protocol('invalid downlink cursor');
|
||||||
|
const length=response.headers.get('Content-Length');let declared=null;
|
||||||
|
if(length!==null){if(!/^(0|[1-9]\d*)$/.test(length))throw protocol('invalid response length');declared=Number(length);if(!Number.isSafeInteger(declared)||declared===0||declared>settings.limit())throw protocol('response body overflow');}
|
||||||
|
if(transaction&&(transaction.base!==base||transaction.next!==next||(transaction.declared!==null&&declared!==null&&transaction.declared!==declared)))throw protocol('changed replay metadata');
|
||||||
|
if(!transaction){
|
||||||
|
const capacity=declared===null?settings.limit():declared,release=await settings.budget.acquire(capacity,signal);
|
||||||
|
if(!current(signal)){release();throw new Error('downlink retired');}
|
||||||
|
transaction={base,next,declared,bytes:new Uint8Array(capacity),retained:0,delivered:0,release};
|
||||||
|
}
|
||||||
|
const owner=transaction;
|
||||||
|
if(owner.declared===null&&declared!==null)owner.declared=declared;
|
||||||
|
await settings.ready(signal);
|
||||||
|
if(!current(signal)||transaction!==owner)throw new Error('downlink retired');
|
||||||
|
if(!response.body)throw protocol('missing response body');
|
||||||
|
const reader=response.body.getReader();let position=0,offset=0,frames=0,chunks=0,complete=false;
|
||||||
|
try{
|
||||||
|
for(;;){
|
||||||
|
let part;try{part=await reader.read();}catch(error){throw settings.failure('network','response stream interrupted');}
|
||||||
|
if(!current(signal)||transaction!==owner)throw new Error('downlink retired');
|
||||||
|
if(part.done)break;
|
||||||
|
const data=part.value;if(!(data instanceof Uint8Array)||++chunks>65536||data.byteLength>owner.bytes.byteLength-position)throw protocol('response body overflow');
|
||||||
|
const compared=Math.min(data.byteLength,Math.max(0,owner.retained-position));
|
||||||
|
for(let i=0;i<compared;i++)if(data[i]!==owner.bytes[position+i])throw protocol('changed replay prefix');
|
||||||
|
owner.bytes.set(data.subarray(compared),position+compared);position+=data.byteLength;owner.retained=Math.max(owner.retained,position);
|
||||||
|
while(position-offset>=8){
|
||||||
|
const view=new DataView(owner.bytes.buffer,offset),type=view.getUint8(0),id=(view.getUint8(1)<<16)|(view.getUint8(2)<<8)|view.getUint8(3),size=view.getUint32(4),end=offset+8+size;
|
||||||
|
if(!shape(type,id,size)||(settings.lane!==null&&id!==settings.lane)||end>owner.bytes.byteLength)throw protocol('invalid downlink frame');
|
||||||
|
if(end>position)break;if(++frames>4096)throw protocol('too many downlink frames');
|
||||||
|
if(type===4&&view.getUint32(8)===0)throw protocol('invalid WINDOW credit');
|
||||||
|
if(end>owner.delivered){
|
||||||
|
if(offset!==owner.delivered)throw protocol('invalid replay boundary');
|
||||||
|
const frame=owner.bytes.slice(offset,end).buffer;
|
||||||
|
settings.deliver(frame);owner.delivered=end;
|
||||||
|
}
|
||||||
|
offset=end;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(!position||offset!==position||position<owner.retained||(owner.declared!==null&&position!==owner.declared))throw protocol('incomplete downlink batch');
|
||||||
|
settings.advance(next);complete=true;dispose();return new ArrayBuffer(0);
|
||||||
|
}finally{
|
||||||
|
if(!complete)try{const cancelled=reader.cancel();if(cancelled)cancelled.catch(()=>{});}catch(error){}
|
||||||
|
reader.releaseLock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {read,close(){closed=true;dispose()}};
|
||||||
|
}
|
||||||
|
globalThis.TelemtBridgeDownlink=Object.freeze({create,budget});
|
||||||
|
})();
|
||||||
@@ -26,12 +26,13 @@ function create(settings){
|
|||||||
if(status===204&&(path==='/api/v1/up'||path==='/api/v1/down'))return {limit:0,exact:true,reason:'protocol'};
|
if(status===204&&(path==='/api/v1/up'||path==='/api/v1/down'))return {limit:0,exact:true,reason:'protocol'};
|
||||||
return {limit:0,exact:true,reason:'http'};
|
return {limit:0,exact:true,reason:'http'};
|
||||||
}
|
}
|
||||||
async function send(path,frozenOptions,remainingBudget,maxAttempts){
|
async function send(path,frozenOptions,remainingBudget,maxAttempts,receiver){
|
||||||
let delay=250,attempt=0,lastReason='network';maxAttempts=maxAttempts||9;
|
let delay=250,attempt=0,lastReason='network';maxAttempts=maxAttempts||9;
|
||||||
|
const attempts=()=>typeof maxAttempts==='function'?maxAttempts():maxAttempts;
|
||||||
const initialBudget=remainingBudget?Math.min(settings.retryMs(),remainingBudget()):settings.retryMs();
|
const initialBudget=remainingBudget?Math.min(settings.retryMs(),remainingBudget()):settings.retryMs();
|
||||||
const deadline=Date.now()+Math.max(0,initialBudget),external=frozenOptions.signal;
|
const deadline=Date.now()+Math.max(0,initialBudget),external=frozenOptions.signal;
|
||||||
const attemptLimit=path==='/api/v1/down'?settings.longPollMs()+settings.requestMs():settings.requestMs();
|
const attemptLimit=path==='/api/v1/down'?settings.longPollMs()+settings.requestMs():settings.requestMs();
|
||||||
while(attempt<maxAttempts){
|
while(attempt<attempts()){
|
||||||
if(settings.closed()||(external&&external.aborted))throw new Error('request aborted');
|
if(settings.closed()||(external&&external.aborted))throw new Error('request aborted');
|
||||||
const remaining=Math.min(deadline-Date.now(),remainingBudget?remainingBudget():Infinity);if(remaining<=0)break;attempt++;
|
const remaining=Math.min(deadline-Date.now(),remainingBudget?remainingBudget():Infinity);if(remaining<=0)break;attempt++;
|
||||||
const controller=new AbortController(),abort=()=>controller.abort();let timedOut=false;
|
const controller=new AbortController(),abort=()=>controller.abort();let timedOut=false;
|
||||||
@@ -45,12 +46,12 @@ function create(settings){
|
|||||||
lastReason='http';wait=retryAfterMs(fetched);settings.cancel(fetched);
|
lastReason='http';wait=retryAfterMs(fetched);settings.cancel(fetched);
|
||||||
}else{
|
}else{
|
||||||
const policy=responsePolicy(path,fetched.status);let body;
|
const policy=responsePolicy(path,fetched.status);let body;
|
||||||
try{body=await settings.read(fetched,policy.limit,policy.exact,controller.signal)}
|
try{body=receiver&&(fetched.status===200||fetched.status===204)?await receiver(fetched,controller.signal):await settings.read(fetched,policy.limit,policy.exact,controller.signal);}
|
||||||
catch(error){
|
catch(error){
|
||||||
controller.abort();
|
controller.abort();
|
||||||
if(external&&external.aborted)throw error;
|
if(external&&external.aborted)throw error;
|
||||||
if(timedOut)throw settings.failure('timeout','response deadline exceeded');
|
if(timedOut)throw settings.failure('timeout','response deadline exceeded');
|
||||||
throw settings.failure(policy.reason,error&&error.message);
|
throw settings.failure(settings.reason(error,policy.reason),error&&error.message);
|
||||||
}
|
}
|
||||||
response={status:fetched.status,headers:fetched.headers,body};return response;
|
response={status:fetched.status,headers:fetched.headers,body};return response;
|
||||||
}
|
}
|
||||||
@@ -60,7 +61,7 @@ function create(settings){
|
|||||||
if(timedOut)lastReason='timeout';
|
if(timedOut)lastReason='timeout';
|
||||||
if(settings.reason(error,'')==='protocol')throw error;
|
if(settings.reason(error,'')==='protocol')throw error;
|
||||||
}finally{clearTimeout(timer);if(external)external.removeEventListener('abort',abort)}
|
}finally{clearTimeout(timer);if(external)external.removeEventListener('abort',abort)}
|
||||||
const after=Math.min(deadline-Date.now(),remainingBudget?remainingBudget():Infinity);if(attempt>=maxAttempts||after<=0)break;
|
const after=Math.min(deadline-Date.now(),remainingBudget?remainingBudget():Infinity);if(attempt>=attempts()||after<=0)break;
|
||||||
settings.retrying();
|
settings.retrying();
|
||||||
const backoff=wait||delay+Math.floor(Math.random()*Math.max(1,delay/4));
|
const backoff=wait||delay+Math.floor(Math.random()*Math.max(1,delay/4));
|
||||||
await pause(Math.min(backoff,after),external);delay=Math.min(delay*2,2000);
|
await pause(Math.min(backoff,after),external);delay=Math.min(delay*2,2000);
|
||||||
|
|||||||
+45
-143
@@ -7,6 +7,8 @@ const responseBody=globalThis.TelemtBridgeResponse;if(!responseBody)throw new Er
|
|||||||
const requestSupport=globalThis.TelemtBridgeRequest;if(!requestSupport)throw new Error('missing request runtime');
|
const requestSupport=globalThis.TelemtBridgeRequest;if(!requestSupport)throw new Error('missing request runtime');
|
||||||
const bufferSupport=globalThis.TelemtBridgeBuffers;if(!bufferSupport)throw new Error('missing buffer runtime');
|
const bufferSupport=globalThis.TelemtBridgeBuffers;if(!bufferSupport)throw new Error('missing buffer runtime');
|
||||||
const recoverySupport=globalThis.TelemtBridgeRecovery;if(!recoverySupport)throw new Error('missing recovery runtime');
|
const recoverySupport=globalThis.TelemtBridgeRecovery;if(!recoverySupport)throw new Error('missing recovery runtime');
|
||||||
|
// Keep the method page-owned so recovery and config rollback cannot change frozen retries.
|
||||||
|
const carrierMethod='__CARRIER_METHOD__';
|
||||||
let negotiationEnabled=__NEGOTIATION_ENABLED__,candidateCount=__CANDIDATE_COUNT__,candidateDeadlines=[__CARRIER_DEADLINES__];
|
let negotiationEnabled=__NEGOTIATION_ENABLED__,candidateCount=__CANDIDATE_COUNT__,candidateDeadlines=[__CARRIER_DEADLINES__];
|
||||||
let longPollMs=__LONG_POLL_SECS__*1000,bridgeRequestMs=__BRIDGE_REQUEST_SECS__*1000,bridgeRetryMs=__BRIDGE_RETRY_SECS__*1000;
|
let longPollMs=__LONG_POLL_SECS__*1000,bridgeRequestMs=__BRIDGE_REQUEST_SECS__*1000,bridgeRetryMs=__BRIDGE_RETRY_SECS__*1000;
|
||||||
let bridgeRecoveryMs=__BRIDGE_RECOVERY_SECS__*1000,websocketOpenMs=__WEBSOCKET_OPEN_SECS__*1000,reconnectGraceMs=__RECONNECT_GRACE_SECS__*1000;
|
let bridgeRecoveryMs=__BRIDGE_RECOVERY_SECS__*1000,websocketOpenMs=__WEBSOCKET_OPEN_SECS__*1000,reconnectGraceMs=__RECONNECT_GRACE_SECS__*1000;
|
||||||
@@ -17,7 +19,7 @@ let laneQueueLimit=Math.min(queueLimit,8388608),laneItemLimit=Math.min(queueItem
|
|||||||
const fragment=location.hash,androidNonce=/^#android=([A-Za-z0-9_-]{43})$/.exec(fragment)?.[1]||'',recoveryPath=location.pathname+location.search;
|
const fragment=location.hash,androidNonce=/^#android=([A-Za-z0-9_-]{43})$/.exec(fragment)?.[1]||'',recoveryPath=location.pathname+location.search;
|
||||||
history.replaceState(null,'',location.pathname);
|
history.replaceState(null,'',location.pathname);
|
||||||
let initialized=false,closed=false,port=null,sessionToken='',cleanupToken='',createStarted=false,socket=null,socketReady=false,carrier='';
|
let initialized=false,closed=false,port=null,sessionToken='',cleanupToken='',createStarted=false,socket=null,socketReady=false,carrier='';
|
||||||
let upSequence=1,downCursor='0',upRunning=false,upLease=null,pollController=null;
|
let upSequence=1,downCursor='0',upRunning=false,upLease=null,httpCarrier=null,upWindow=1;
|
||||||
let helloFrame=null,helloTimer=null,welcomeSent=false,carrierAttempt=1,carrierFailure='',carrierCommitted=false,terminalFailure='';
|
let helloFrame=null,helloTimer=null,welcomeSent=false,carrierAttempt=1,carrierFailure='',carrierCommitted=false,terminalFailure='';
|
||||||
let negotiationStartedAt=0,carrierTimer=null,probeTimer=null,attemptController=null,attemptEpoch=1,candidateRunning=false,switching=false,currentAttempt=null;
|
let negotiationStartedAt=0,carrierTimer=null,probeTimer=null,attemptController=null,attemptEpoch=1,candidateRunning=false,switching=false,currentAttempt=null;
|
||||||
let recoveryController=null,recoveryCommit=null,recoveryReplaced=false,lastSchedulerWall=Date.now(),lastSchedulerMonotonic=performance.now(),schedulerGapPending=0,schedulerTimer=null;
|
let recoveryController=null,recoveryCommit=null,recoveryReplaced=false,lastSchedulerWall=Date.now(),lastSchedulerMonotonic=performance.now(),schedulerGapPending=0,schedulerTimer=null;
|
||||||
@@ -42,7 +44,7 @@ const {reserve,release,releasePending,frameBound,splitFrames,acceptNativeFrames,
|
|||||||
function detachLease(lease){if(lease.lane){if(lease.lane.upLease===lease)lease.lane.upLease=null}else if(upLease===lease)upLease=null}
|
function detachLease(lease){if(lease.lane){if(lease.lane.upLease===lease)lease.lane.upLease=null}else if(upLease===lease)upLease=null}
|
||||||
function settleBatch(lease){if(!buffers.settleBatch(lease))return false;detachLease(lease);return true}
|
function settleBatch(lease){if(!buffers.settleBatch(lease))return false;detachLease(lease);return true}
|
||||||
function cancelBatch(lease){if(!lease||lease.settled)return;buffers.cancelBatch(lease);detachLease(lease)}
|
function cancelBatch(lease){if(!lease||lease.settled)return;buffers.cancelBatch(lease);detachLease(lease)}
|
||||||
const attemptHeaders=(attempt,failure)=>negotiationEnabled?Object.assign({'X-Carrier-Capabilities':carrierCapabilities,'X-Carrier-Attempt':String(attempt)},failure?{'X-Carrier-Failure':failure}:{}):{};
|
const attemptHeaders=(attempt,failure)=>Object.assign({'X-Telemt-Up-Window':'4'},negotiationEnabled?Object.assign({'X-Carrier-Capabilities':carrierCapabilities,'X-Carrier-Attempt':String(attempt)},failure?{'X-Carrier-Failure':failure}:{}):{});
|
||||||
function finishOldRecovery(){
|
function finishOldRecovery(){
|
||||||
recoveryReplaced=false;resetScheduler();status('connected');
|
recoveryReplaced=false;resetScheduler();status('connected');
|
||||||
while(recoveryPending.length&&!closed){const data=recoveryPending.shift();release(data.byteLength,1,null);queueCarrier(data)}
|
while(recoveryPending.length&&!closed){const data=recoveryPending.shift();release(data.byteLength,1,null);queueCarrier(data)}
|
||||||
@@ -56,8 +58,9 @@ function resolveRecoveryCommit(){
|
|||||||
commit.signal.removeEventListener('abort',commit.abort);commit.resolve();
|
commit.signal.removeEventListener('abort',commit.abort);commit.resolve();
|
||||||
}
|
}
|
||||||
function retireCarrier(policy){
|
function retireCarrier(policy){
|
||||||
|
stopHttp(false);upWindow=1;
|
||||||
recoveryReplaced=true;attemptEpoch++;if(carrierTimer)clearTimeout(carrierTimer);carrierTimer=null;clearProbeTimer();
|
recoveryReplaced=true;attemptEpoch++;if(carrierTimer)clearTimeout(carrierTimer);carrierTimer=null;clearProbeTimer();
|
||||||
if(attemptController)attemptController.abort();attemptController=null;if(pollController)pollController.abort();pollController=null;
|
if(attemptController)attemptController.abort();attemptController=null;
|
||||||
if(socket){const previous=socket;socket=null;previous.close()}socketReady=false;cancelBatch(upLease);releasePending(upPending,null);
|
if(socket){const previous=socket;socket=null;previous.close()}socketReady=false;cancelBatch(upLease);releasePending(upPending,null);
|
||||||
for(const lane of lanes.values()){
|
for(const lane of lanes.values()){
|
||||||
if(lane.controller)lane.controller.abort();cancelBatch(lane.upLease);releasePending(lane.pending,lane);if(lane.socket)lane.socket.close();
|
if(lane.controller)lane.controller.abort();cancelBatch(lane.upLease);releasePending(lane.pending,lane);if(lane.socket)lane.socket.close();
|
||||||
@@ -116,9 +119,10 @@ function fail(reason){
|
|||||||
}
|
}
|
||||||
function knownCarrier(value){return value==='https'||value==='https-lanes'||value==='websocket'||value==='websocket-lanes'}
|
function knownCarrier(value){return value==='https'||value==='https-lanes'||value==='websocket'||value==='websocket-lanes'}
|
||||||
function sessionEcho(response,expectedAttempt,states,exactAttempt){
|
function sessionEcho(response,expectedAttempt,states,exactAttempt){
|
||||||
const selected=response.headers.get('X-Carrier-Mode')||'',echo=response.headers.get('X-Carrier-Attempt')||'';
|
const selected=response.headers.get('X-Carrier-Mode')||'',echo=response.headers.get('X-Carrier-Attempt')||'',windowHeader=response.headers.get('X-Telemt-Up-Window');
|
||||||
|
if(windowHeader!==null&&!/^[1-4]$/.test(windowHeader))throw new Error('invalid conveyor window');const upWindow=windowHeader===null?1:Number(windowHeader);
|
||||||
if(!knownCarrier(selected))throw new Error('invalid carrier mode');
|
if(!knownCarrier(selected))throw new Error('invalid carrier mode');
|
||||||
if(!negotiationEnabled){if(echo!=='')throw new Error('unexpected carrier attempt');return {selected,state:''}}
|
if(!negotiationEnabled){if(echo!=='')throw new Error('unexpected carrier attempt');return {selected,state:'',upWindow}}
|
||||||
const count=response.headers.get('X-Carrier-Candidate-Count')||'',deadline=response.headers.get('X-Carrier-Deadline')||'',state=response.headers.get('X-Carrier-State')||'';
|
const count=response.headers.get('X-Carrier-Candidate-Count')||'',deadline=response.headers.get('X-Carrier-Deadline')||'',state=response.headers.get('X-Carrier-State')||'';
|
||||||
if(!/^[1-4]$/.test(count)||!/^[1-9]\d*$/.test(deadline)||!states.includes(state))throw new Error('invalid carrier state');
|
if(!/^[1-4]$/.test(count)||!/^[1-9]\d*$/.test(deadline)||!states.includes(state))throw new Error('invalid carrier state');
|
||||||
const echoedAttempt=Number(echo),parsedCount=Number(count),parsedDeadline=Number(deadline);
|
const echoedAttempt=Number(echo),parsedCount=Number(count),parsedDeadline=Number(deadline);
|
||||||
@@ -127,7 +131,7 @@ function sessionEcho(response,expectedAttempt,states,exactAttempt){
|
|||||||
if(!negotiatedFrozen){negotiatedCandidateCount=parsedCount;negotiatedFinalDeadline=parsedDeadline;negotiatedFrozen=true}
|
if(!negotiatedFrozen){negotiatedCandidateCount=parsedCount;negotiatedFinalDeadline=parsedDeadline;negotiatedFrozen=true}
|
||||||
else if(parsedCount!==negotiatedCandidateCount||parsedDeadline!==negotiatedFinalDeadline)throw new Error('changed carrier bounds');
|
else if(parsedCount!==negotiatedCandidateCount||parsedDeadline!==negotiatedFinalDeadline)throw new Error('changed carrier bounds');
|
||||||
if(echoedAttempt>negotiatedCandidateCount)throw new Error('carrier attempt exceeds candidates');
|
if(echoedAttempt>negotiatedCandidateCount)throw new Error('carrier attempt exceeds candidates');
|
||||||
return {selected,state};
|
return {selected,state,upWindow};
|
||||||
}
|
}
|
||||||
function armCarrierDeadline(epoch){
|
function armCarrierDeadline(epoch){
|
||||||
if(!negotiationStartedAt||epoch!==attemptEpoch)return;
|
if(!negotiationStartedAt||epoch!==attemptEpoch)return;
|
||||||
@@ -138,8 +142,8 @@ function armCarrierDeadline(epoch){
|
|||||||
}
|
}
|
||||||
function clearProbeTimer(){if(probeTimer){clearTimeout(probeTimer.timer);probeTimer=null}}
|
function clearProbeTimer(){if(probeTimer){clearTimeout(probeTimer.timer);probeTimer=null}}
|
||||||
function resetCandidate(){
|
function resetCandidate(){
|
||||||
|
stopHttp(true);upWindow=1;
|
||||||
clearProbeTimer();
|
clearProbeTimer();
|
||||||
if(pollController)pollController.abort();pollController=null;
|
|
||||||
if(socket){const previous=socket;socket=null;previous.close()}socketReady=false;
|
if(socket){const previous=socket;socket=null;previous.close()}socketReady=false;
|
||||||
cancelBatch(upLease);releasePending(upPending,null);
|
cancelBatch(upLease);releasePending(upPending,null);
|
||||||
for(const lane of lanes.values()){
|
for(const lane of lanes.values()){
|
||||||
@@ -196,8 +200,8 @@ function startCandidate(probe,epoch){
|
|||||||
else advanceCarrier('protocol',epoch);
|
else advanceCarrier('protocol',epoch);
|
||||||
}
|
}
|
||||||
function maybeStartCandidate(){
|
function maybeStartCandidate(){
|
||||||
if(closed||carrierCommitted||!sessionToken||candidateRunning)return;const epoch=attemptEpoch;
|
if(closed||carrierCommitted||!sessionToken)return;if(candidateRunning){try{flushHttpPending()}catch(error){fail(failureReason(error,'protocol'))}return}const epoch=attemptEpoch;
|
||||||
let probe;try{probe=findProbe(probeCoalesceMs>0)}catch(error){fail('protocol');return}if(!probe)return;
|
let probe;try{probe=findProbe(true)}catch(error){fail('protocol');return}if(!probe)return;
|
||||||
if(!probeCoalesceMs||probe.hasData){startCandidate(probe,epoch);return}
|
if(!probeCoalesceMs||probe.hasData){startCandidate(probe,epoch);return}
|
||||||
if(probeTimer)return;const owner={epoch,timer:null};
|
if(probeTimer)return;const owner={epoch,timer:null};
|
||||||
owner.timer=setTimeout(()=>{if(probeTimer!==owner||closed||owner.epoch!==attemptEpoch)return;probeTimer=null;let current;try{current=findProbe(false)}catch(error){fail('protocol');return}startCandidate(current,owner.epoch)},probeCoalesceMs);
|
owner.timer=setTimeout(()=>{if(probeTimer!==owner||closed||owner.epoch!==attemptEpoch)return;probeTimer=null;let current;try{current=findProbe(false)}catch(error){fail('protocol');return}startCandidate(current,owner.epoch)},probeCoalesceMs);
|
||||||
@@ -217,74 +221,58 @@ async function createSession(epoch){
|
|||||||
const token=response.headers.get('X-Session-Token')||'',cursor=response.headers.get('X-Down-Cursor')||'';
|
const token=response.headers.get('X-Session-Token')||'',cursor=response.headers.get('X-Down-Cursor')||'';
|
||||||
if(!token||cursor!=='0'){advanceCarrier('protocol',epoch);return}
|
if(!token||cursor!=='0'){advanceCarrier('protocol',epoch);return}
|
||||||
const welcome=response.body;if(closed||epoch!==attemptEpoch)return;
|
const welcome=response.body;if(closed||epoch!==attemptEpoch)return;
|
||||||
carrier=selected;sessionToken=token;cleanupToken=token;downCursor=cursor;
|
carrier=selected;sessionToken=token;cleanupToken=token;downCursor=cursor;upWindow=echo.upWindow;
|
||||||
if(!welcomeSent){welcomeSent=true;port.postMessage(welcome,[welcome]);status('connecting')}
|
if(!welcomeSent){welcomeSent=true;port.postMessage(welcome,[welcome]);status('connecting')}
|
||||||
if(carrier==='websocket')openCandidateSocket(null,null,epoch);
|
if(carrier==='websocket')openCandidateSocket(null,null,epoch);
|
||||||
maybeStartCandidate();
|
maybeStartCandidate();
|
||||||
}catch(error){if(closed||epoch!==attemptEpoch)return;advanceCarrier(failureReason(error,'network'),epoch)}
|
}catch(error){if(closed||epoch!==attemptEpoch)return;advanceCarrier(failureReason(error,'network'),epoch)}
|
||||||
}
|
}
|
||||||
async function probeHttp(probe,laneID,epoch){
|
function stopHttp(restore){
|
||||||
try{
|
if(!httpCarrier)return;const saved=httpCarrier.close(restore);httpCarrier=null;
|
||||||
const headers={'X-Up-Seq':'1'},token=sessionToken,controller=attemptController,body=probe.data;if(laneID!==null)headers['X-Lane-ID']=String(laneID);
|
for(const data of saved){if(!reserve(data,null)){fail('capacity');return}pending.push(data)}
|
||||||
const response=await request('/api/v1/up',options('POST',token,body,headers,controller.signal));
|
}
|
||||||
if(closed||epoch!==attemptEpoch)return
|
function flushHttpPending(){
|
||||||
if(response.status!==204){advanceCarrier('http',epoch);return}
|
while(httpCarrier&&pending.length&&!closed){const data=pending.shift();release(data.byteLength,1,null);httpCarrier.enqueue(data,true)}
|
||||||
if(response.headers.get('X-Up-Ack')!=='1'){advanceCarrier('protocol',epoch);return}
|
if(httpCarrier&&!closed)httpCarrier.flush();
|
||||||
if(laneID===null)upSequence=2;else ensureLane(laneID).sequence=2;
|
}
|
||||||
commitCarrier(probe,epoch);
|
function probeHttp(probe,laneID,epoch){
|
||||||
}catch(error){if(!closed&&epoch===attemptEpoch)advanceCarrier(failureReason(error,'network'),epoch)}
|
const token=sessionToken;
|
||||||
|
httpCarrier=globalThis.TelemtBridgeConveyor.create({
|
||||||
|
token,window:upWindow,method:carrierMethod,lanes:carrier==='https-lanes',buffers,options,request,
|
||||||
|
queueLimit:()=>queueLimit,batchLimit:()=>batchLimit,committed:()=>carrierCommitted,failure,cancel:responseBody.cancel,
|
||||||
|
alive:lane=>!closed&&epoch===attemptEpoch&&token===sessionToken&&(!lane||lanes.get(lane.id)===lane),
|
||||||
|
commit:()=>commitCarrier(null,epoch),recover:recoverTransport,connected:()=>status('connected'),
|
||||||
|
traffic:(up,down)=>port.postMessage({t:'traffic',up,down}),
|
||||||
|
deliver:data=>{observeServerFrames(data);port.postMessage({t:'traffic',up:0,down:data.byteLength});port.postMessage(data,[data])},
|
||||||
|
lane:(id,type)=>{
|
||||||
|
let lane=lanes.get(id);if(!lane&&(type===2||type===3||type===4))return null;
|
||||||
|
if(!lane&&closedLanes.has(id))throw failure('protocol','closed lane reused');
|
||||||
|
if(!lane&&id!==0&&type!==1)throw failure('protocol','lane did not begin with OPEN');
|
||||||
|
return lane||ensureLane(id);
|
||||||
|
},
|
||||||
|
finish:finishLane,failed:error=>{if(closed||epoch!==attemptEpoch)return;if(carrierCommitted)fail(failureReason(error,'network'));else advanceCarrier(failureReason(error,'network'),epoch)}
|
||||||
|
});
|
||||||
|
try{flushHttpPending()}catch(error){fail(failureReason(error,'protocol'))}
|
||||||
}
|
}
|
||||||
function commitCarrier(probe,epoch){
|
function commitCarrier(probe,epoch){
|
||||||
if(closed||carrierCommitted||epoch!==attemptEpoch)return;
|
if(closed||carrierCommitted||epoch!==attemptEpoch)return;
|
||||||
if(switching){fail('protocol');return}
|
if(switching){fail('protocol');return}
|
||||||
clearProbeTimer();try{consumeProbe(probe)}catch(error){fail('protocol');return}
|
clearProbeTimer();try{if(probe)consumeProbe(probe)}catch(error){fail('protocol');return}
|
||||||
carrierCommitted=true;candidateRunning=false;if(carrierTimer)clearTimeout(carrierTimer);carrierTimer=null;
|
carrierCommitted=true;candidateRunning=false;if(carrierTimer)clearTimeout(carrierTimer);carrierTimer=null;
|
||||||
resetScheduler();
|
resetScheduler();
|
||||||
attemptController=null;currentAttempt=null;
|
attemptController=null;currentAttempt=null;
|
||||||
status('connected');
|
status('connected');
|
||||||
if(carrier==='https')poll();
|
|
||||||
else if(carrier==='https-lanes'){const lane=lanes.get(probe.id);if(lane&&!lane.polling)pollLane(lane)}
|
|
||||||
while(pending.length&&!closed){const data=pending.shift();release(data.byteLength,1,null);queueCarrier(data)}
|
while(pending.length&&!closed){const data=pending.shift();release(data.byteLength,1,null);queueCarrier(data)}
|
||||||
resolveRecoveryCommit();
|
resolveRecoveryCommit();
|
||||||
}
|
}
|
||||||
function queueCarrier(data){
|
function queueCarrier(data){
|
||||||
if(closed)return;
|
if(closed)return;
|
||||||
try{
|
try{
|
||||||
if(carrier==='https')queueUp(data);
|
if(carrier==='https'||carrier==='https-lanes')httpCarrier.enqueue(data);
|
||||||
else if(carrier==='websocket')queueSocket(data);
|
else if(carrier==='websocket')queueSocket(data);
|
||||||
else for(const value of splitFrames(data)){if(closed)break;queueLane(value)}
|
else for(const value of splitFrames(data)){if(closed)break;queueLane(value)}
|
||||||
}catch(error){fail('protocol')}
|
}catch(error){fail('protocol')}
|
||||||
}
|
}
|
||||||
function queueUp(data){if(!reserve(data,null)){fail('capacity');return}upPending.push(data);runUp()}
|
|
||||||
async function runUp(){
|
|
||||||
if(upRunning)return;upRunning=true;const ownerEpoch=attemptEpoch;let lease=null;
|
|
||||||
try{
|
|
||||||
while(!closed&&sessionToken&&upPending.length){
|
|
||||||
lease=takeBatch(upPending,null);upLease=lease;lease.controller=new AbortController();const sequence=String(upSequence),token=sessionToken;
|
|
||||||
for(;;){
|
|
||||||
try{
|
|
||||||
const response=await request('/api/v1/up',options('POST',token,lease.body,{'X-Up-Seq':sequence},lease.controller.signal),null,1);
|
|
||||||
if(response.status!==204)throw failure('http','uplink rejected');
|
|
||||||
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','uplink acknowledgement rejected');
|
|
||||||
break;
|
|
||||||
}catch(error){
|
|
||||||
let replayed=false;
|
|
||||||
const recovered=await recoverTransport(error,async(signal,remaining)=>{
|
|
||||||
const response=await request('/api/v1/up',options('POST',token,lease.body,{'X-Up-Seq':sequence},signal),remaining,2);
|
|
||||||
if(response.status!==204)throw failure('http','uplink replay rejected');
|
|
||||||
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','uplink replay acknowledgement rejected');
|
|
||||||
replayed=true;
|
|
||||||
});
|
|
||||||
if(!recovered||closed||lease.cancelled||sessionToken!==token)return;
|
|
||||||
if(!replayed)continue;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if(!settleBatch(lease))return;port.postMessage({t:'traffic',up:lease.total,down:0});upSequence++;lease=null;
|
|
||||||
}
|
|
||||||
}catch(error){if(!closed&&!(lease&&lease.cancelled))fail(failureReason(error,'network'))}
|
|
||||||
finally{if(ownerEpoch===attemptEpoch){upRunning=false;if(!closed&&sessionToken&&upPending.length)runUp()}}
|
|
||||||
}
|
|
||||||
function sendCandidateSocket(next){
|
function sendCandidateSocket(next){
|
||||||
const state=next.telemt;if(!state||state.sent||next.readyState!==WebSocket.OPEN||!state.probe)return;
|
const state=next.telemt;if(!state||state.sent||next.readyState!==WebSocket.OPEN||!state.probe)return;
|
||||||
let probe=state.probe;
|
let probe=state.probe;
|
||||||
@@ -338,30 +326,6 @@ async function runSocketUp(){
|
|||||||
}catch(error){if(!closed&&!(lease&&lease.cancelled))recoverTransport(error,null)}
|
}catch(error){if(!closed&&!(lease&&lease.cancelled))recoverTransport(error,null)}
|
||||||
finally{if(ownerEpoch===attemptEpoch){upRunning=false;if(!closed&&socketReady&&upPending.length)runSocketUp()}}
|
finally{if(ownerEpoch===attemptEpoch){upRunning=false;if(!closed&&socketReady&&upPending.length)runSocketUp()}}
|
||||||
}
|
}
|
||||||
async function poll(){
|
|
||||||
while(!closed&&sessionToken){
|
|
||||||
const token=sessionToken,cursor=downCursor;
|
|
||||||
try{
|
|
||||||
pollController=new AbortController();
|
|
||||||
const response=await request('/api/v1/down',options('POST',token,null,{'X-Down-Cursor':cursor},pollController.signal),null,1);
|
|
||||||
if(closed||sessionToken!==token)return;
|
|
||||||
if(response.status===204){status('connected');continue}
|
|
||||||
if(response.status!==200)throw failure('http','downlink rejected');
|
|
||||||
const next=response.headers.get('X-Down-Cursor')||'',data=response.body;
|
|
||||||
if(!next||!data.byteLength)throw failure('protocol','invalid downlink response');
|
|
||||||
if(closed)return;
|
|
||||||
observeServerFrames(data);port.postMessage({t:'traffic',up:0,down:data.byteLength});port.postMessage(data,[data]);downCursor=next;status('connected');
|
|
||||||
}catch(error){
|
|
||||||
if(closed)return;
|
|
||||||
const recovered=await recoverTransport(error,async(signal,remaining)=>{
|
|
||||||
const response=await request('/api/v1/down',options('POST',token,null,{'X-Down-Cursor':cursor},signal),remaining,2);
|
|
||||||
if(response.status===204)return;
|
|
||||||
if(response.status!==200||!response.body.byteLength||!response.headers.get('X-Down-Cursor'))throw failure('http','downlink replay rejected');
|
|
||||||
});
|
|
||||||
if(!recovered||closed||sessionToken!==token)return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
function ensureLane(id){
|
function ensureLane(id){
|
||||||
let lane=lanes.get(id);
|
let lane=lanes.get(id);
|
||||||
if(!lane){lane={id,sequence:1,cursor:'0',pending:[],bytes:0,items:0,running:false,upLease:null,polling:false,controller:null,socket:null,ready:false,remoteClosed:false};lanes.set(id,lane)}
|
if(!lane){lane={id,sequence:1,cursor:'0',pending:[],bytes:0,items:0,running:false,upLease:null,polling:false,controller:null,socket:null,ready:false,remoteClosed:false};lanes.set(id,lane)}
|
||||||
@@ -373,7 +337,7 @@ function rememberLaneClosed(id){
|
|||||||
closedLanes.add(id);closedLaneOrder.push(id);
|
closedLanes.add(id);closedLaneOrder.push(id);
|
||||||
}
|
}
|
||||||
function finishLane(lane,notifyClient){
|
function finishLane(lane,notifyClient){
|
||||||
if(lanes.get(lane.id)!==lane)return;
|
if(lanes.get(lane.id)!==lane)return;if(httpCarrier)httpCarrier.finish(lane);
|
||||||
if(lane.controller)lane.controller.abort();lane.controller=null;cancelBatch(lane.upLease);
|
if(lane.controller)lane.controller.abort();lane.controller=null;cancelBatch(lane.upLease);
|
||||||
if(lane.socket&&lane.socket.readyState<WebSocket.CLOSING)lane.socket.close();
|
if(lane.socket&&lane.socket.readyState<WebSocket.CLOSING)lane.socket.close();
|
||||||
releasePending(lane.pending,lane);lanes.delete(lane.id);rememberLaneClosed(lane.id);
|
releasePending(lane.pending,lane);lanes.delete(lane.id);rememberLaneClosed(lane.id);
|
||||||
@@ -387,7 +351,7 @@ function queueLane(value){
|
|||||||
lane=lane||ensureLane(value.id);
|
lane=lane||ensureLane(value.id);
|
||||||
if(!reserve(value.data,lane)){fail('capacity');return}
|
if(!reserve(value.data,lane)){fail('capacity');return}
|
||||||
lane.pending.push(value.data);
|
lane.pending.push(value.data);
|
||||||
if(carrier==='websocket-lanes'){openLaneSocket(lane);runLaneSocketUp(lane)}else runLaneUp(lane);
|
openLaneSocket(lane);runLaneSocketUp(lane);
|
||||||
}
|
}
|
||||||
function openLaneSocket(lane){
|
function openLaneSocket(lane){
|
||||||
if(lane.socket||closed)return;lane.socket=new WebSocket(socketURL(),'tproxy-lane-v1.'+sessionToken+'.'+String(lane.id));lane.socket.binaryType='arraybuffer';
|
if(lane.socket||closed)return;lane.socket=new WebSocket(socketURL(),'tproxy-lane-v1.'+sessionToken+'.'+String(lane.id));lane.socket.binaryType='arraybuffer';
|
||||||
@@ -414,74 +378,12 @@ async function runLaneSocketUp(lane){
|
|||||||
}catch(error){if(!closed&&lanes.get(lane.id)===lane&&!(lease&&lease.cancelled))finishLane(lane,true)}
|
}catch(error){if(!closed&&lanes.get(lane.id)===lane&&!(lease&&lease.cancelled))finishLane(lane,true)}
|
||||||
finally{lane.running=false;if(!closed&&lanes.get(lane.id)===lane&&lane.ready&&lane.pending.length)runLaneSocketUp(lane)}
|
finally{lane.running=false;if(!closed&&lanes.get(lane.id)===lane&&lane.ready&&lane.pending.length)runLaneSocketUp(lane)}
|
||||||
}
|
}
|
||||||
async function runLaneUp(lane){
|
|
||||||
if(lane.running)return;lane.running=true;let lease=null;
|
|
||||||
try{
|
|
||||||
while(!closed&&sessionToken&&lane.pending.length){
|
|
||||||
lease=takeBatch(lane.pending,lane);lane.upLease=lease;lease.controller=new AbortController();
|
|
||||||
const sequence=String(lane.sequence),laneID=String(lane.id),token=sessionToken;
|
|
||||||
for(;;){
|
|
||||||
try{
|
|
||||||
const response=await request('/api/v1/up',options('POST',token,lease.body,{'X-Up-Seq':sequence,'X-Lane-ID':laneID},lease.controller.signal),null,1);
|
|
||||||
if(response.status!==204)throw failure('http','lane uplink rejected');
|
|
||||||
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','lane uplink acknowledgement rejected');
|
|
||||||
break;
|
|
||||||
}catch(error){
|
|
||||||
let replayed=false;
|
|
||||||
const recovered=await recoverTransport(error,async(signal,remaining)=>{
|
|
||||||
const response=await request('/api/v1/up',options('POST',token,lease.body,{'X-Up-Seq':sequence,'X-Lane-ID':laneID},signal),remaining,2);
|
|
||||||
if(response.status!==204)throw failure('http','lane uplink replay rejected');
|
|
||||||
if(response.headers.get('X-Up-Ack')!==sequence)throw failure('protocol','lane uplink replay acknowledgement rejected');
|
|
||||||
replayed=true;
|
|
||||||
});
|
|
||||||
if(!recovered||closed||lease.cancelled||sessionToken!==token||lanes.get(lane.id)!==lane)return;
|
|
||||||
if(!replayed)continue;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if(!settleBatch(lease))return;port.postMessage({t:'traffic',up:lease.total,down:0});lane.sequence++;lease=null;
|
|
||||||
if(!lane.polling)pollLane(lane);
|
|
||||||
}
|
|
||||||
}catch(error){if(!closed&&lanes.get(lane.id)===lane&&!(lease&&lease.cancelled))fail(failureReason(error,'network'))}
|
|
||||||
finally{lane.running=false;if(!closed&&lanes.get(lane.id)===lane&&sessionToken&&lane.pending.length)runLaneUp(lane)}
|
|
||||||
}
|
|
||||||
async function pollLane(lane){
|
|
||||||
if(!lane||lane.polling)return;lane.polling=true;let restart=false,failedToken='',failedCursor='',failedLaneID='';
|
|
||||||
try{
|
|
||||||
while(!closed&&sessionToken&&lanes.get(lane.id)===lane){
|
|
||||||
const controller=new AbortController(),laneID=String(lane.id),token=sessionToken,cursor=lane.cursor;lane.controller=controller;
|
|
||||||
failedToken=token;failedCursor=cursor;failedLaneID=laneID;
|
|
||||||
const response=await request('/api/v1/down',options('POST',token,null,{'X-Down-Cursor':cursor,'X-Lane-ID':laneID},controller.signal),null,1);
|
|
||||||
if(closed||sessionToken!==token||lanes.get(lane.id)!==lane)return;
|
|
||||||
if(response.status===204){
|
|
||||||
if(response.headers.get('X-Lane-Closed')==='1'){finishLane(lane,false);return}
|
|
||||||
status('connected');continue;
|
|
||||||
}
|
|
||||||
if(response.status!==200)throw failure('http','lane downlink rejected');
|
|
||||||
const next=response.headers.get('X-Down-Cursor')||'',data=response.body;
|
|
||||||
if(!next||!data.byteLength)throw failure('protocol','invalid lane downlink response');
|
|
||||||
for(const value of splitFrames(data))if(value.id!==lane.id)throw new Error('cross-lane frame');
|
|
||||||
if(closed)return;
|
|
||||||
observeServerFrames(data);port.postMessage({t:'traffic',up:0,down:data.byteLength});port.postMessage(data,[data]);lane.cursor=next;status('connected');
|
|
||||||
}
|
|
||||||
}catch(error){
|
|
||||||
if(!closed&&lanes.get(lane.id)===lane){
|
|
||||||
const recovered=await recoverTransport(error,async(signal,remaining)=>{
|
|
||||||
const response=await request('/api/v1/down',options('POST',failedToken,null,{'X-Down-Cursor':failedCursor,'X-Lane-ID':failedLaneID},signal),remaining,2);
|
|
||||||
if(response.status===204)return;
|
|
||||||
if(response.status!==200||!response.body.byteLength||!response.headers.get('X-Down-Cursor'))throw failure('http','lane downlink replay rejected');
|
|
||||||
});
|
|
||||||
restart=recovered&&!closed&&sessionToken===failedToken&&lanes.get(lane.id)===lane;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
finally{lane.polling=false;lane.controller=null;if(restart)pollLane(lane)}
|
|
||||||
}
|
|
||||||
function deleteSession(){
|
function deleteSession(){
|
||||||
const token=cleanupToken||sessionToken,headers=canonicalFailures.includes(terminalFailure)?{'X-Carrier-Failure':terminalFailure}:null;
|
const token=cleanupToken||sessionToken,headers=canonicalFailures.includes(terminalFailure)?{'X-Carrier-Failure':terminalFailure}:null;
|
||||||
if(token)fetch(relayBase+'/api/v1/session',options('DELETE',token,null,headers,undefined,true)).catch(()=>{});
|
if(token)fetch(relayBase+'/api/v1/session',options('DELETE',token,null,headers,undefined,true)).catch(()=>{});
|
||||||
}
|
}
|
||||||
function close(notifyServer){
|
function close(notifyServer){
|
||||||
if(closed)return;closed=true;if(recoveryController)recoveryController.cancel();rejectRecoveryCommit(failure('network','bridge closed'));if(helloTimer)clearTimeout(helloTimer);helloTimer=null;if(carrierTimer)clearTimeout(carrierTimer);clearProbeTimer();if(schedulerTimer)clearTimeout(schedulerTimer);schedulerTimer=null;if(attemptController)attemptController.abort();if(pollController)pollController.abort();
|
if(closed)return;closed=true;stopHttp(false);if(recoveryController)recoveryController.cancel();rejectRecoveryCommit(failure('network','bridge closed'));if(helloTimer)clearTimeout(helloTimer);helloTimer=null;if(carrierTimer)clearTimeout(carrierTimer);clearProbeTimer();if(schedulerTimer)clearTimeout(schedulerTimer);schedulerTimer=null;if(attemptController)attemptController.abort();
|
||||||
if(socket)socket.close();cancelBatch(upLease);releasePending(upPending,null);
|
if(socket)socket.close();cancelBatch(upLease);releasePending(upPending,null);
|
||||||
for(const lane of lanes.values()){
|
for(const lane of lanes.values()){
|
||||||
if(lane.controller)lane.controller.abort();cancelBatch(lane.upLease);releasePending(lane.pending,lane);if(lane.socket)lane.socket.close();
|
if(lane.controller)lane.controller.abort();cancelBatch(lane.upLease);releasePending(lane.pending,lane);if(lane.socket)lane.socket.close();
|
||||||
|
|||||||
+67
-2
@@ -1,5 +1,64 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn carrier_method_is_page_owned_and_used_by_every_https_request() {
|
||||||
|
for method in [WebCarrierMethod::Post, WebCarrierMethod::Put] {
|
||||||
|
let page = render(
|
||||||
|
"proxy.example.com",
|
||||||
|
"/telegram/web/",
|
||||||
|
"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||||
|
2 * 1024 * 1024,
|
||||||
|
32 * 1024 * 1024,
|
||||||
|
16 * 1024,
|
||||||
|
1024,
|
||||||
|
true,
|
||||||
|
4,
|
||||||
|
[3, 5, 8, 12],
|
||||||
|
25,
|
||||||
|
10,
|
||||||
|
90,
|
||||||
|
15,
|
||||||
|
15,
|
||||||
|
120,
|
||||||
|
0,
|
||||||
|
true,
|
||||||
|
method,
|
||||||
|
&SecureRandom::new(),
|
||||||
|
);
|
||||||
|
assert!(!page.body.contains("__"));
|
||||||
|
assert!(
|
||||||
|
page.body
|
||||||
|
.contains(&format!("const carrierMethod='{}';", method.as_str()))
|
||||||
|
);
|
||||||
|
assert_eq!(page.body.matches("carrierMethod=").count(), 1);
|
||||||
|
// Execute the rendered POST/PUT page, including retries, instead of counting call sites.
|
||||||
|
behavior_tests::run(&page);
|
||||||
|
assert_eq!(page.body.matches("options('POST',bootstrap,").count(), 2);
|
||||||
|
assert!(
|
||||||
|
page.body
|
||||||
|
.contains("fetch(relayBase+'/api/v1/diagnostic',{method:'POST'")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
page.body
|
||||||
|
.contains("options('DELETE',token,null,headers,undefined,true)")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
page.body
|
||||||
|
.contains("method:'GET',signal:requestController.signal")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
page.body
|
||||||
|
.contains("exactKeys(value,['v','bootstrap','limits','timeouts','negotiation'])")
|
||||||
|
);
|
||||||
|
assert!(!page.body.contains("policy.carrier_method"));
|
||||||
|
assert!(page.body.contains("port.postMessage({t:'status',state})"));
|
||||||
|
assert!(!page.body.contains("port.postMessage({t:'status',state,"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[path = "behavior_tests.rs"]
|
||||||
|
mod behavior_tests;
|
||||||
|
|
||||||
fn render_page(bootstrap: &str, candidate_count: usize) -> BridgePage {
|
fn render_page(bootstrap: &str, candidate_count: usize) -> BridgePage {
|
||||||
render(
|
render(
|
||||||
"proxy.example.com",
|
"proxy.example.com",
|
||||||
@@ -20,6 +79,7 @@ fn render_page(bootstrap: &str, candidate_count: usize) -> BridgePage {
|
|||||||
120,
|
120,
|
||||||
0,
|
0,
|
||||||
false,
|
false,
|
||||||
|
WebCarrierMethod::Post,
|
||||||
&SecureRandom::new(),
|
&SecureRandom::new(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -44,6 +104,7 @@ fn render_diagnostic_page(bootstrap: &str) -> BridgePage {
|
|||||||
120,
|
120,
|
||||||
0,
|
0,
|
||||||
true,
|
true,
|
||||||
|
WebCarrierMethod::Post,
|
||||||
&SecureRandom::new(),
|
&SecureRandom::new(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -95,6 +156,7 @@ fn rendered_page_resolves_carriers_against_the_exact_base_path() {
|
|||||||
120,
|
120,
|
||||||
0,
|
0,
|
||||||
true,
|
true,
|
||||||
|
WebCarrierMethod::Post,
|
||||||
&SecureRandom::new(),
|
&SecureRandom::new(),
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -148,6 +210,7 @@ fn rendered_page_embeds_the_configured_bridge_timing_policy() {
|
|||||||
119,
|
119,
|
||||||
4,
|
4,
|
||||||
false,
|
false,
|
||||||
|
WebCarrierMethod::Post,
|
||||||
&SecureRandom::new(),
|
&SecureRandom::new(),
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -201,6 +264,7 @@ fn disabled_negotiation_does_not_arm_a_carrier_deadline() {
|
|||||||
120,
|
120,
|
||||||
0,
|
0,
|
||||||
false,
|
false,
|
||||||
|
WebCarrierMethod::Post,
|
||||||
&SecureRandom::new(),
|
&SecureRandom::new(),
|
||||||
);
|
);
|
||||||
assert!(page.body.contains(
|
assert!(page.body.contains(
|
||||||
@@ -216,8 +280,9 @@ fn disabled_negotiation_does_not_arm_a_carrier_deadline() {
|
|||||||
fn retry_and_attempt_state_are_frozen_before_fetch() {
|
fn retry_and_attempt_state_are_frozen_before_fetch() {
|
||||||
let page = render_page("EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE", 4);
|
let page = render_page("EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE", 4);
|
||||||
assert!(
|
assert!(
|
||||||
page.body
|
page.body.contains(
|
||||||
.contains("async function send(path,frozenOptions,remainingBudget,maxAttempts)")
|
"async function send(path,frozenOptions,remainingBudget,maxAttempts,receiver)"
|
||||||
|
)
|
||||||
);
|
);
|
||||||
assert!(!page.body.contains("makeOptions"));
|
assert!(!page.body.contains("makeOptions"));
|
||||||
assert!(page.body.contains(
|
assert!(page.body.contains(
|
||||||
|
|||||||
+5
-72
@@ -36,6 +36,8 @@ mod decoy;
|
|||||||
mod diagnostic;
|
mod diagnostic;
|
||||||
// Downlink long-poll handling remains isolated from request routing.
|
// Downlink long-poll handling remains isolated from request routing.
|
||||||
mod down;
|
mod down;
|
||||||
|
// Uplink admission and conveyor waits own their bounded request bodies.
|
||||||
|
mod up;
|
||||||
// Canonical request parsing rejects ambiguous credentials before routing.
|
// Canonical request parsing rejects ambiguous credentials before routing.
|
||||||
mod request;
|
mod request;
|
||||||
// Positive-only recovery representation stays separate from ordinary bridge rendering.
|
// Positive-only recovery representation stays separate from ordinary bridge rendering.
|
||||||
@@ -44,9 +46,9 @@ mod recovery;
|
|||||||
mod response;
|
mod response;
|
||||||
// Session creation and replacement negotiation remain separate from request routing.
|
// Session creation and replacement negotiation remain separate from request routing.
|
||||||
mod session;
|
mod session;
|
||||||
// RFC 6455 upgrade validation and carrier drivers remain isolated from HTTP routing.
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests;
|
mod tests;
|
||||||
|
// RFC 6455 upgrade validation and carrier drivers remain isolated from HTTP routing.
|
||||||
mod websocket;
|
mod websocket;
|
||||||
// Enabled-debug integration coverage remains separate from carrier behavior tests.
|
// Enabled-debug integration coverage remains separate from carrier behavior tests.
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -68,6 +70,7 @@ use response::{
|
|||||||
insert_header, service_unavailable,
|
insert_header, service_unavailable,
|
||||||
};
|
};
|
||||||
use session::handle_session;
|
use session::handle_session;
|
||||||
|
use up::handle_up;
|
||||||
|
|
||||||
type BoxError = Box<dyn Error + Send + Sync>;
|
type BoxError = Box<dyn Error + Send + Sync>;
|
||||||
type HttpBody = UnsyncBoxBody<Bytes, BoxError>;
|
type HttpBody = UnsyncBoxBody<Bytes, BoxError>;
|
||||||
@@ -389,6 +392,7 @@ async fn handle_root(
|
|||||||
config.web.timeouts.reconnect_grace_secs,
|
config.web.timeouts.reconnect_grace_secs,
|
||||||
config.web.timeouts.carrier_probe_coalesce_ms,
|
config.web.timeouts.carrier_probe_coalesce_ms,
|
||||||
config.web.debug.bridge_diagnostics_enabled(),
|
config.web.debug.bridge_diagnostics_enabled(),
|
||||||
|
config.web.carrier_method,
|
||||||
&generation.rng,
|
&generation.rng,
|
||||||
);
|
);
|
||||||
let mut response = full_response(StatusCode::OK, Bytes::from(page.body));
|
let mut response = full_response(StatusCode::OK, Bytes::from(page.body));
|
||||||
@@ -456,77 +460,6 @@ async fn handle_api(
|
|||||||
_ => serve_decoy(request, vhost, true, &runtime).await,
|
_ => serve_decoy(request, vhost, true, &runtime).await,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_up(
|
|
||||||
request: Request<RequestBody>,
|
|
||||||
runtime: Arc<WebProcessRuntime>,
|
|
||||||
vhost: Arc<WebRuntimeVhost>,
|
|
||||||
token_hash: crate::web::manager::TokenHash,
|
|
||||||
) -> HttpResponse {
|
|
||||||
if request.method() != Method::POST || !binary_content_type(&request) {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
}
|
|
||||||
let Some(sequence) = canonical_u64_header(&request, "x-up-seq").filter(|value| *value != 0)
|
|
||||||
else {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
};
|
|
||||||
let Ok(session) = runtime.get_session(token_hash, &vhost.host) else {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
};
|
|
||||||
if session.carrier().uses_websocket() {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
}
|
|
||||||
if let Some(trace) = request_trace(&request) {
|
|
||||||
trace.set_route(TraceRoute::Uplink);
|
|
||||||
trace.bind_identity(session.trace_identity());
|
|
||||||
}
|
|
||||||
let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
};
|
|
||||||
let limit = session.limits().max_body_bytes;
|
|
||||||
let CollectedBody {
|
|
||||||
request,
|
|
||||||
body,
|
|
||||||
_body_budget,
|
|
||||||
} = match collect_body(
|
|
||||||
request,
|
|
||||||
&runtime,
|
|
||||||
Duration::from_secs(session.timeouts().body_secs),
|
|
||||||
limit,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(result) => result,
|
|
||||||
Err(CollectBodyError::Limit) => return service_unavailable(),
|
|
||||||
Err(CollectBodyError::Invalid(request)) => {
|
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if let Some(trace) = request_trace(&request) {
|
|
||||||
trace.record_frames(TraceDirection::Request, &body, session.limits());
|
|
||||||
}
|
|
||||||
let result = match lane_id {
|
|
||||||
Some(lane_id) => session.process_up_lane(lane_id, sequence, &body),
|
|
||||||
None => session.process_up(sequence, &body),
|
|
||||||
};
|
|
||||||
match result {
|
|
||||||
Ok(ack) => {
|
|
||||||
let mut response = carrier_empty(StatusCode::NO_CONTENT);
|
|
||||||
insert_header(
|
|
||||||
&mut response,
|
|
||||||
HeaderName::from_static("x-up-ack"),
|
|
||||||
&ack.to_string(),
|
|
||||||
);
|
|
||||||
response
|
|
||||||
}
|
|
||||||
Err(ManagerError::Backpressure | ManagerError::Concurrent | ManagerError::Limit) => {
|
|
||||||
service_unavailable()
|
|
||||||
}
|
|
||||||
Err(_) => serve_decoy(request, vhost, true, &runtime).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn strip_query<B>(request: &mut Request<B>) {
|
fn strip_query<B>(request: &mut Request<B>) {
|
||||||
if request.uri().query().is_some()
|
if request.uri().query().is_some()
|
||||||
&& let Ok(uri) = request.uri().path().parse()
|
&& let Ok(uri) = request.uri().path().parse()
|
||||||
|
|||||||
@@ -0,0 +1,483 @@
|
|||||||
|
use super::super::session_policy_tests::{open_keepalive, read_http_response};
|
||||||
|
use super::super::*;
|
||||||
|
use crate::config::WebCarrierMethod;
|
||||||
|
|
||||||
|
#[path = "conveyor_tests.rs"]
|
||||||
|
mod conveyor_tests;
|
||||||
|
|
||||||
|
fn carrier_request(method: &str, path: &str, token: &str, headers: &str, body: &[u8]) -> Vec<u8> {
|
||||||
|
let mut bytes = format!(
|
||||||
|
"{method} {path} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.60\r\nAuthorization: Bearer {token}\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n",
|
||||||
|
body.len()
|
||||||
|
)
|
||||||
|
.into_bytes();
|
||||||
|
bytes.extend_from_slice(body);
|
||||||
|
bytes
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn bridge_page(
|
||||||
|
listener: &TcpListener,
|
||||||
|
runtime: &Arc<WebProcessRuntime>,
|
||||||
|
capability: [u8; 32],
|
||||||
|
base: &str,
|
||||||
|
) -> Vec<u8> {
|
||||||
|
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||||
|
request(
|
||||||
|
listener,
|
||||||
|
runtime,
|
||||||
|
format!(
|
||||||
|
"GET {base}?bridge={encoded} HTTP/1.1\r\nHost: proxy.example.com\r\nX-Forwarded-For: 192.0.2.60\r\nConnection: close\r\n\r\n"
|
||||||
|
)
|
||||||
|
.into_bytes(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bootstrap_from(page: &[u8]) -> &str {
|
||||||
|
let (headers, body) = split_response(page);
|
||||||
|
assert!(headers.starts_with(b"HTTP/1.1 200"));
|
||||||
|
std::str::from_utf8(body)
|
||||||
|
.unwrap()
|
||||||
|
.split_once("bootstrap=\"")
|
||||||
|
.unwrap()
|
||||||
|
.1
|
||||||
|
.split_once('"')
|
||||||
|
.unwrap()
|
||||||
|
.0
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn create_session(
|
||||||
|
listener: &TcpListener,
|
||||||
|
runtime: &Arc<WebProcessRuntime>,
|
||||||
|
bootstrap: &str,
|
||||||
|
base: &str,
|
||||||
|
) -> String {
|
||||||
|
let hello = frame::encode(FrameType::Hello, 0, &[1]);
|
||||||
|
let bytes = carrier_request(
|
||||||
|
"POST",
|
||||||
|
&format!("{base}api/v1/session"),
|
||||||
|
bootstrap,
|
||||||
|
"Content-Type: application/octet-stream\r\n",
|
||||||
|
&hello,
|
||||||
|
);
|
||||||
|
let response = request(listener, runtime, bytes).await;
|
||||||
|
let (headers, _) = split_response(&response);
|
||||||
|
assert!(headers.starts_with(b"HTTP/1.1 200"));
|
||||||
|
response_header(headers, "x-session-token").to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn session_token(
|
||||||
|
listener: &TcpListener,
|
||||||
|
runtime: &Arc<WebProcessRuntime>,
|
||||||
|
capability: [u8; 32],
|
||||||
|
base: &str,
|
||||||
|
) -> String {
|
||||||
|
let page = bridge_page(listener, runtime, capability, base).await;
|
||||||
|
create_session(listener, runtime, bootstrap_from(&page), base).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn assert_put_accepted(path: &str) {
|
||||||
|
for (carrier, base, method) in [WebCarrier::Https, WebCarrier::HttpsLanes]
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|carrier| {
|
||||||
|
["/", "/telegram/web/"].into_iter().flat_map(move |base| {
|
||||||
|
[WebCarrierMethod::Post, WebCarrierMethod::Put]
|
||||||
|
.map(|method| (carrier, base, method))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
{
|
||||||
|
let capability = [60; 32];
|
||||||
|
let mut config = runtime_config_with_base(capability, carrier, base);
|
||||||
|
config.web.carrier_method = method;
|
||||||
|
config.web.timeouts.long_poll_secs = 0;
|
||||||
|
let generation = test_runtime_generation(1, config);
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let token = session_token(&listener, &runtime, capability, base).await;
|
||||||
|
let lane = if carrier.uses_lanes() {
|
||||||
|
"X-Lane-ID: 0\r\n"
|
||||||
|
} else {
|
||||||
|
""
|
||||||
|
};
|
||||||
|
let pong = frame::encode(FrameType::Pong, 0, &[]);
|
||||||
|
let (headers, body) = if path.ends_with("/up") {
|
||||||
|
(
|
||||||
|
format!("Content-Type: application/octet-stream\r\nX-Up-Seq: 1\r\n{lane}"),
|
||||||
|
pong.as_ref(),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
(format!("X-Down-Cursor: 0\r\n{lane}"), &[][..])
|
||||||
|
};
|
||||||
|
let path = format!("{base}{}", path.trim_start_matches('/'));
|
||||||
|
let bytes = carrier_request("PUT", &path, &token, &headers, body);
|
||||||
|
let response = request(&listener, &runtime, bytes).await;
|
||||||
|
assert!(
|
||||||
|
response.starts_with(b"HTTP/1.1 204"),
|
||||||
|
"{carrier:?}: {}",
|
||||||
|
String::from_utf8_lossy(&response)
|
||||||
|
);
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn put_uplink_is_accepted_for_both_https_carriers() {
|
||||||
|
assert_put_accepted("/api/v1/up").await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn put_downlink_is_accepted_for_both_https_carriers() {
|
||||||
|
assert_put_accepted("/api/v1/down").await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn carrier_method_mixed_retries_share_sequence_and_cursor_state() {
|
||||||
|
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||||
|
for method in [WebCarrierMethod::Post, WebCarrierMethod::Put] {
|
||||||
|
let capability = [61; 32];
|
||||||
|
let mut config = runtime_config(capability, carrier);
|
||||||
|
config.web.carrier_method = method;
|
||||||
|
config.web.timeouts.long_poll_secs = 0;
|
||||||
|
// Exhausted admission queues a deterministic CLOSE without starting a relay task.
|
||||||
|
config.web.limits.max_streams_global = 0;
|
||||||
|
let generation = test_runtime_generation(1, config);
|
||||||
|
let runtime =
|
||||||
|
WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let token = session_token(&listener, &runtime, capability, "/").await;
|
||||||
|
for (index, stream_id) in [7, 8].into_iter().enumerate() {
|
||||||
|
let lane = if carrier.uses_lanes() {
|
||||||
|
format!("X-Lane-ID: {stream_id}\r\n")
|
||||||
|
} else {
|
||||||
|
String::new()
|
||||||
|
};
|
||||||
|
let sequence = if carrier.uses_lanes() { 1 } else { index + 1 };
|
||||||
|
let cursor = if carrier.uses_lanes() { 0 } else { index };
|
||||||
|
let open = frame::encode(FrameType::Open, stream_id, &[]);
|
||||||
|
let up_headers = format!(
|
||||||
|
"Content-Type: application/octet-stream\r\nX-Up-Seq: {sequence}\r\n{lane}"
|
||||||
|
);
|
||||||
|
let methods = if index == 0 {
|
||||||
|
["POST", "PUT"]
|
||||||
|
} else {
|
||||||
|
["PUT", "POST"]
|
||||||
|
};
|
||||||
|
for verb in methods {
|
||||||
|
let bytes = carrier_request(verb, "/api/v1/up", &token, &up_headers, &open);
|
||||||
|
let response = request(&listener, &runtime, bytes).await;
|
||||||
|
let (headers, body) = split_response(&response);
|
||||||
|
assert!(headers.starts_with(b"HTTP/1.1 204"));
|
||||||
|
assert_eq!(response_header(headers, "x-up-ack"), sequence.to_string());
|
||||||
|
assert!(body.is_empty());
|
||||||
|
}
|
||||||
|
for verb in methods {
|
||||||
|
let headers = format!("X-Down-Cursor: {cursor}\r\n{lane}");
|
||||||
|
let bytes = carrier_request(verb, "/api/v1/down", &token, &headers, &[]);
|
||||||
|
let response = request(&listener, &runtime, bytes).await;
|
||||||
|
let (headers, body) = split_response(&response);
|
||||||
|
assert!(headers.starts_with(b"HTTP/1.1 200"));
|
||||||
|
assert_eq!(
|
||||||
|
response_header(headers, "x-down-cursor"),
|
||||||
|
(cursor + 1).to_string()
|
||||||
|
);
|
||||||
|
assert_eq!(body, frame::encode(FrameType::Close, stream_id, &[]));
|
||||||
|
}
|
||||||
|
let headers = format!("X-Down-Cursor: {}\r\n{lane}", cursor + 1);
|
||||||
|
let bytes = carrier_request("PUT", "/api/v1/down", &token, &headers, &[]);
|
||||||
|
let response = request(&listener, &runtime, bytes).await;
|
||||||
|
assert!(response.starts_with(b"HTTP/1.1 204"));
|
||||||
|
}
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn carrier_method_changed_body_retry_keeps_protocol_failure() {
|
||||||
|
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||||
|
let capability = [62; 32];
|
||||||
|
let generation = test_runtime_generation(1, runtime_config(capability, carrier));
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let token = session_token(&listener, &runtime, capability, "/").await;
|
||||||
|
let lane = if carrier.uses_lanes() {
|
||||||
|
"X-Lane-ID: 0\r\n"
|
||||||
|
} else {
|
||||||
|
""
|
||||||
|
};
|
||||||
|
let headers = format!("Content-Type: application/octet-stream\r\nX-Up-Seq: 1\r\n{lane}");
|
||||||
|
let pong = frame::encode(FrameType::Pong, 0, &[]);
|
||||||
|
let bytes = carrier_request("POST", "/api/v1/up", &token, &headers, &pong);
|
||||||
|
let response = request(&listener, &runtime, bytes).await;
|
||||||
|
assert!(response.starts_with(b"HTTP/1.1 204"));
|
||||||
|
let mut changed = pong.to_vec();
|
||||||
|
changed.extend_from_slice(&pong);
|
||||||
|
let bytes = carrier_request("PUT", "/api/v1/up", &token, &headers, &changed);
|
||||||
|
let response = request(&listener, &runtime, bytes).await;
|
||||||
|
assert_private_decoy(&response);
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_private_decoy(response: &[u8]) {
|
||||||
|
let (headers, body) = split_response(response);
|
||||||
|
assert!(headers.starts_with(b"HTTP/1.1 404"));
|
||||||
|
assert_eq!(response_header(headers, "cache-control"), "no-store");
|
||||||
|
assert_eq!(body, b"not found\n");
|
||||||
|
let headers = std::str::from_utf8(headers).unwrap().to_ascii_lowercase();
|
||||||
|
for name in ["x-up-ack:", "x-down-cursor:", "x-session-token:", "allow:"] {
|
||||||
|
assert!(!headers.contains(name));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn carrier_method_put_preserves_authenticated_request_shape_checks() {
|
||||||
|
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||||
|
let capability = [63; 32];
|
||||||
|
let base = "/telegram/web/";
|
||||||
|
let generation =
|
||||||
|
test_runtime_generation(1, runtime_config_with_base(capability, carrier, base));
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let token = session_token(&listener, &runtime, capability, base).await;
|
||||||
|
let pong = frame::encode(FrameType::Pong, 0, &[]);
|
||||||
|
let lane = if carrier.uses_lanes() {
|
||||||
|
"X-Lane-ID: 0\r\n"
|
||||||
|
} else {
|
||||||
|
""
|
||||||
|
};
|
||||||
|
let up = format!("{base}api/v1/up");
|
||||||
|
let down = format!("{base}api/v1/down");
|
||||||
|
let up_headers = format!("Content-Type: application/octet-stream\r\nX-Up-Seq: 1\r\n{lane}");
|
||||||
|
let down_headers = format!("X-Down-Cursor: 0\r\n{lane}");
|
||||||
|
let mut invalid = Vec::new();
|
||||||
|
for media in [
|
||||||
|
"",
|
||||||
|
"Content-Type: text/plain\r\n",
|
||||||
|
"Content-Type: application/octet-stream\r\nContent-Type: application/octet-stream\r\n",
|
||||||
|
] {
|
||||||
|
let headers = format!("{media}X-Up-Seq: 1\r\n{lane}");
|
||||||
|
invalid.push(carrier_request("PUT", &up, &token, &headers, &pong));
|
||||||
|
}
|
||||||
|
for sequence in ["0", "01", ""] {
|
||||||
|
let sequence = if sequence.is_empty() {
|
||||||
|
String::new()
|
||||||
|
} else {
|
||||||
|
format!("X-Up-Seq: {sequence}\r\n")
|
||||||
|
};
|
||||||
|
let headers = format!("Content-Type: application/octet-stream\r\n{sequence}{lane}");
|
||||||
|
invalid.push(carrier_request("PUT", &up, &token, &headers, &pong));
|
||||||
|
}
|
||||||
|
let invalid_lane = if carrier.uses_lanes() {
|
||||||
|
""
|
||||||
|
} else {
|
||||||
|
"X-Lane-ID: 0\r\n"
|
||||||
|
};
|
||||||
|
let headers =
|
||||||
|
format!("Content-Type: application/octet-stream\r\nX-Up-Seq: 1\r\n{invalid_lane}");
|
||||||
|
invalid.push(carrier_request("PUT", &up, &token, &headers, &pong));
|
||||||
|
let headers = format!("{down_headers}Content-Type: application/octet-stream\r\n");
|
||||||
|
invalid.push(carrier_request("PUT", &down, &token, &headers, &[]));
|
||||||
|
invalid.push(carrier_request("PUT", &down, &token, &down_headers, &[0]));
|
||||||
|
let headers = format!("X-Down-Cursor: 01\r\n{lane}");
|
||||||
|
invalid.push(carrier_request("PUT", &down, &token, &headers, &[]));
|
||||||
|
for verb in ["GET", "PATCH"] {
|
||||||
|
invalid.push(carrier_request(verb, &up, &token, &up_headers, &pong));
|
||||||
|
invalid.push(carrier_request(verb, &down, &token, &down_headers, &[]));
|
||||||
|
}
|
||||||
|
for path in [
|
||||||
|
"/api/v1/up",
|
||||||
|
"/telegram/web/api/v1/up/",
|
||||||
|
"/telegram/web/api/v1/up?q=1",
|
||||||
|
"/telegram/web/api/v1//up",
|
||||||
|
"/telegram/web/api/v1/%75p",
|
||||||
|
] {
|
||||||
|
invalid.push(carrier_request("PUT", path, &token, &up_headers, &pong));
|
||||||
|
}
|
||||||
|
let wrong_host = String::from_utf8(carrier_request("PUT", &up, &token, &up_headers, &pong))
|
||||||
|
.unwrap()
|
||||||
|
.replace("Host: proxy.example.com", "Host: other.example.com")
|
||||||
|
.into_bytes();
|
||||||
|
invalid.push(wrong_host);
|
||||||
|
let path = format!("{base}api/v1/session");
|
||||||
|
invalid.push(carrier_request("PUT", &path, &token, &up_headers, &pong));
|
||||||
|
for bytes in invalid {
|
||||||
|
let response = request(&listener, &runtime, bytes).await;
|
||||||
|
assert_private_decoy(&response);
|
||||||
|
}
|
||||||
|
// Rejected shapes must not consume the first sequence or close the valid session.
|
||||||
|
let bytes = carrier_request("PUT", &up, &token, &up_headers, &pong);
|
||||||
|
let response = request(&listener, &runtime, bytes).await;
|
||||||
|
assert!(response.starts_with(b"HTTP/1.1 204"));
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn carrier_method_post_and_put_reuse_one_private_http_connection() {
|
||||||
|
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||||
|
let capability = [64; 32];
|
||||||
|
let mut config = runtime_config(capability, carrier);
|
||||||
|
config.web.timeouts.long_poll_secs = 0;
|
||||||
|
config.web.limits.max_streams_global = 0;
|
||||||
|
let generation = test_runtime_generation(1, config);
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let token = session_token(&listener, &runtime, capability, "/").await;
|
||||||
|
let (mut client, cancellation, task) = open_keepalive(&listener, &runtime).await;
|
||||||
|
let lane = if carrier.uses_lanes() {
|
||||||
|
"X-Lane-ID: 7\r\n"
|
||||||
|
} else {
|
||||||
|
""
|
||||||
|
};
|
||||||
|
let open = frame::encode(FrameType::Open, 7, &[]);
|
||||||
|
let close = frame::encode(FrameType::Close, 7, &[]);
|
||||||
|
for verb in ["POST", "PUT", "PUT", "POST"] {
|
||||||
|
for (path, headers, body, ack_header) in [
|
||||||
|
(
|
||||||
|
"/api/v1/up",
|
||||||
|
format!("Content-Type: application/octet-stream\r\nX-Up-Seq: 1\r\n{lane}"),
|
||||||
|
open.as_ref(),
|
||||||
|
"x-up-ack",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"/api/v1/down",
|
||||||
|
format!("X-Down-Cursor: 0\r\n{lane}"),
|
||||||
|
&[][..],
|
||||||
|
"x-down-cursor",
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let bytes = carrier_request(verb, path, &token, &headers, body);
|
||||||
|
let bytes = String::from_utf8(bytes)
|
||||||
|
.unwrap()
|
||||||
|
.replace("Connection: close\r\n", "Connection: keep-alive\r\n")
|
||||||
|
.into_bytes();
|
||||||
|
client.write_all(&bytes).await.unwrap();
|
||||||
|
let response = tokio::time::timeout(
|
||||||
|
std::time::Duration::from_secs(5),
|
||||||
|
read_http_response(&mut client),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let (headers, body) = split_response(&response);
|
||||||
|
assert_eq!(response_header(headers, ack_header), "1");
|
||||||
|
assert!(
|
||||||
|
!std::str::from_utf8(headers)
|
||||||
|
.unwrap()
|
||||||
|
.to_ascii_lowercase()
|
||||||
|
.contains("connection: close")
|
||||||
|
);
|
||||||
|
if path.ends_with("/up") {
|
||||||
|
assert!(headers.starts_with(b"HTTP/1.1 204"));
|
||||||
|
assert!(body.is_empty());
|
||||||
|
} else {
|
||||||
|
assert!(headers.starts_with(b"HTTP/1.1 200"));
|
||||||
|
assert_eq!(
|
||||||
|
response_header(headers, "content-length"),
|
||||||
|
close.len().to_string()
|
||||||
|
);
|
||||||
|
assert_eq!(body, close);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cancellation.cancel();
|
||||||
|
task.await.unwrap();
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn carrier_method_reload_preserves_old_pages_bootstraps_sessions_and_recovery() {
|
||||||
|
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||||
|
let capability = [65; 32];
|
||||||
|
let mut config = runtime_config(capability, carrier);
|
||||||
|
config.web.limits.max_bootstraps_per_ip = 6;
|
||||||
|
let generation = test_runtime_generation(1, config.clone());
|
||||||
|
let active_runtime = Arc::new(ArcSwap::from(Arc::clone(&generation)));
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::clone(&active_runtime));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let post_page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
let old_session =
|
||||||
|
create_session(&listener, &runtime, bootstrap_from(&post_page), "/").await;
|
||||||
|
let unused_post_page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
config.web.carrier_method = WebCarrierMethod::Put;
|
||||||
|
let put_generation = test_runtime_generation(2, config.clone());
|
||||||
|
active_runtime.store(Arc::clone(&put_generation));
|
||||||
|
let post_session =
|
||||||
|
create_session(&listener, &runtime, bootstrap_from(&unused_post_page), "/").await;
|
||||||
|
let put_page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
assert!(String::from_utf8_lossy(&put_page).contains("const carrierMethod='PUT';"));
|
||||||
|
config.web.carrier_method = WebCarrierMethod::Post;
|
||||||
|
let rollback_generation = test_runtime_generation(3, config);
|
||||||
|
active_runtime.store(Arc::clone(&rollback_generation));
|
||||||
|
let put_session = create_session(&listener, &runtime, bootstrap_from(&put_page), "/").await;
|
||||||
|
let rollback_page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
assert!(String::from_utf8_lossy(&rollback_page).contains("const carrierMethod='POST';"));
|
||||||
|
assert!(String::from_utf8_lossy(&post_page).contains("const carrierMethod='POST';"));
|
||||||
|
let lane = if carrier.uses_lanes() {
|
||||||
|
"X-Lane-ID: 0\r\n"
|
||||||
|
} else {
|
||||||
|
""
|
||||||
|
};
|
||||||
|
let headers = format!("Content-Type: application/octet-stream\r\nX-Up-Seq: 1\r\n{lane}");
|
||||||
|
let pong = frame::encode(FrameType::Pong, 0, &[]);
|
||||||
|
for token in [&old_session, &post_session, &put_session] {
|
||||||
|
for verb in ["PUT", "POST"] {
|
||||||
|
let bytes = carrier_request(verb, "/api/v1/up", token, &headers, &pong);
|
||||||
|
let response = request(&listener, &runtime, bytes).await;
|
||||||
|
assert!(response.starts_with(b"HTTP/1.1 204"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let encoded = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(capability);
|
||||||
|
let bytes = carrier_request(
|
||||||
|
"GET",
|
||||||
|
&format!("/?bridge={encoded}"),
|
||||||
|
&put_session,
|
||||||
|
"Accept: application/vnd.telemt.web-recovery+json\r\n",
|
||||||
|
&[],
|
||||||
|
);
|
||||||
|
let recovery = request(&listener, &runtime, bytes).await;
|
||||||
|
let (recovery_headers, body) = split_response(&recovery);
|
||||||
|
assert!(recovery_headers.starts_with(b"HTTP/1.1 200"));
|
||||||
|
assert_eq!(
|
||||||
|
response_header(recovery_headers, "content-type"),
|
||||||
|
"application/vnd.telemt.web-recovery+json"
|
||||||
|
);
|
||||||
|
let document: serde_json::Value = serde_json::from_slice(body).unwrap();
|
||||||
|
let keys: std::collections::BTreeSet<_> = document
|
||||||
|
.as_object()
|
||||||
|
.unwrap()
|
||||||
|
.keys()
|
||||||
|
.map(String::as_str)
|
||||||
|
.collect();
|
||||||
|
assert_eq!(
|
||||||
|
keys,
|
||||||
|
["v", "bootstrap", "limits", "timeouts", "negotiation"]
|
||||||
|
.into_iter()
|
||||||
|
.collect()
|
||||||
|
);
|
||||||
|
let bootstrap = document["bootstrap"].as_str().unwrap();
|
||||||
|
let recovered = create_session(&listener, &runtime, bootstrap, "/").await;
|
||||||
|
let bytes = carrier_request("PUT", "/api/v1/up", &recovered, &headers, &pong);
|
||||||
|
let response = request(&listener, &runtime, bytes).await;
|
||||||
|
assert!(response.starts_with(b"HTTP/1.1 204"));
|
||||||
|
let bytes = carrier_request("PUT", "/api/v1/up", &put_session, &headers, &pong);
|
||||||
|
let retired = request(&listener, &runtime, bytes).await;
|
||||||
|
assert_private_decoy(&retired);
|
||||||
|
runtime.shutdown().await;
|
||||||
|
for generation in [generation, put_generation, rollback_generation] {
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,358 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
async fn create_with_headers(
|
||||||
|
listener: &TcpListener,
|
||||||
|
runtime: &Arc<WebProcessRuntime>,
|
||||||
|
bootstrap: &str,
|
||||||
|
headers: &str,
|
||||||
|
) -> Vec<u8> {
|
||||||
|
request(
|
||||||
|
listener,
|
||||||
|
runtime,
|
||||||
|
carrier_request(
|
||||||
|
"POST",
|
||||||
|
"/api/v1/session",
|
||||||
|
bootstrap,
|
||||||
|
&format!("Content-Type: application/octet-stream\r\n{headers}"),
|
||||||
|
&frame::encode(FrameType::Hello, 0, &[1]),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn up(token: &str, seq: u64, confirmed: &str, lane: &str) -> Vec<u8> {
|
||||||
|
carrier_request(
|
||||||
|
"POST",
|
||||||
|
"/api/v1/up",
|
||||||
|
token,
|
||||||
|
&format!("Content-Type: application/octet-stream\r\nX-Up-Seq: {seq}\r\n{confirmed}{lane}"),
|
||||||
|
&frame::encode(FrameType::Pong, 0, &[]),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn conveyor_negotiation_is_opt_in_bounded_and_invisible_to_native_welcome() {
|
||||||
|
for enabled in [false, true] {
|
||||||
|
for carrier in [
|
||||||
|
WebCarrier::Https,
|
||||||
|
WebCarrier::HttpsLanes,
|
||||||
|
WebCarrier::Websocket,
|
||||||
|
] {
|
||||||
|
let capability = [71; 32];
|
||||||
|
let mut config = runtime_config(capability, carrier);
|
||||||
|
config.web.conveyor = enabled;
|
||||||
|
let generation = test_runtime_generation(1, config);
|
||||||
|
let runtime =
|
||||||
|
WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
for offered in [None, Some(1), Some(2), Some(4)] {
|
||||||
|
let page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
let bootstrap = bootstrap_from(&page);
|
||||||
|
let offer = offered.map_or(String::new(), |value| {
|
||||||
|
format!("X-Telemt-Up-Window: {value}\r\n")
|
||||||
|
});
|
||||||
|
let response = create_with_headers(&listener, &runtime, bootstrap, &offer).await;
|
||||||
|
let (headers, body) = split_response(&response);
|
||||||
|
assert!(headers.starts_with(b"HTTP/1.1 200"));
|
||||||
|
assert_eq!(body, frame::encode(FrameType::Welcome, 0, &[]));
|
||||||
|
if let Some(offered) = offered {
|
||||||
|
let window = if enabled && !carrier.uses_websocket() {
|
||||||
|
offered
|
||||||
|
} else {
|
||||||
|
1
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
response_header(headers, "x-telemt-up-window"),
|
||||||
|
window.to_string()
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
assert!(
|
||||||
|
!String::from_utf8_lossy(headers)
|
||||||
|
.to_lowercase()
|
||||||
|
.contains("x-telemt-up-window")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let replay = create_with_headers(&listener, &runtime, bootstrap, &offer).await;
|
||||||
|
let (replayed, _) = split_response(&replay);
|
||||||
|
assert_eq!(
|
||||||
|
response_header(headers, "x-session-token"),
|
||||||
|
response_header(replayed, "x-session-token")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn malformed_window_does_not_consume_bootstrap_or_enable_a_conveyor() {
|
||||||
|
let capability = [72; 32];
|
||||||
|
let generation = test_runtime_generation(1, runtime_config(capability, WebCarrier::Https));
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
let bootstrap = bootstrap_from(&page);
|
||||||
|
for offer in ["0", "5", "04", "-1", "4, 4", "4\r\nX-Telemt-Up-Window: 4"] {
|
||||||
|
let response = create_with_headers(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
bootstrap,
|
||||||
|
&format!("X-Telemt-Up-Window: {offer}\r\n"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(
|
||||||
|
response.starts_with(b"HTTP/1.1 404"),
|
||||||
|
"{offer}: {:?}",
|
||||||
|
response
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let response =
|
||||||
|
create_with_headers(&listener, &runtime, bootstrap, "X-Telemt-Up-Window: 4\r\n").await;
|
||||||
|
assert!(response.starts_with(b"HTTP/1.1 200"));
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reload_changes_only_new_sessions_and_exact_creation_replay_keeps_its_window() {
|
||||||
|
let capability = [73; 32];
|
||||||
|
let config = runtime_config(capability, WebCarrier::Https);
|
||||||
|
let generation = test_runtime_generation(1, config.clone());
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
let bootstrap = bootstrap_from(&page);
|
||||||
|
let offer = "X-Telemt-Up-Window: 4\r\n";
|
||||||
|
let original = create_with_headers(&listener, &runtime, bootstrap, offer).await;
|
||||||
|
let (original_headers, _) = split_response(&original);
|
||||||
|
let token = response_header(original_headers, "x-session-token");
|
||||||
|
let mut disabled = config;
|
||||||
|
disabled.web.conveyor = false;
|
||||||
|
let replacement = test_runtime_generation(2, disabled);
|
||||||
|
runtime.activate_generation(Arc::clone(&replacement));
|
||||||
|
let replay = create_with_headers(&listener, &runtime, bootstrap, offer).await;
|
||||||
|
let (headers, _) = split_response(&replay);
|
||||||
|
assert_eq!(response_header(headers, "x-telemt-up-window"), "4");
|
||||||
|
assert_eq!(response_header(headers, "x-session-token"), token);
|
||||||
|
let old_up = request(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
up(token, 1, "X-Telemt-Up-Confirmed: 0\r\n", ""),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(old_up.starts_with(b"HTTP/1.1 204"));
|
||||||
|
let page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
let created = create_with_headers(&listener, &runtime, bootstrap_from(&page), offer).await;
|
||||||
|
let (headers, _) = split_response(&created);
|
||||||
|
assert_eq!(response_header(headers, "x-telemt-up-window"), "1");
|
||||||
|
let token = response_header(headers, "x-session-token");
|
||||||
|
let mode_error = request(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
up(token, 1, "X-Telemt-Up-Confirmed: 0\r\n", ""),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(mode_error.starts_with(b"HTTP/1.1 409"));
|
||||||
|
let legacy = request(&listener, &runtime, up(token, 1, "", "")).await;
|
||||||
|
assert!(legacy.starts_with(b"HTTP/1.1 204"));
|
||||||
|
runtime.shutdown().await;
|
||||||
|
for generation in [generation, replacement] {
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn future_http_request_is_not_acked_until_head_applies_for_both_carriers() {
|
||||||
|
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||||
|
let capability = [74; 32];
|
||||||
|
let generation = test_runtime_generation(1, runtime_config(capability, carrier));
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
let created = create_with_headers(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
bootstrap_from(&page),
|
||||||
|
"X-Telemt-Up-Window: 4\r\n",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let (headers, _) = split_response(&created);
|
||||||
|
let token = response_header(headers, "x-session-token");
|
||||||
|
let lane = if carrier.uses_lanes() {
|
||||||
|
"X-Lane-ID: 0\r\n"
|
||||||
|
} else {
|
||||||
|
""
|
||||||
|
};
|
||||||
|
let (mut tail, cancellation, task) = open_keepalive(&listener, &runtime).await;
|
||||||
|
tail.write_all(&up(token, 2, "X-Telemt-Up-Confirmed: 0\r\n", lane))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(std::time::Duration::from_millis(20), tail.read_u8())
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let head = request(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
up(token, 1, "X-Telemt-Up-Confirmed: 0\r\n", lane),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(head.starts_with(b"HTTP/1.1 204"));
|
||||||
|
let response = read_http_response(&mut tail).await;
|
||||||
|
let (headers, _) = split_response(&response);
|
||||||
|
assert!(headers.starts_with(b"HTTP/1.1 204"));
|
||||||
|
assert_eq!(response_header(headers, "x-up-ack"), "2");
|
||||||
|
cancellation.cancel();
|
||||||
|
task.await.unwrap();
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn parked_uplink_keeps_its_session_deadline_after_reload() {
|
||||||
|
let capability = [75; 32];
|
||||||
|
let mut config = runtime_config(capability, WebCarrier::Https);
|
||||||
|
config.web.timeouts.body_secs = 4;
|
||||||
|
let generation = test_runtime_generation(1, config.clone());
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
let response = create_with_headers(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
bootstrap_from(&page),
|
||||||
|
"X-Telemt-Up-Window: 4\r\n",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let (headers, _) = split_response(&response);
|
||||||
|
let token = response_header(headers, "x-session-token");
|
||||||
|
config.web.timeouts.body_secs = 1;
|
||||||
|
config.web.timeouts.http_idle_secs = 2;
|
||||||
|
let replacement = test_runtime_generation(2, config);
|
||||||
|
runtime.activate_generation(Arc::clone(&replacement));
|
||||||
|
let (mut tail, cancellation, task) = open_keepalive(&listener, &runtime).await;
|
||||||
|
tail.write_all(&up(token, 2, "X-Telemt-Up-Confirmed: 0\r\n", ""))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(2200)).await;
|
||||||
|
assert!(
|
||||||
|
!task.is_finished(),
|
||||||
|
"connection idle timeout must not replace the frozen parked-UP deadline"
|
||||||
|
);
|
||||||
|
let head = request(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
up(token, 1, "X-Telemt-Up-Confirmed: 0\r\n", ""),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(head.starts_with(b"HTTP/1.1 204"));
|
||||||
|
assert!(
|
||||||
|
read_http_response(&mut tail)
|
||||||
|
.await
|
||||||
|
.starts_with(b"HTTP/1.1 204")
|
||||||
|
);
|
||||||
|
cancellation.cancel();
|
||||||
|
task.await.unwrap();
|
||||||
|
runtime.shutdown().await;
|
||||||
|
for generation in [generation, replacement] {
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn empty_long_poll_cannot_steal_the_reserved_head_body_capacity() {
|
||||||
|
let capability = [76; 32];
|
||||||
|
let mut config = runtime_config(capability, WebCarrier::Https);
|
||||||
|
config.web.limits.max_body_readers = 3;
|
||||||
|
config.web.limits.max_body_bytes = 32;
|
||||||
|
config.web.limits.max_body_bytes_global = 96;
|
||||||
|
let generation = test_runtime_generation(1, config);
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let page = bridge_page(&listener, &runtime, capability, "/").await;
|
||||||
|
let response = create_with_headers(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
bootstrap_from(&page),
|
||||||
|
"X-Telemt-Up-Window: 4\r\n",
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let (headers, _) = split_response(&response);
|
||||||
|
let token = response_header(headers, "x-session-token");
|
||||||
|
let (mut down, down_cancel, down_task) = open_keepalive(&listener, &runtime).await;
|
||||||
|
down.write_all(&carrier_request(
|
||||||
|
"POST",
|
||||||
|
"/api/v1/down",
|
||||||
|
token,
|
||||||
|
"X-Down-Cursor: 0\r\n",
|
||||||
|
&[],
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let used = |resource| {
|
||||||
|
runtime
|
||||||
|
.capacity_snapshot()
|
||||||
|
.resources
|
||||||
|
.into_iter()
|
||||||
|
.find(|value| value.resource == resource)
|
||||||
|
.unwrap()
|
||||||
|
.used
|
||||||
|
};
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||||
|
while used("lane_polls") != 1 {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
used("body_bytes"),
|
||||||
|
0,
|
||||||
|
"an empty long poll owns no buffered request body"
|
||||||
|
);
|
||||||
|
let mut tails = Vec::new();
|
||||||
|
for sequence in [2, 3] {
|
||||||
|
let (mut client, cancel, task) = open_keepalive(&listener, &runtime).await;
|
||||||
|
client
|
||||||
|
.write_all(&up(token, sequence, "X-Telemt-Up-Confirmed: 0\r\n", ""))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
tails.push((client, cancel, task));
|
||||||
|
}
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||||
|
while used("body_bytes") != 64 {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let head = request(
|
||||||
|
&listener,
|
||||||
|
&runtime,
|
||||||
|
up(token, 1, "X-Telemt-Up-Confirmed: 0\r\n", ""),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(head.starts_with(b"HTTP/1.1 204"));
|
||||||
|
for (mut client, cancel, task) in tails {
|
||||||
|
assert!(
|
||||||
|
read_http_response(&mut client)
|
||||||
|
.await
|
||||||
|
.starts_with(b"HTTP/1.1 204")
|
||||||
|
);
|
||||||
|
cancel.cancel();
|
||||||
|
task.await.unwrap();
|
||||||
|
}
|
||||||
|
down_cancel.cancel();
|
||||||
|
down_task.await.unwrap();
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
@@ -22,7 +22,7 @@ pub(super) async fn handle_down(
|
|||||||
vhost: Arc<WebRuntimeVhost>,
|
vhost: Arc<WebRuntimeVhost>,
|
||||||
token_hash: TokenHash,
|
token_hash: TokenHash,
|
||||||
) -> HttpResponse {
|
) -> HttpResponse {
|
||||||
if request.method() != hyper::Method::POST
|
if !matches!(*request.method(), hyper::Method::POST | hyper::Method::PUT)
|
||||||
|| request.headers().contains_key(header::CONTENT_TYPE)
|
|| request.headers().contains_key(header::CONTENT_TYPE)
|
||||||
{
|
{
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
@@ -65,6 +65,8 @@ pub(super) async fn handle_down(
|
|||||||
if !body.is_empty() {
|
if !body.is_empty() {
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
}
|
}
|
||||||
|
// Empty polls must not retain even one byte from a conveyor head's body reserve.
|
||||||
|
drop(_body_budget);
|
||||||
let Some(_down_poll) = runtime.try_lane_poll(false) else {
|
let Some(_down_poll) = runtime.try_lane_poll(false) else {
|
||||||
return service_unavailable();
|
return service_unavailable();
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,5 +1,9 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
// HTTP method compatibility keeps platform workarounds outside session state.
|
||||||
|
#[path = "carrier_method_tests.rs"]
|
||||||
|
mod carrier_method_tests;
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn windows_restricted_webview_empty_cookie_preserves_the_carrier_flow() {
|
async fn windows_restricted_webview_empty_cookie_preserves_the_carrier_flow() {
|
||||||
for (index, carrier) in [WebCarrier::Https, WebCarrier::HttpsLanes]
|
for (index, carrier) in [WebCarrier::Https, WebCarrier::HttpsLanes]
|
||||||
|
|||||||
@@ -76,6 +76,17 @@ pub(super) async fn handle_session(
|
|||||||
let Some(carrier_request) = carrier_request(&request, &vhost.host) else {
|
let Some(carrier_request) = carrier_request(&request, &vhost.host) else {
|
||||||
return serve_decoy(request, vhost, true, &runtime).await;
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
};
|
};
|
||||||
|
let up_window = if request.headers().contains_key("x-telemt-up-window") {
|
||||||
|
let Some(window) = super::request::canonical_u64_header(&request, "x-telemt-up-window")
|
||||||
|
.filter(|window| (1..=4).contains(window))
|
||||||
|
else {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
};
|
||||||
|
Some(window as u8)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let carrier_request = carrier_request.with_up_window(up_window);
|
||||||
let ip_learning_eligible = carrier_ip_learning_eligible(&request, client_ip);
|
let ip_learning_eligible = carrier_ip_learning_eligible(&request, client_ip);
|
||||||
let Some((trace_session_id, profile, body_timeout)) =
|
let Some((trace_session_id, profile, body_timeout)) =
|
||||||
runtime.bootstrap_trace_identity(token_hash, &vhost.host)
|
runtime.bootstrap_trace_identity(token_hash, &vhost.host)
|
||||||
@@ -124,6 +135,13 @@ pub(super) async fn handle_session(
|
|||||||
}
|
}
|
||||||
let mut response = full_response(StatusCode::OK, welcome);
|
let mut response = full_response(StatusCode::OK, welcome);
|
||||||
carrier_headers(&mut response);
|
carrier_headers(&mut response);
|
||||||
|
if let Some(window) = result.up_window {
|
||||||
|
insert_header(
|
||||||
|
&mut response,
|
||||||
|
HeaderName::from_static("x-telemt-up-window"),
|
||||||
|
&window.to_string(),
|
||||||
|
);
|
||||||
|
}
|
||||||
insert_header(
|
insert_header(
|
||||||
&mut response,
|
&mut response,
|
||||||
HeaderName::from_static("x-session-token"),
|
HeaderName::from_static("x-session-token"),
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
async fn open_keepalive(
|
/// Opens one reusable private HTTP connection for integration scenarios.
|
||||||
|
pub(super) async fn open_keepalive(
|
||||||
listener: &TcpListener,
|
listener: &TcpListener,
|
||||||
runtime: &Arc<WebProcessRuntime>,
|
runtime: &Arc<WebProcessRuntime>,
|
||||||
) -> (TcpStream, CancellationToken, tokio::task::JoinHandle<()>) {
|
) -> (TcpStream, CancellationToken, tokio::task::JoinHandle<()>) {
|
||||||
@@ -21,7 +22,8 @@ async fn open_keepalive(
|
|||||||
(client.unwrap(), cancellation, task)
|
(client.unwrap(), cancellation, task)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn read_http_response(client: &mut TcpStream) -> Vec<u8> {
|
/// Reads one length-delimited response without waiting for connection closure.
|
||||||
|
pub(super) async fn read_http_response(client: &mut TcpStream) -> Vec<u8> {
|
||||||
let mut response = Vec::new();
|
let mut response = Vec::new();
|
||||||
while !response.ends_with(b"\r\n\r\n") {
|
while !response.ends_with(b"\r\n\r\n") {
|
||||||
assert!(response.len() < 16 * 1024);
|
assert!(response.len() < 16 * 1024);
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::web::session::ConveyorError;
|
||||||
|
|
||||||
|
/// Handles one bounded legacy or explicitly negotiated conveyor uplink.
|
||||||
|
pub(super) async fn handle_up(
|
||||||
|
request: Request<RequestBody>,
|
||||||
|
runtime: Arc<WebProcessRuntime>,
|
||||||
|
vhost: Arc<WebRuntimeVhost>,
|
||||||
|
token_hash: crate::web::manager::TokenHash,
|
||||||
|
) -> HttpResponse {
|
||||||
|
if !matches!(*request.method(), Method::POST | Method::PUT) || !binary_content_type(&request) {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
}
|
||||||
|
let Some(sequence) = canonical_u64_header(&request, "x-up-seq").filter(|value| *value != 0)
|
||||||
|
else {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
};
|
||||||
|
let Ok(session) = runtime.get_session(token_hash, &vhost.host) else {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
};
|
||||||
|
if session.carrier().uses_websocket() {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
}
|
||||||
|
if let Some(trace) = request_trace(&request) {
|
||||||
|
trace.set_route(TraceRoute::Uplink);
|
||||||
|
trace.bind_identity(session.trace_identity());
|
||||||
|
}
|
||||||
|
let Some(lane_id) = carrier_lane(&request, session.carrier()) else {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
};
|
||||||
|
|
||||||
|
let confirmed = if request.headers().contains_key("x-telemt-up-confirmed") {
|
||||||
|
let Some(value) = canonical_u64_header(&request, "x-telemt-up-confirmed") else {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
};
|
||||||
|
Some(value)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let claim = match session.claim_conveyor(lane_id, sequence, confirmed) {
|
||||||
|
Ok(claim) => claim,
|
||||||
|
Err(ConveyorError::Stale | ConveyorError::Mode) => {
|
||||||
|
return carrier_empty(StatusCode::CONFLICT);
|
||||||
|
}
|
||||||
|
Err(ConveyorError::Manager(
|
||||||
|
ManagerError::Backpressure | ManagerError::Concurrent | ManagerError::Limit,
|
||||||
|
)) => {
|
||||||
|
return service_unavailable();
|
||||||
|
}
|
||||||
|
Err(_) => return serve_decoy(request, vhost, true, &runtime).await,
|
||||||
|
};
|
||||||
|
let limit = session.limits().max_body_bytes;
|
||||||
|
let CollectedBody {
|
||||||
|
request,
|
||||||
|
body,
|
||||||
|
_body_budget,
|
||||||
|
} = match collect_body(
|
||||||
|
request,
|
||||||
|
&runtime,
|
||||||
|
Duration::from_secs(session.timeouts().body_secs),
|
||||||
|
limit,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(CollectBodyError::Limit) => return service_unavailable(),
|
||||||
|
Err(CollectBodyError::Invalid(request)) => {
|
||||||
|
return serve_decoy(request, vhost, true, &runtime).await;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if let Some(trace) = request_trace(&request) {
|
||||||
|
trace.record_frames(TraceDirection::Request, &body, session.limits());
|
||||||
|
}
|
||||||
|
let result = if let Some(claim) = claim {
|
||||||
|
let _deadline_lease = match super::request_deadline(&request) {
|
||||||
|
Some(deadline) => {
|
||||||
|
match deadline.lease_for(Duration::from_secs(session.timeouts().body_secs)) {
|
||||||
|
Some(lease) => Some(lease),
|
||||||
|
None => return service_unavailable(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
claim.process(&body).await
|
||||||
|
} else {
|
||||||
|
match lane_id {
|
||||||
|
Some(lane_id) => session.process_up_lane(lane_id, sequence, &body),
|
||||||
|
None => session.process_up(sequence, &body),
|
||||||
|
}
|
||||||
|
.map_err(ConveyorError::Manager)
|
||||||
|
};
|
||||||
|
match result {
|
||||||
|
Ok(ack) => {
|
||||||
|
let mut response = carrier_empty(StatusCode::NO_CONTENT);
|
||||||
|
insert_header(
|
||||||
|
&mut response,
|
||||||
|
HeaderName::from_static("x-up-ack"),
|
||||||
|
&ack.to_string(),
|
||||||
|
);
|
||||||
|
response
|
||||||
|
}
|
||||||
|
Err(ConveyorError::Stale | ConveyorError::Mode) => carrier_empty(StatusCode::CONFLICT),
|
||||||
|
Err(ConveyorError::Manager(
|
||||||
|
ManagerError::Backpressure | ManagerError::Concurrent | ManagerError::Limit,
|
||||||
|
)) => service_unavailable(),
|
||||||
|
Err(_) => serve_decoy(request, vhost, true, &runtime).await,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -177,6 +177,8 @@ impl ManagerError {
|
|||||||
|
|
||||||
/// Successful idempotent session creation result.
|
/// Successful idempotent session creation result.
|
||||||
pub(crate) struct CreateResult {
|
pub(crate) struct CreateResult {
|
||||||
|
/// HTTP window echoed only to bridge pages advertising the extension.
|
||||||
|
pub(crate) up_window: Option<u8>,
|
||||||
/// Opaque bearer token for the created or replayed session.
|
/// Opaque bearer token for the created or replayed session.
|
||||||
pub(crate) token: String,
|
pub(crate) token: String,
|
||||||
/// Carrier frozen into the created or replayed session.
|
/// Carrier frozen into the created or replayed session.
|
||||||
@@ -230,6 +232,7 @@ pub(crate) struct WebProcessRuntime {
|
|||||||
lane_aux_polls: Arc<Semaphore>,
|
lane_aux_polls: Arc<Semaphore>,
|
||||||
body_readers: Arc<Semaphore>,
|
body_readers: Arc<Semaphore>,
|
||||||
body_bytes: Arc<Semaphore>,
|
body_bytes: Arc<Semaphore>,
|
||||||
|
conveyor_waiters: Arc<Semaphore>,
|
||||||
stream_handshakes: Arc<Semaphore>,
|
stream_handshakes: Arc<Semaphore>,
|
||||||
websocket_connections: Arc<Semaphore>,
|
websocket_connections: Arc<Semaphore>,
|
||||||
websockets: Mutex<websocket::WebSocketRegistry>,
|
websockets: Mutex<websocket::WebSocketRegistry>,
|
||||||
@@ -299,6 +302,9 @@ impl WebProcessRuntime {
|
|||||||
lane_aux_polls: Arc::new(Semaphore::new(lane_aux_poll_limit)),
|
lane_aux_polls: Arc::new(Semaphore::new(lane_aux_poll_limit)),
|
||||||
body_readers: Arc::new(Semaphore::new(limits.max_body_readers)),
|
body_readers: Arc::new(Semaphore::new(limits.max_body_readers)),
|
||||||
body_bytes: Arc::new(Semaphore::new(limits.max_body_bytes_global)),
|
body_bytes: Arc::new(Semaphore::new(limits.max_body_bytes_global)),
|
||||||
|
conveyor_waiters: Arc::new(Semaphore::new(crate::web::session::conveyor_waiter_limit(
|
||||||
|
&limits,
|
||||||
|
))),
|
||||||
stream_handshakes: Arc::new(Semaphore::new(limits.max_stream_handshakes)),
|
stream_handshakes: Arc::new(Semaphore::new(limits.max_stream_handshakes)),
|
||||||
websocket_connections: Arc::new(Semaphore::new(websocket_connections)),
|
websocket_connections: Arc::new(Semaphore::new(websocket_connections)),
|
||||||
websockets: Mutex::new(websocket::WebSocketRegistry::default()),
|
websockets: Mutex::new(websocket::WebSocketRegistry::default()),
|
||||||
|
|||||||
@@ -337,6 +337,11 @@ impl Drop for WebSocketBudgetLease {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl WebProcessRuntime {
|
impl WebProcessRuntime {
|
||||||
|
/// Reserves a parked future uplink without consuming the head request's capacity.
|
||||||
|
pub(crate) fn try_conveyor_waiter(&self) -> Option<OwnedSemaphorePermit> {
|
||||||
|
Arc::clone(&self.conveyor_waiters).try_acquire_owned().ok()
|
||||||
|
}
|
||||||
|
|
||||||
/// Reserves one body reader and its declared bounded body allocation.
|
/// Reserves one body reader and its declared bounded body allocation.
|
||||||
pub(crate) fn try_body_budget(
|
pub(crate) fn try_body_budget(
|
||||||
&self,
|
&self,
|
||||||
|
|||||||
@@ -136,6 +136,7 @@ impl WebProcessRuntime {
|
|||||||
self.lane_aux_polls.close();
|
self.lane_aux_polls.close();
|
||||||
self.body_readers.close();
|
self.body_readers.close();
|
||||||
self.body_bytes.close();
|
self.body_bytes.close();
|
||||||
|
self.conveyor_waiters.close();
|
||||||
self.stream_handshakes.close();
|
self.stream_handshakes.close();
|
||||||
self.websocket_connections.close();
|
self.websocket_connections.close();
|
||||||
let sessions = {
|
let sessions = {
|
||||||
|
|||||||
@@ -130,6 +130,7 @@ pub(crate) struct CarrierRequest {
|
|||||||
attempt: Option<u8>,
|
attempt: Option<u8>,
|
||||||
failure: Option<CarrierFailure>,
|
failure: Option<CarrierFailure>,
|
||||||
user_agent_hash: [u8; 32],
|
user_agent_hash: [u8; 32],
|
||||||
|
up_window: Option<u8>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CarrierRequest {
|
impl CarrierRequest {
|
||||||
@@ -141,6 +142,7 @@ impl CarrierRequest {
|
|||||||
attempt: None,
|
attempt: None,
|
||||||
failure: None,
|
failure: None,
|
||||||
user_agent_hash,
|
user_agent_hash,
|
||||||
|
up_window: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -152,6 +154,7 @@ impl CarrierRequest {
|
|||||||
attempt: None,
|
attempt: None,
|
||||||
failure: None,
|
failure: None,
|
||||||
user_agent_hash,
|
user_agent_hash,
|
||||||
|
up_window: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -169,9 +172,21 @@ impl CarrierRequest {
|
|||||||
attempt: Some(attempt),
|
attempt: Some(attempt),
|
||||||
failure,
|
failure,
|
||||||
user_agent_hash,
|
user_agent_hash,
|
||||||
|
up_window: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Attaches the separately validated HTTP conveyor capability.
|
||||||
|
pub(crate) const fn with_up_window(mut self, window: Option<u8>) -> Self {
|
||||||
|
self.up_window = window;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the separately advertised HTTP conveyor capability.
|
||||||
|
pub(crate) const fn up_window(self) -> Option<u8> {
|
||||||
|
self.up_window
|
||||||
|
}
|
||||||
|
|
||||||
/// Returns whether this request participates in server-side negotiation.
|
/// Returns whether this request participates in server-side negotiation.
|
||||||
pub(crate) const fn is_automatic(self) -> bool {
|
pub(crate) const fn is_automatic(self) -> bool {
|
||||||
self.capabilities.is_some()
|
self.capabilities.is_some()
|
||||||
@@ -215,6 +230,7 @@ impl CarrierRequest {
|
|||||||
self.class == other.class
|
self.class == other.class
|
||||||
&& self.capabilities_bits() == other.capabilities_bits()
|
&& self.capabilities_bits() == other.capabilities_bits()
|
||||||
&& self.user_agent_hash == other.user_agent_hash
|
&& self.user_agent_hash == other.user_agent_hash
|
||||||
|
&& self.up_window == other.up_window
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Checks the complete idempotent identity of one exact attempt request.
|
/// Checks the complete idempotent identity of one exact attempt request.
|
||||||
|
|||||||
@@ -125,6 +125,7 @@ impl WebProcessRuntime {
|
|||||||
entry.carrier_phase.as_str()
|
entry.carrier_phase.as_str()
|
||||||
};
|
};
|
||||||
let result = CreateResult {
|
let result = CreateResult {
|
||||||
|
up_window: carrier_request.up_window().map(|_| session.up_window()),
|
||||||
token: entry.session_token.as_str().to_owned(),
|
token: entry.session_token.as_str().to_owned(),
|
||||||
carrier: session.carrier(),
|
carrier: session.carrier(),
|
||||||
attempt: carrier_request.attempt(),
|
attempt: carrier_request.attempt(),
|
||||||
@@ -357,6 +358,11 @@ impl WebProcessRuntime {
|
|||||||
issued_timeouts.clone(),
|
issued_timeouts.clone(),
|
||||||
Some(user_registration),
|
Some(user_registration),
|
||||||
);
|
);
|
||||||
|
session.configure_conveyor(if config.web.conveyor {
|
||||||
|
carrier_request.up_window().unwrap_or(1)
|
||||||
|
} else {
|
||||||
|
1
|
||||||
|
});
|
||||||
state.sessions.insert(session_hash, Arc::clone(&session));
|
state.sessions.insert(session_hash, Arc::clone(&session));
|
||||||
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
|
*state.sessions_per_ip.entry(client_ip).or_insert(0) += 1;
|
||||||
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
|
*state.sessions_per_profile.entry(profile_key).or_insert(0) += 1;
|
||||||
@@ -399,6 +405,7 @@ impl WebProcessRuntime {
|
|||||||
}
|
}
|
||||||
let identity = session.trace_identity();
|
let identity = session.trace_identity();
|
||||||
let result = CreateResult {
|
let result = CreateResult {
|
||||||
|
up_window: carrier_request.up_window().map(|_| session.up_window()),
|
||||||
token: session_token,
|
token: session_token,
|
||||||
carrier,
|
carrier,
|
||||||
attempt: carrier_request.attempt(),
|
attempt: carrier_request.attempt(),
|
||||||
|
|||||||
@@ -104,6 +104,7 @@ impl WebProcessRuntime {
|
|||||||
replacement.old_session.timeouts().clone(),
|
replacement.old_session.timeouts().clone(),
|
||||||
Some(user_registration),
|
Some(user_registration),
|
||||||
);
|
);
|
||||||
|
session.configure_conveyor(replacement.old_session.conveyor_offer());
|
||||||
let Some(supersede) = replacement.old_session.prepare_carrier_supersede() else {
|
let Some(supersede) = replacement.old_session.prepare_carrier_supersede() else {
|
||||||
drop(user_publication);
|
drop(user_publication);
|
||||||
drop(state);
|
drop(state);
|
||||||
@@ -149,6 +150,7 @@ impl WebProcessRuntime {
|
|||||||
crate::web::session::SessionCloseReason::CarrierSuperseded,
|
crate::web::session::SessionCloseReason::CarrierSuperseded,
|
||||||
);
|
);
|
||||||
let result = CreateResult {
|
let result = CreateResult {
|
||||||
|
up_window: replacement.request.up_window().map(|_| session.up_window()),
|
||||||
token: session_token,
|
token: session_token,
|
||||||
carrier: replacement.carrier,
|
carrier: replacement.carrier,
|
||||||
attempt: Some(replacement.attempt),
|
attempt: Some(replacement.attempt),
|
||||||
|
|||||||
@@ -54,6 +54,9 @@ use lifecycle::SessionNegotiationPhase;
|
|||||||
pub(crate) use lifecycle::{SessionCloseOutcome, SessionCloseReason};
|
pub(crate) use lifecycle::{SessionCloseOutcome, SessionCloseReason};
|
||||||
// Uplink batches own exactly-once sequencing and client-frame validation.
|
// Uplink batches own exactly-once sequencing and client-frame validation.
|
||||||
mod uplink;
|
mod uplink;
|
||||||
|
// HTTP conveyor ownership bounds reordered requests without changing native frames.
|
||||||
|
mod conveyor;
|
||||||
|
pub(crate) use conveyor::{ConveyorError, conveyor_waiter_limit};
|
||||||
// Logical stream polling owns cancellation-safe waker registration.
|
// Logical stream polling owns cancellation-safe waker registration.
|
||||||
mod stream_io;
|
mod stream_io;
|
||||||
|
|
||||||
@@ -160,6 +163,7 @@ impl CarrierLane {
|
|||||||
}
|
}
|
||||||
|
|
||||||
struct SessionState {
|
struct SessionState {
|
||||||
|
conveyor: conveyor::ConveyorState,
|
||||||
streams: HashMap<u32, StreamState>,
|
streams: HashMap<u32, StreamState>,
|
||||||
closing_streams: HashMap<u32, u64>,
|
closing_streams: HashMap<u32, u64>,
|
||||||
next_stream_instance: u64,
|
next_stream_instance: u64,
|
||||||
@@ -295,6 +299,7 @@ impl WebSession {
|
|||||||
timeouts,
|
timeouts,
|
||||||
_user_registration: user_registration,
|
_user_registration: user_registration,
|
||||||
state: Mutex::new(SessionState {
|
state: Mutex::new(SessionState {
|
||||||
|
conveyor: conveyor::ConveyorState::default(),
|
||||||
streams: HashMap::new(),
|
streams: HashMap::new(),
|
||||||
closing_streams: HashMap::new(),
|
closing_streams: HashMap::new(),
|
||||||
next_stream_instance: 1,
|
next_stream_instance: 1,
|
||||||
|
|||||||
@@ -0,0 +1,421 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use subtle::ConstantTimeEq;
|
||||||
|
use tokio::sync::{Notify, OwnedSemaphorePermit};
|
||||||
|
|
||||||
|
use super::{SessionCloseReason, SessionNegotiationPhase, WebSession};
|
||||||
|
use crate::config::WebLimitsConfig;
|
||||||
|
use crate::web::manager::{ManagerError, TokenHash};
|
||||||
|
use crate::web::telemetry::WebSessionLifecycleObservation;
|
||||||
|
|
||||||
|
const MAX_WINDOW: usize = 4;
|
||||||
|
|
||||||
|
/// Preserves one head body and handler capacity beyond the bounded down-poll plane.
|
||||||
|
pub(crate) fn conveyor_waiter_limit(limits: &WebLimitsConfig) -> usize {
|
||||||
|
(limits.max_http_handlers / 4)
|
||||||
|
.min(limits.max_body_readers.saturating_sub(1))
|
||||||
|
.min((limits.max_body_bytes_global / limits.max_body_bytes.max(1)).saturating_sub(1))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// HTTP-only failures distinguish expired exchanges from terminal protocol faults.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(crate) enum ConveyorError {
|
||||||
|
/// The bridge already confirmed this sequence, so its digest was retired.
|
||||||
|
Stale,
|
||||||
|
/// A negotiated mode cannot change within the same bearer session.
|
||||||
|
Mode,
|
||||||
|
/// Existing carrier failure semantics remain authoritative.
|
||||||
|
Manager(ManagerError),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<ManagerError> for ConveyorError {
|
||||||
|
fn from(error: ManagerError) -> Self {
|
||||||
|
Self::Manager(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
struct Slot {
|
||||||
|
sequence: u64,
|
||||||
|
owner: u64,
|
||||||
|
digest: Option<TokenHash>,
|
||||||
|
committed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Channel {
|
||||||
|
confirmed: u64,
|
||||||
|
committed: u64,
|
||||||
|
slots: [Option<Slot>; MAX_WINDOW],
|
||||||
|
notify: Arc<Notify>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Channel {
|
||||||
|
fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
confirmed: 0,
|
||||||
|
committed: 0,
|
||||||
|
slots: [None; MAX_WINDOW],
|
||||||
|
notify: Arc::new(Notify::new()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Session-owned replay metadata; bodies stay in their bounded HTTP handlers.
|
||||||
|
pub(super) struct ConveyorState {
|
||||||
|
offered: u8,
|
||||||
|
mode: Option<bool>,
|
||||||
|
next_owner: u64,
|
||||||
|
channels: HashMap<Option<u32>, Channel>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for ConveyorState {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
offered: 1,
|
||||||
|
mode: None,
|
||||||
|
next_owner: 1,
|
||||||
|
channels: HashMap::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ConveyorState {
|
||||||
|
/// Publishes commit history under the same lock as frame application.
|
||||||
|
pub(super) fn commit(&mut self, lane: Option<u32>, sequence: u64, digest: TokenHash) {
|
||||||
|
if self.mode != Some(true) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if let Some(channel) = self.channels.get_mut(&lane) {
|
||||||
|
let index = sequence as usize % MAX_WINDOW;
|
||||||
|
channel.slots[index] = Some(Slot {
|
||||||
|
sequence,
|
||||||
|
digest: Some(digest),
|
||||||
|
committed: true,
|
||||||
|
owner: 0,
|
||||||
|
});
|
||||||
|
channel.committed = sequence;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Detaches retired-lane ordering state for notification after unlocking.
|
||||||
|
pub(super) fn remove(&mut self, lane: Option<u32>) -> Option<Arc<Notify>> {
|
||||||
|
self.channels.remove(&lane).map(|channel| channel.notify)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Detaches all waiter notifications for the session's deferred close effects.
|
||||||
|
pub(super) fn clear(&mut self) -> Vec<Arc<Notify>> {
|
||||||
|
self.channels
|
||||||
|
.drain()
|
||||||
|
.map(|(_, channel)| channel.notify)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An exact request owner; its body and ordinary body permit stay in the HTTP handler.
|
||||||
|
pub(crate) struct ConveyorClaim {
|
||||||
|
session: Arc<WebSession>,
|
||||||
|
lane: Option<u32>,
|
||||||
|
sequence: u64,
|
||||||
|
owner: u64,
|
||||||
|
notify: Arc<Notify>,
|
||||||
|
_waiter: Option<OwnedSemaphorePermit>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebSession {
|
||||||
|
/// Freezes the offer before the newly created session is published by its manager.
|
||||||
|
pub(crate) fn configure_conveyor(&self, requested: u8) {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
if state.conveyor.mode.is_none() {
|
||||||
|
state.conveyor.offered = if conveyor_waiter_limit(&self.limits) == 0 {
|
||||||
|
1
|
||||||
|
} else {
|
||||||
|
requested.clamp(1, MAX_WINDOW as u8)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the immutable chain ceiling even across a WebSocket candidate.
|
||||||
|
pub(crate) fn conveyor_offer(&self) -> u8 {
|
||||||
|
self.state.lock().conveyor.offered
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the HTTP window without changing carrier capability negotiation.
|
||||||
|
pub(crate) fn up_window(&self) -> u8 {
|
||||||
|
if self.carrier().uses_websocket() {
|
||||||
|
1
|
||||||
|
} else {
|
||||||
|
self.conveyor_offer()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Claims an HTTP sequence before allocating a body, or locks in legacy serialization.
|
||||||
|
pub(crate) fn claim_conveyor(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
lane: Option<u32>,
|
||||||
|
sequence: u64,
|
||||||
|
confirmed: Option<u64>,
|
||||||
|
) -> Result<Option<ConveyorClaim>, ConveyorError> {
|
||||||
|
let mut state = self.state.lock();
|
||||||
|
self.ensure_carrier_active_locked(&state)?;
|
||||||
|
if state.closed {
|
||||||
|
return Err(ManagerError::Closed.into());
|
||||||
|
}
|
||||||
|
let enabled = confirmed.is_some();
|
||||||
|
if (enabled && state.conveyor.offered <= 1)
|
||||||
|
|| state.conveyor.mode.is_some_and(|mode| mode != enabled)
|
||||||
|
{
|
||||||
|
return Err(ConveyorError::Mode);
|
||||||
|
}
|
||||||
|
if !enabled {
|
||||||
|
state.conveyor.mode = Some(false);
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let confirmed = confirmed.unwrap_or(0);
|
||||||
|
let window = u64::from(state.conveyor.offered);
|
||||||
|
let existing = state.conveyor.channels.get(&lane);
|
||||||
|
// Lane eviction retires its bounded digest history, not the parent session.
|
||||||
|
if existing.is_none() && lane.is_some() && confirmed > 0 {
|
||||||
|
return Err(ConveyorError::Stale);
|
||||||
|
}
|
||||||
|
let committed = existing.map_or(0, |channel| channel.committed);
|
||||||
|
let floor = existing
|
||||||
|
.map_or(0, |channel| channel.confirmed)
|
||||||
|
.max(confirmed);
|
||||||
|
if confirmed > committed || sequence == 0 || sequence > floor.saturating_add(window) {
|
||||||
|
drop(state);
|
||||||
|
self.close(SessionCloseReason::Protocol);
|
||||||
|
return Err(ManagerError::Protocol.into());
|
||||||
|
}
|
||||||
|
if sequence <= floor {
|
||||||
|
return Err(ConveyorError::Stale);
|
||||||
|
}
|
||||||
|
let index = sequence as usize % MAX_WINDOW;
|
||||||
|
if existing
|
||||||
|
.and_then(|channel| channel.slots[index])
|
||||||
|
.is_some_and(|slot| slot.sequence == sequence && !slot.committed)
|
||||||
|
{
|
||||||
|
return Err(ManagerError::Concurrent.into());
|
||||||
|
}
|
||||||
|
if existing.is_none() {
|
||||||
|
let limit = self
|
||||||
|
.profile
|
||||||
|
.max_streams_per_session
|
||||||
|
.saturating_add(self.limits.max_tombstones_per_session)
|
||||||
|
.saturating_add(1);
|
||||||
|
let provisional = state
|
||||||
|
.conveyor
|
||||||
|
.channels
|
||||||
|
.keys()
|
||||||
|
.filter(|id| id.is_some_and(|id| !state.carrier_lanes.contains_key(&id)))
|
||||||
|
.count();
|
||||||
|
if state.conveyor.channels.len() >= limit
|
||||||
|
|| (lane.is_some_and(|id| !state.carrier_lanes.contains_key(&id))
|
||||||
|
&& state.carrier_lanes.len().saturating_add(provisional) >= limit)
|
||||||
|
{
|
||||||
|
return Err(ManagerError::Limit.into());
|
||||||
|
}
|
||||||
|
if lane.is_some_and(|id| {
|
||||||
|
state.closed_streams.contains(&id) && !state.carrier_lanes.contains_key(&id)
|
||||||
|
}) {
|
||||||
|
return Err(ConveyorError::Stale);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let owner = state.conveyor.next_owner;
|
||||||
|
let next_owner = owner.checked_add(1).ok_or(ManagerError::Limit)?;
|
||||||
|
// All fallible validation precedes the permit so no early return drops it under state.
|
||||||
|
let waiter = if sequence > committed.saturating_add(1) {
|
||||||
|
Some(
|
||||||
|
self.manager
|
||||||
|
.upgrade()
|
||||||
|
.ok_or(ManagerError::Closed)?
|
||||||
|
.try_conveyor_waiter()
|
||||||
|
.ok_or(ManagerError::Backpressure)?,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
state.conveyor.mode = Some(true);
|
||||||
|
state.conveyor.next_owner = next_owner;
|
||||||
|
let channel = state
|
||||||
|
.conveyor
|
||||||
|
.channels
|
||||||
|
.entry(lane)
|
||||||
|
.or_insert_with(Channel::new);
|
||||||
|
channel.confirmed = floor;
|
||||||
|
for slot in &mut channel.slots {
|
||||||
|
if slot.is_some_and(|slot| slot.sequence <= floor) {
|
||||||
|
*slot = None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let replay = sequence <= channel.committed;
|
||||||
|
if !replay {
|
||||||
|
channel.slots[index] = Some(Slot {
|
||||||
|
sequence,
|
||||||
|
owner,
|
||||||
|
digest: None,
|
||||||
|
committed: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(Some(ConveyorClaim {
|
||||||
|
session: Arc::clone(self),
|
||||||
|
lane,
|
||||||
|
sequence,
|
||||||
|
owner: if replay { 0 } else { owner },
|
||||||
|
notify: Arc::clone(&channel.notify),
|
||||||
|
_waiter: waiter,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ConveyorClaim {
|
||||||
|
/// Revalidates exact ownership under the frame-application transaction's lock.
|
||||||
|
pub(super) fn validate_locked(
|
||||||
|
&self,
|
||||||
|
state: &super::SessionState,
|
||||||
|
digest: &TokenHash,
|
||||||
|
) -> Result<(), ManagerError> {
|
||||||
|
let channel = state
|
||||||
|
.conveyor
|
||||||
|
.channels
|
||||||
|
.get(&self.lane)
|
||||||
|
.filter(|channel| Arc::ptr_eq(&channel.notify, &self.notify))
|
||||||
|
.ok_or(ManagerError::Closed)?;
|
||||||
|
let slot = channel.slots[self.sequence as usize % MAX_WINDOW]
|
||||||
|
.filter(|slot| {
|
||||||
|
slot.sequence == self.sequence
|
||||||
|
&& slot.owner == self.owner
|
||||||
|
&& !slot.committed
|
||||||
|
&& slot
|
||||||
|
.digest
|
||||||
|
.is_some_and(|value| bool::from(value.ct_eq(digest)))
|
||||||
|
})
|
||||||
|
.ok_or(ManagerError::Closed)?;
|
||||||
|
if channel.committed.checked_add(1) != Some(slot.sequence) {
|
||||||
|
return Err(ManagerError::Concurrent);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Applies only the next sequence; duplicates verify the retained digest without replaying frames.
|
||||||
|
pub(crate) async fn process(&self, body: &[u8]) -> Result<u64, ConveyorError> {
|
||||||
|
let digest: TokenHash = Sha256::digest(body).into();
|
||||||
|
let operation = async {
|
||||||
|
loop {
|
||||||
|
let notified = self.notify.notified();
|
||||||
|
tokio::pin!(notified);
|
||||||
|
notified.as_mut().enable();
|
||||||
|
let ready = {
|
||||||
|
let mut state = self.session.state.lock();
|
||||||
|
self.session.ensure_carrier_active_locked(&state)?;
|
||||||
|
if state.closed {
|
||||||
|
return Err(ManagerError::Closed.into());
|
||||||
|
}
|
||||||
|
let Some(channel) = state
|
||||||
|
.conveyor
|
||||||
|
.channels
|
||||||
|
.get_mut(&self.lane)
|
||||||
|
.filter(|channel| Arc::ptr_eq(&channel.notify, &self.notify))
|
||||||
|
else {
|
||||||
|
return Err(ConveyorError::Stale);
|
||||||
|
};
|
||||||
|
if self.sequence <= channel.confirmed {
|
||||||
|
return Err(ConveyorError::Stale);
|
||||||
|
}
|
||||||
|
let Some(slot) = &mut channel.slots[self.sequence as usize % MAX_WINDOW] else {
|
||||||
|
return Err(ConveyorError::Stale);
|
||||||
|
};
|
||||||
|
if slot.sequence != self.sequence
|
||||||
|
|| (!slot.committed && slot.owner != self.owner)
|
||||||
|
{
|
||||||
|
return Err(ConveyorError::Stale);
|
||||||
|
}
|
||||||
|
if slot
|
||||||
|
.digest
|
||||||
|
.is_some_and(|previous| !bool::from(previous.ct_eq(&digest)))
|
||||||
|
{
|
||||||
|
drop(state);
|
||||||
|
self.session.close(SessionCloseReason::Protocol);
|
||||||
|
return Err(ManagerError::Protocol.into());
|
||||||
|
}
|
||||||
|
slot.digest = Some(digest);
|
||||||
|
if slot.committed {
|
||||||
|
self.session.touch_peer_locked(
|
||||||
|
&mut state,
|
||||||
|
Instant::now(),
|
||||||
|
WebSessionLifecycleObservation::HttpActivityAfterGap,
|
||||||
|
);
|
||||||
|
if self.session.automatic_carrier
|
||||||
|
&& state.negotiation_phase != SessionNegotiationPhase::Committed
|
||||||
|
{
|
||||||
|
return Err(ManagerError::Backpressure.into());
|
||||||
|
}
|
||||||
|
return Ok(self.sequence);
|
||||||
|
}
|
||||||
|
channel.committed.checked_add(1) == Some(self.sequence)
|
||||||
|
&& (self.lane.is_some()
|
||||||
|
|| !self
|
||||||
|
.session
|
||||||
|
.up_active
|
||||||
|
.load(std::sync::atomic::Ordering::Acquire))
|
||||||
|
};
|
||||||
|
if ready {
|
||||||
|
let result = match self.lane {
|
||||||
|
Some(lane) => self.session.process_up_lane_inner(
|
||||||
|
lane,
|
||||||
|
self.sequence,
|
||||||
|
body,
|
||||||
|
Some((self, digest)),
|
||||||
|
),
|
||||||
|
None => self
|
||||||
|
.session
|
||||||
|
.process_claimed_up(self.sequence, body, self, digest),
|
||||||
|
};
|
||||||
|
self.notify.notify_waiters();
|
||||||
|
return result.map_err(ConveyorError::from);
|
||||||
|
}
|
||||||
|
tokio::select! {
|
||||||
|
_ = self.session.cancel.cancelled() => return Err(ManagerError::Closed.into()),
|
||||||
|
_ = notified => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
tokio::time::timeout(
|
||||||
|
Duration::from_secs(self.session.timeouts.body_secs),
|
||||||
|
operation,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ConveyorError::Manager(ManagerError::Backpressure))?
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for ConveyorClaim {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if self.owner == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let mut state = self.session.state.lock();
|
||||||
|
if let Some(channel) = state
|
||||||
|
.conveyor
|
||||||
|
.channels
|
||||||
|
.get_mut(&self.lane)
|
||||||
|
.filter(|channel| Arc::ptr_eq(&channel.notify, &self.notify))
|
||||||
|
{
|
||||||
|
let slot = &mut channel.slots[self.sequence as usize % MAX_WINDOW];
|
||||||
|
if slot.is_some_and(|slot| !slot.committed && slot.owner == self.owner) {
|
||||||
|
*slot = None;
|
||||||
|
}
|
||||||
|
if channel.committed == 0 && channel.slots.iter().all(Option::is_none) {
|
||||||
|
state.conveyor.channels.remove(&self.lane);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
drop(state);
|
||||||
|
self.notify.notify_waiters();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "conveyor/tests.rs"]
|
||||||
|
mod tests;
|
||||||
@@ -0,0 +1,524 @@
|
|||||||
|
use super::*;
|
||||||
|
use std::collections::VecDeque;
|
||||||
|
use std::future::Future;
|
||||||
|
use std::task::{Context, Poll, Wake, Waker};
|
||||||
|
|
||||||
|
use crate::config::{ProxyConfig, WebCarrier, WebRuntimeProfile, WebSecretMode, WebTimeoutsConfig};
|
||||||
|
use crate::maestro::generation::test_runtime_generation;
|
||||||
|
use crate::web::frame::{self, FrameType};
|
||||||
|
use crate::web::manager::{CarrierClientClass, WebProcessRuntime};
|
||||||
|
use crate::web::session::{CarrierLane, StreamState};
|
||||||
|
use arc_swap::ArcSwap;
|
||||||
|
|
||||||
|
fn fixture(
|
||||||
|
carrier: WebCarrier,
|
||||||
|
limits: WebLimitsConfig,
|
||||||
|
automatic: bool,
|
||||||
|
) -> (Arc<WebSession>, Arc<WebProcessRuntime>) {
|
||||||
|
let mut config = ProxyConfig::default();
|
||||||
|
config.web.limits = limits.clone();
|
||||||
|
let manager =
|
||||||
|
WebProcessRuntime::start(Arc::new(ArcSwap::from(test_runtime_generation(1, config))));
|
||||||
|
let profile = Arc::new(WebRuntimeProfile {
|
||||||
|
host: "proxy.example.com".into(),
|
||||||
|
public_addr: "203.0.113.10:443".parse().unwrap(),
|
||||||
|
user: "default".into(),
|
||||||
|
secret_mode: WebSecretMode::Plain,
|
||||||
|
carrier,
|
||||||
|
carrier_negotiation_enabled: automatic,
|
||||||
|
carrier_learning: false,
|
||||||
|
carriers: Arc::from([carrier]),
|
||||||
|
carrier_negotiation_deadlines_secs: [3, 5, 8, 12],
|
||||||
|
capability: [0; 32],
|
||||||
|
credential_id: [0; 16],
|
||||||
|
key_fingerprint: "0000000000000000".into(),
|
||||||
|
max_sessions: 2,
|
||||||
|
max_streams: 2,
|
||||||
|
max_streams_per_session: 2,
|
||||||
|
});
|
||||||
|
let session = WebSession::new(
|
||||||
|
Arc::downgrade(&manager),
|
||||||
|
[1; 32],
|
||||||
|
"192.0.2.1".parse().unwrap(),
|
||||||
|
1,
|
||||||
|
profile,
|
||||||
|
[2; 32],
|
||||||
|
carrier,
|
||||||
|
1,
|
||||||
|
[3; 32],
|
||||||
|
None,
|
||||||
|
if automatic {
|
||||||
|
CarrierClientClass::Bridge
|
||||||
|
} else {
|
||||||
|
CarrierClientClass::Legacy
|
||||||
|
},
|
||||||
|
None,
|
||||||
|
automatic,
|
||||||
|
false,
|
||||||
|
limits,
|
||||||
|
WebTimeoutsConfig::default(),
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
session.configure_conveyor(4);
|
||||||
|
(session, manager)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn claim(session: &Arc<WebSession>, lane: Option<u32>, seq: u64, ack: u64) -> ConveyorClaim {
|
||||||
|
session
|
||||||
|
.claim_conveyor(lane, seq, Some(ack))
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stream(session: &WebSession, carrier: WebCarrier) -> Option<u32> {
|
||||||
|
let mut state = session.state.lock();
|
||||||
|
state.streams.insert(
|
||||||
|
7,
|
||||||
|
StreamState {
|
||||||
|
instance: 1,
|
||||||
|
inbound: VecDeque::new(),
|
||||||
|
receive_window: frame::INITIAL_STREAM_WINDOW,
|
||||||
|
send_credit: 0,
|
||||||
|
read_waker: None,
|
||||||
|
write_waker: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
if carrier.uses_lanes() {
|
||||||
|
state.carrier_lanes.insert(7, CarrierLane::new(7));
|
||||||
|
Some(7)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cleanup(session: Arc<WebSession>, manager: Arc<WebProcessRuntime>) {
|
||||||
|
session.close(SessionCloseReason::ApiClose);
|
||||||
|
session.wait().await;
|
||||||
|
manager.shutdown().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reordered_data_and_window_apply_once_in_sequence_for_both_carriers() {
|
||||||
|
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||||
|
let (session, manager) = fixture(carrier, WebLimitsConfig::default(), false);
|
||||||
|
let lane = stream(&session, carrier);
|
||||||
|
let first = frame::encode(FrameType::Data, 7, b"first");
|
||||||
|
let second = frame::encode(FrameType::Data, 7, b"second");
|
||||||
|
let credit = frame::encode(FrameType::Window, 7, &5u32.to_be_bytes());
|
||||||
|
let second_claim = claim(&session, lane, 2, 0);
|
||||||
|
let credit_claim = claim(&session, lane, 3, 0);
|
||||||
|
let mut second_wait = Box::pin(second_claim.process(&second));
|
||||||
|
let mut credit_wait = Box::pin(credit_claim.process(&credit));
|
||||||
|
assert!(futures::poll!(&mut second_wait).is_pending());
|
||||||
|
assert!(futures::poll!(&mut credit_wait).is_pending());
|
||||||
|
assert!(session.state.lock().streams[&7].inbound.is_empty());
|
||||||
|
assert_eq!(claim(&session, lane, 1, 0).process(&first).await, Ok(1));
|
||||||
|
assert!(futures::poll!(&mut credit_wait).is_pending());
|
||||||
|
assert_eq!(second_wait.await, Ok(2));
|
||||||
|
assert_eq!(credit_wait.await, Ok(3));
|
||||||
|
for (seq, body) in [(1, &first), (2, &second), (3, &credit)] {
|
||||||
|
assert_eq!(claim(&session, lane, seq, 0).process(body).await, Ok(seq));
|
||||||
|
}
|
||||||
|
{
|
||||||
|
let state = session.state.lock();
|
||||||
|
let stream = &state.streams[&7];
|
||||||
|
assert_eq!(stream.inbound.len(), 2);
|
||||||
|
assert_eq!(stream.inbound[0].bytes.as_ref(), b"first");
|
||||||
|
assert_eq!(stream.inbound[1].bytes.as_ref(), b"second");
|
||||||
|
assert_eq!(stream.send_credit, 5);
|
||||||
|
assert_eq!(state.conveyor.channels[&lane].committed, 3);
|
||||||
|
}
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn confirmed_floor_retires_only_acknowledged_history_and_never_replays_it() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
let body = frame::encode(FrameType::Pong, 0, b"same");
|
||||||
|
for seq in 1..=4 {
|
||||||
|
assert_eq!(claim(&session, None, seq, 0).process(&body).await, Ok(seq));
|
||||||
|
}
|
||||||
|
// ACK2 through ACK4 do not constitute confirmation of a lost ACK1.
|
||||||
|
assert_eq!(claim(&session, None, 1, 0).process(&body).await, Ok(1));
|
||||||
|
assert_eq!(claim(&session, None, 5, 4).process(&body).await, Ok(5));
|
||||||
|
assert!(matches!(
|
||||||
|
session.claim_conveyor(None, 1, Some(0)),
|
||||||
|
Err(ConveyorError::Stale)
|
||||||
|
));
|
||||||
|
assert!(!session.state.lock().closed);
|
||||||
|
assert_eq!(session.state.lock().last_up_sequence, 5);
|
||||||
|
assert_eq!(
|
||||||
|
session.state.lock().conveyor.channels[&None]
|
||||||
|
.slots
|
||||||
|
.iter()
|
||||||
|
.flatten()
|
||||||
|
.count(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn digest_mismatch_and_window_violations_are_terminal() {
|
||||||
|
for invalid in [0, 1, 2] {
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
let body = frame::encode(FrameType::Pong, 0, b"first");
|
||||||
|
assert_eq!(claim(&session, None, 1, 0).process(&body).await, Ok(1));
|
||||||
|
if invalid == 0 {
|
||||||
|
let changed = frame::encode(FrameType::Pong, 0, b"changed");
|
||||||
|
assert_eq!(
|
||||||
|
claim(&session, None, 1, 0).process(&changed).await,
|
||||||
|
Err(ConveyorError::Manager(ManagerError::Protocol))
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
let (sequence, confirmed) = if invalid == 1 { (5, 0) } else { (3, 2) };
|
||||||
|
assert!(matches!(
|
||||||
|
session.claim_conveyor(None, sequence, Some(confirmed)),
|
||||||
|
Err(ConveyorError::Manager(ManagerError::Protocol))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
assert!(session.state.lock().closed);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn duplicate_pending_claims_and_cancelled_owners_do_not_consume_or_erase_new_slots() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
let first = claim(&session, None, 2, 0);
|
||||||
|
assert!(matches!(
|
||||||
|
session.claim_conveyor(None, 2, Some(0)),
|
||||||
|
Err(ConveyorError::Manager(ManagerError::Concurrent))
|
||||||
|
));
|
||||||
|
drop(first);
|
||||||
|
assert!(session.state.lock().conveyor.channels.is_empty());
|
||||||
|
let cancelled = claim(&session, None, 2, 0);
|
||||||
|
session.with_state_effects(|state, effects| {
|
||||||
|
effects.notify(state.conveyor.remove(None).unwrap());
|
||||||
|
});
|
||||||
|
let replacement = claim(&session, None, 2, 0);
|
||||||
|
drop(cancelled);
|
||||||
|
assert_eq!(
|
||||||
|
session.state.lock().conveyor.channels[&None].slots[2]
|
||||||
|
.unwrap()
|
||||||
|
.owner,
|
||||||
|
replacement.owner
|
||||||
|
);
|
||||||
|
drop(replacement);
|
||||||
|
assert!(session.state.lock().conveyor.channels.is_empty());
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(start_paused = true)]
|
||||||
|
async fn missing_head_times_out_without_committing_and_returns_waiter_capacity() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
let body = frame::encode(FrameType::Pong, 0, &[]);
|
||||||
|
let tail = claim(&session, None, 2, 0);
|
||||||
|
let mut waiting = Box::pin(tail.process(&body));
|
||||||
|
assert!(futures::poll!(&mut waiting).is_pending());
|
||||||
|
tokio::time::advance(Duration::from_secs(session.timeouts.body_secs)).await;
|
||||||
|
assert_eq!(
|
||||||
|
waiting.await,
|
||||||
|
Err(ConveyorError::Manager(ManagerError::Backpressure))
|
||||||
|
);
|
||||||
|
assert_eq!(session.state.lock().last_up_sequence, 0);
|
||||||
|
drop(tail);
|
||||||
|
let mut permits = Vec::new();
|
||||||
|
while let Some(permit) = manager.try_conveyor_waiter() {
|
||||||
|
permits.push(permit);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
permits.len(),
|
||||||
|
conveyor_waiter_limit(&WebLimitsConfig::default())
|
||||||
|
);
|
||||||
|
drop(permits);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn waiter_ceiling_always_preserves_head_handler_reader_and_bytes() {
|
||||||
|
let defaults = WebLimitsConfig::default();
|
||||||
|
assert_eq!(conveyor_waiter_limit(&defaults), 31);
|
||||||
|
for (handlers, readers, bodies, expected) in [
|
||||||
|
(8, 4, 8, 2),
|
||||||
|
(100, 2, 8, 1),
|
||||||
|
(100, 8, 2, 1),
|
||||||
|
(3, 8, 8, 0),
|
||||||
|
(8, 1, 8, 0),
|
||||||
|
(8, 8, 1, 0),
|
||||||
|
] {
|
||||||
|
let limits = WebLimitsConfig {
|
||||||
|
max_http_handlers: handlers,
|
||||||
|
max_body_readers: readers,
|
||||||
|
max_body_bytes: 32,
|
||||||
|
max_body_bytes_global: bodies * 32,
|
||||||
|
..defaults.clone()
|
||||||
|
};
|
||||||
|
assert_eq!(conveyor_waiter_limit(&limits), expected);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn future_request_saturation_keeps_head_admissible_before_any_body_is_read() {
|
||||||
|
let limits = WebLimitsConfig {
|
||||||
|
max_http_handlers: 8,
|
||||||
|
max_body_readers: 3,
|
||||||
|
max_body_bytes: 32,
|
||||||
|
max_body_bytes_global: 96,
|
||||||
|
..WebLimitsConfig::default()
|
||||||
|
};
|
||||||
|
let (session, manager) = fixture(WebCarrier::HttpsLanes, limits, false);
|
||||||
|
let a = claim(&session, Some(7), 2, 0);
|
||||||
|
let b = claim(&session, Some(8), 2, 0);
|
||||||
|
let a_body = manager.try_body_budget(32).unwrap();
|
||||||
|
let b_body = manager.try_body_budget(32).unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
session.claim_conveyor(Some(9), 2, Some(0)),
|
||||||
|
Err(ConveyorError::Manager(ManagerError::Backpressure))
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
!session
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.conveyor
|
||||||
|
.channels
|
||||||
|
.contains_key(&Some(9))
|
||||||
|
);
|
||||||
|
let head = claim(&session, Some(7), 1, 0);
|
||||||
|
let head_body = manager
|
||||||
|
.try_body_budget(32)
|
||||||
|
.expect("head reader and byte reserve");
|
||||||
|
assert!(!session.state.lock().carrier_lanes.contains_key(&7));
|
||||||
|
drop((a, b, head, a_body, b_body, head_body));
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn mode_is_frozen_and_control_only_ack_cannot_commit_an_automatic_carrier() {
|
||||||
|
for carrier in [WebCarrier::Https, WebCarrier::HttpsLanes] {
|
||||||
|
let (session, manager) = fixture(carrier, WebLimitsConfig::default(), true);
|
||||||
|
let lane = carrier.uses_lanes().then_some(0);
|
||||||
|
let body = frame::encode(FrameType::Pong, 0, &[]);
|
||||||
|
for _ in 0..2 {
|
||||||
|
assert_eq!(
|
||||||
|
claim(&session, lane, 1, 0).process(&body).await,
|
||||||
|
Err(ConveyorError::Manager(ManagerError::Backpressure))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(!session.is_carrier_committed());
|
||||||
|
assert!(matches!(
|
||||||
|
session.claim_conveyor(lane, 2, None),
|
||||||
|
Err(ConveyorError::Mode)
|
||||||
|
));
|
||||||
|
assert!(!session.state.lock().closed);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
assert!(session.claim_conveyor(None, 1, None).unwrap().is_none());
|
||||||
|
assert!(matches!(
|
||||||
|
session.claim_conveyor(None, 1, Some(0)),
|
||||||
|
Err(ConveyorError::Mode)
|
||||||
|
));
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn first_data_on_unknown_lane_is_not_silently_acknowledged() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::HttpsLanes, WebLimitsConfig::default(), false);
|
||||||
|
let body = frame::encode(FrameType::Data, 7, b"not opened");
|
||||||
|
assert_eq!(
|
||||||
|
claim(&session, Some(7), 1, 0).process(&body).await,
|
||||||
|
Err(ConveyorError::Manager(ManagerError::Protocol))
|
||||||
|
);
|
||||||
|
assert!(session.state.lock().closed);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct LockProbe {
|
||||||
|
session: Arc<WebSession>,
|
||||||
|
notified: std::sync::atomic::AtomicBool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Wake for LockProbe {
|
||||||
|
fn wake(self: Arc<Self>) {
|
||||||
|
self.wake_by_ref();
|
||||||
|
}
|
||||||
|
fn wake_by_ref(self: &Arc<Self>) {
|
||||||
|
assert!(
|
||||||
|
self.session.state.try_lock().is_some(),
|
||||||
|
"wake under session lock"
|
||||||
|
);
|
||||||
|
self.notified
|
||||||
|
.store(true, std::sync::atomic::Ordering::Release);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn close_and_lane_retirement_wake_parked_requests_only_after_unlock() {
|
||||||
|
for close in [false, true] {
|
||||||
|
let (session, manager) = fixture(WebCarrier::HttpsLanes, WebLimitsConfig::default(), false);
|
||||||
|
let body = frame::encode(FrameType::Data, 7, b"future");
|
||||||
|
let tail = claim(&session, Some(7), 2, 0);
|
||||||
|
let mut waiting = Box::pin(tail.process(&body));
|
||||||
|
let probe = Arc::new(LockProbe {
|
||||||
|
session: Arc::clone(&session),
|
||||||
|
notified: std::sync::atomic::AtomicBool::new(false),
|
||||||
|
});
|
||||||
|
let waker = Waker::from(Arc::clone(&probe));
|
||||||
|
assert!(matches!(
|
||||||
|
waiting.as_mut().poll(&mut Context::from_waker(&waker)),
|
||||||
|
Poll::Pending
|
||||||
|
));
|
||||||
|
if close {
|
||||||
|
session.close(SessionCloseReason::ApiClose);
|
||||||
|
} else {
|
||||||
|
session.with_state_effects(|state, effects| {
|
||||||
|
session.release_lane_locked(state, effects, 7);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
assert!(probe.notified.load(std::sync::atomic::Ordering::Acquire));
|
||||||
|
assert!(waiting.await.is_err());
|
||||||
|
assert!(session.state.lock().conveyor.channels.is_empty());
|
||||||
|
drop(tail);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn delayed_retry_after_lane_history_retirement_is_request_local() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::HttpsLanes, WebLimitsConfig::default(), false);
|
||||||
|
let lane = stream(&session, WebCarrier::HttpsLanes);
|
||||||
|
let body = frame::encode(FrameType::Window, 7, &1u32.to_be_bytes());
|
||||||
|
for seq in 1..=2 {
|
||||||
|
assert_eq!(
|
||||||
|
claim(&session, lane, seq, seq - 1).process(&body).await,
|
||||||
|
Ok(seq)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
session.with_state_effects(|state, effects| {
|
||||||
|
state.streams.remove(&7);
|
||||||
|
session.release_lane_locked(state, effects, 7);
|
||||||
|
});
|
||||||
|
assert!(matches!(
|
||||||
|
session.claim_conveyor(lane, 2, Some(1)),
|
||||||
|
Err(ConveyorError::Stale)
|
||||||
|
));
|
||||||
|
assert!(!session.state.lock().closed);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn verified_replay_renews_peer_activity_without_reapplying_credit() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
stream(&session, WebCarrier::Https);
|
||||||
|
let body = frame::encode(FrameType::Window, 7, &1u32.to_be_bytes());
|
||||||
|
assert_eq!(claim(&session, None, 1, 0).process(&body).await, Ok(1));
|
||||||
|
let now = std::time::Instant::now();
|
||||||
|
session.state.lock().activity =
|
||||||
|
crate::web::session::activity::SessionActivity::new(now - Duration::from_secs(60));
|
||||||
|
assert_eq!(claim(&session, None, 1, 0).process(&body).await, Ok(1));
|
||||||
|
assert!(session.state.lock().activity.peer_idle(now) < Duration::from_secs(1));
|
||||||
|
assert_eq!(session.state.lock().streams[&7].send_credit, 1);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn full_lane_registry_still_admits_first_control_channel_request() {
|
||||||
|
let limits = WebLimitsConfig {
|
||||||
|
max_tombstones_per_session: 0,
|
||||||
|
..WebLimitsConfig::default()
|
||||||
|
};
|
||||||
|
let (session, manager) = fixture(WebCarrier::HttpsLanes, limits, false);
|
||||||
|
stream(&session, WebCarrier::HttpsLanes);
|
||||||
|
session
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.carrier_lanes
|
||||||
|
.insert(8, CarrierLane::new(8));
|
||||||
|
let body = frame::encode(FrameType::Pong, 0, &[]);
|
||||||
|
assert_eq!(claim(&session, Some(0), 1, 0).process(&body).await, Ok(1));
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn per_channel_memory_estimate_covers_window_notify_and_hash_table_slack() {
|
||||||
|
// The memory-envelope validator reserves 512 additional bytes per bounded lane slot.
|
||||||
|
let key_and_value = std::mem::size_of::<(Option<u32>, Channel)>();
|
||||||
|
let table_slack = key_and_value.div_ceil(7);
|
||||||
|
let notify_allocation = std::mem::size_of::<Notify>() + 2 * std::mem::size_of::<usize>();
|
||||||
|
assert!(key_and_value + table_slack + notify_allocation + 32 <= 512);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||||
|
async fn concurrent_windows_preserve_contiguous_application_under_scheduler_pressure() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
let body = frame::encode(FrameType::Pong, 0, &[]);
|
||||||
|
for window in 0..256u64 {
|
||||||
|
let confirmed = window * 4;
|
||||||
|
let mut tasks = Vec::new();
|
||||||
|
for offset in (1..=4).rev() {
|
||||||
|
let claim = claim(&session, None, confirmed + offset, confirmed);
|
||||||
|
let body = body.clone();
|
||||||
|
tasks.push(tokio::spawn(async move {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
claim.process(&body).await
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
for (task, offset) in tasks.into_iter().zip((1..=4).rev()) {
|
||||||
|
assert_eq!(task.await.unwrap(), Ok(confirmed + offset));
|
||||||
|
}
|
||||||
|
let state = session.state.lock();
|
||||||
|
assert_eq!(state.last_up_sequence, confirmed + 4);
|
||||||
|
assert_eq!(state.conveyor.channels[&None].committed, confirmed + 4);
|
||||||
|
assert_eq!(
|
||||||
|
state.conveyor.channels[&None]
|
||||||
|
.slots
|
||||||
|
.iter()
|
||||||
|
.flatten()
|
||||||
|
.count(),
|
||||||
|
4
|
||||||
|
);
|
||||||
|
}
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn next_head_waits_for_previous_synchronous_uplink_guard_to_drop() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::Https, WebLimitsConfig::default(), false);
|
||||||
|
let body = frame::encode(FrameType::Pong, 0, &[]);
|
||||||
|
let head = claim(&session, None, 1, 0);
|
||||||
|
session
|
||||||
|
.up_active
|
||||||
|
.store(true, std::sync::atomic::Ordering::Release);
|
||||||
|
let mut waiting = Box::pin(head.process(&body));
|
||||||
|
assert!(futures::poll!(&mut waiting).is_pending());
|
||||||
|
session
|
||||||
|
.up_active
|
||||||
|
.store(false, std::sync::atomic::Ordering::Release);
|
||||||
|
head.notify.notify_waiters();
|
||||||
|
assert_eq!(waiting.await, Ok(1));
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn new_lane_future_data_waits_for_open_without_creating_a_backend_early() {
|
||||||
|
let (session, manager) = fixture(WebCarrier::HttpsLanes, WebLimitsConfig::default(), true);
|
||||||
|
let body = frame::encode(FrameType::Data, 7, b"first inner bytes");
|
||||||
|
let future = claim(&session, Some(7), 2, 0);
|
||||||
|
let mut waiting = Box::pin(future.process(&body));
|
||||||
|
assert!(futures::poll!(&mut waiting).is_pending());
|
||||||
|
assert!(session.state.lock().streams.is_empty());
|
||||||
|
assert!(!session.state.lock().carrier_lanes.contains_key(&7));
|
||||||
|
assert_eq!(session.tasks_live(), 0);
|
||||||
|
assert!(!session.is_carrier_committed());
|
||||||
|
let open = frame::encode(FrameType::Open, 7, &[]);
|
||||||
|
assert_eq!(claim(&session, Some(7), 1, 0).process(&open).await, Ok(1));
|
||||||
|
assert!(session.is_carrier_committed());
|
||||||
|
assert_eq!(waiting.await, Ok(2));
|
||||||
|
assert_eq!(
|
||||||
|
session.state.lock().streams[&7].inbound[0].bytes,
|
||||||
|
b"first inner bytes"[..]
|
||||||
|
);
|
||||||
|
cleanup(session, manager).await;
|
||||||
|
}
|
||||||
@@ -20,6 +20,17 @@ impl WebSession {
|
|||||||
lane_id: u32,
|
lane_id: u32,
|
||||||
sequence: u64,
|
sequence: u64,
|
||||||
body: &[u8],
|
body: &[u8],
|
||||||
|
) -> Result<u64, ManagerError> {
|
||||||
|
self.process_up_lane_inner(lane_id, sequence, body, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Applies one lane batch only while its optional conveyor claim still owns the turn.
|
||||||
|
pub(super) fn process_up_lane_inner(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
lane_id: u32,
|
||||||
|
sequence: u64,
|
||||||
|
body: &[u8],
|
||||||
|
claim: Option<(&super::conveyor::ConveyorClaim, TokenHash)>,
|
||||||
) -> Result<u64, ManagerError> {
|
) -> Result<u64, ManagerError> {
|
||||||
if self.carrier() != WebCarrier::HttpsLanes || lane_id > frame::MAX_STREAM_ID {
|
if self.carrier() != WebCarrier::HttpsLanes || lane_id > frame::MAX_STREAM_ID {
|
||||||
return Err(ManagerError::Protocol);
|
return Err(ManagerError::Protocol);
|
||||||
@@ -42,7 +53,8 @@ impl WebSession {
|
|||||||
self.close(SessionCloseReason::Protocol);
|
self.close(SessionCloseReason::Protocol);
|
||||||
return Err(ManagerError::Protocol);
|
return Err(ManagerError::Protocol);
|
||||||
}
|
}
|
||||||
let digest: TokenHash = Sha256::digest(body).into();
|
let digest: TokenHash =
|
||||||
|
claim.map_or_else(|| Sha256::digest(body).into(), |(_, digest)| digest);
|
||||||
let mut opened = Vec::new();
|
let mut opened = Vec::new();
|
||||||
let mut committed = false;
|
let mut committed = false;
|
||||||
let mut healthy = None;
|
let mut healthy = None;
|
||||||
@@ -53,9 +65,13 @@ impl WebSession {
|
|||||||
return Err(ManagerError::Closed);
|
return Err(ManagerError::Closed);
|
||||||
}
|
}
|
||||||
self.ensure_carrier_active_locked(&state)?;
|
self.ensure_carrier_active_locked(&state)?;
|
||||||
|
if let Some((claim, _)) = claim {
|
||||||
|
claim.validate_locked(&state, &digest)?;
|
||||||
|
}
|
||||||
let new_lane = !state.carrier_lanes.contains_key(&lane_id);
|
let new_lane = !state.carrier_lanes.contains_key(&lane_id);
|
||||||
if new_lane {
|
if new_lane {
|
||||||
if lane_id != 0
|
if claim.is_none()
|
||||||
|
&& lane_id != 0
|
||||||
&& frames
|
&& frames
|
||||||
.first()
|
.first()
|
||||||
.is_some_and(|value| value.frame_type != FrameType::Open)
|
.is_some_and(|value| value.frame_type != FrameType::Open)
|
||||||
@@ -190,6 +206,7 @@ impl WebSession {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if applied {
|
if applied {
|
||||||
|
state.conveyor.commit(Some(lane_id), sequence, digest);
|
||||||
(committed, healthy) = self.record_uplink_progress_locked(&mut state, progress);
|
(committed, healthy) = self.record_uplink_progress_locked(&mut state, progress);
|
||||||
}
|
}
|
||||||
applied.then_some(sequence).ok_or(ManagerError::Closed)
|
applied.then_some(sequence).ok_or(ManagerError::Closed)
|
||||||
|
|||||||
@@ -469,12 +469,16 @@ impl WebSession {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Retires exact lane queues and ordering history, deferring every waiter notification.
|
||||||
pub(super) fn release_lane_locked(
|
pub(super) fn release_lane_locked(
|
||||||
&self,
|
&self,
|
||||||
state: &mut SessionState,
|
state: &mut SessionState,
|
||||||
effects: &mut DeferredSessionEffects,
|
effects: &mut DeferredSessionEffects,
|
||||||
lane_id: u32,
|
lane_id: u32,
|
||||||
) {
|
) {
|
||||||
|
if let Some(notify) = state.conveyor.remove(Some(lane_id)) {
|
||||||
|
effects.notify(notify);
|
||||||
|
}
|
||||||
let Some(mut lane) = state.carrier_lanes.remove(&lane_id) else {
|
let Some(mut lane) = state.carrier_lanes.remove(&lane_id) else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -279,6 +279,9 @@ impl WebSession {
|
|||||||
let recovery_closed_before_commit = self.recovery && !state.recovery_committed;
|
let recovery_closed_before_commit = self.recovery && !state.recovery_committed;
|
||||||
state.close_requested = Some(reason);
|
state.close_requested = Some(reason);
|
||||||
state.closed = true;
|
state.closed = true;
|
||||||
|
for notify in state.conveyor.clear() {
|
||||||
|
effects.notify(notify);
|
||||||
|
}
|
||||||
if reason == SessionCloseReason::CarrierSuperseded {
|
if reason == SessionCloseReason::CarrierSuperseded {
|
||||||
state.negotiation_phase = SessionNegotiationPhase::Superseded;
|
state.negotiation_phase = SessionNegotiationPhase::Superseded;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ impl WebSession {
|
|||||||
sequence,
|
sequence,
|
||||||
body,
|
body,
|
||||||
WebSessionLifecycleObservation::HttpActivityAfterGap,
|
WebSessionLifecycleObservation::HttpActivityAfterGap,
|
||||||
|
None,
|
||||||
)?;
|
)?;
|
||||||
if self.automatic_carrier && !progressed && !self.is_carrier_committed() {
|
if self.automatic_carrier && !progressed && !self.is_carrier_committed() {
|
||||||
return Err(ManagerError::Backpressure);
|
return Err(ManagerError::Backpressure);
|
||||||
@@ -56,15 +57,37 @@ impl WebSession {
|
|||||||
sequence,
|
sequence,
|
||||||
body,
|
body,
|
||||||
WebSessionLifecycleObservation::WebSocketActivityAfterGap,
|
WebSessionLifecycleObservation::WebSocketActivityAfterGap,
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
.map(|(_, progress)| progress)
|
.map(|(_, progress)| progress)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Applies an admitted conveyor head without weakening the carrier commit gate.
|
||||||
|
pub(super) fn process_claimed_up(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
sequence: u64,
|
||||||
|
body: &[u8],
|
||||||
|
claim: &super::conveyor::ConveyorClaim,
|
||||||
|
digest: TokenHash,
|
||||||
|
) -> Result<u64, ManagerError> {
|
||||||
|
let (acknowledged, progressed) = self.process_up_inner(
|
||||||
|
sequence,
|
||||||
|
body,
|
||||||
|
WebSessionLifecycleObservation::HttpActivityAfterGap,
|
||||||
|
Some((claim, digest)),
|
||||||
|
)?;
|
||||||
|
if self.automatic_carrier && !progressed && !self.is_carrier_committed() {
|
||||||
|
return Err(ManagerError::Backpressure);
|
||||||
|
}
|
||||||
|
Ok(acknowledged)
|
||||||
|
}
|
||||||
|
|
||||||
fn process_up_inner(
|
fn process_up_inner(
|
||||||
self: &Arc<Self>,
|
self: &Arc<Self>,
|
||||||
sequence: u64,
|
sequence: u64,
|
||||||
body: &[u8],
|
body: &[u8],
|
||||||
observation: WebSessionLifecycleObservation,
|
observation: WebSessionLifecycleObservation,
|
||||||
|
claim: Option<(&super::conveyor::ConveyorClaim, TokenHash)>,
|
||||||
) -> Result<(u64, bool), ManagerError> {
|
) -> Result<(u64, bool), ManagerError> {
|
||||||
if !self.carrier().is_multiplexed() {
|
if !self.carrier().is_multiplexed() {
|
||||||
return Err(ManagerError::Protocol);
|
return Err(ManagerError::Protocol);
|
||||||
@@ -95,7 +118,8 @@ impl WebSession {
|
|||||||
self.close(SessionCloseReason::Protocol);
|
self.close(SessionCloseReason::Protocol);
|
||||||
return Err(ManagerError::Protocol);
|
return Err(ManagerError::Protocol);
|
||||||
}
|
}
|
||||||
let digest: TokenHash = Sha256::digest(body).into();
|
let digest: TokenHash =
|
||||||
|
claim.map_or_else(|| Sha256::digest(body).into(), |(_, digest)| digest);
|
||||||
let mut opened = Vec::new();
|
let mut opened = Vec::new();
|
||||||
let mut committed = false;
|
let mut committed = false;
|
||||||
let mut healthy = None;
|
let mut healthy = None;
|
||||||
@@ -106,6 +130,9 @@ impl WebSession {
|
|||||||
return Err(ManagerError::Closed);
|
return Err(ManagerError::Closed);
|
||||||
}
|
}
|
||||||
self.ensure_carrier_active_locked(&state)?;
|
self.ensure_carrier_active_locked(&state)?;
|
||||||
|
if let Some((claim, _)) = claim {
|
||||||
|
claim.validate_locked(&state, &digest)?;
|
||||||
|
}
|
||||||
if sequence == state.last_up_sequence && sequence != 0 {
|
if sequence == state.last_up_sequence && sequence != 0 {
|
||||||
return if bool::from(state.last_up_digest.ct_eq(&digest)) {
|
return if bool::from(state.last_up_digest.ct_eq(&digest)) {
|
||||||
self.touch_peer_locked(&mut state, Instant::now(), observation);
|
self.touch_peer_locked(&mut state, Instant::now(), observation);
|
||||||
@@ -155,6 +182,7 @@ impl WebSession {
|
|||||||
} else {
|
} else {
|
||||||
state.last_up_sequence = sequence;
|
state.last_up_sequence = sequence;
|
||||||
state.last_up_digest = digest;
|
state.last_up_digest = digest;
|
||||||
|
state.conveyor.commit(None, sequence, digest);
|
||||||
(committed, healthy) = self.record_uplink_progress_locked(&mut state, progress);
|
(committed, healthy) = self.record_uplink_progress_locked(&mut state, progress);
|
||||||
Ok((sequence, progress.any()))
|
Ok((sequence, progress.any()))
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user